diff --git a/configs/test_cam1_ppe11_yolo_debug.json b/configs/test_cam1_ppe11_yolo_debug.json index 0bf2b33..e1d4fa0 100644 --- a/configs/test_cam1_ppe11_yolo_debug.json +++ b/configs/test_cam1_ppe11_yolo_debug.json @@ -1,4 +1,8 @@ { + "global": { + "metrics_port": 9001, + "web_root": "web" + }, "queue": { "size": 8, "strategy": "drop_oldest" @@ -45,6 +49,7 @@ "model_w": 768, "model_h": 768, "num_classes": 11, + "v8_box_format": "xyxy", "conf": 0.45, "nms": 0.45, "class_filter": [3, 6], diff --git a/docs/命令.md b/docs/命令.md index 5b5a011..f77c779 100644 --- a/docs/命令.md +++ b/docs/命令.md @@ -6,16 +6,17 @@ ffmpeg -f dshow -video_size 1280x720 -vcodec mjpeg -i video="1080P USB Camera" -c:v libx264 -preset ultrafast -pix_fmt yuv420p -f rtsp rtsp://localhost:8554/cam cmake -S . -B build \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TESTS=OFF \ - -DBUILD_SAMPLES=ON \ - -DRK3588_ENABLE_FFMPEG=ON \ - -DRK3588_ENABLE_MPP=ON \ - -DRK3588_ENABLE_RGA=ON \ - -DRK3588_ENABLE_ZLMEDIAKIT=ON \ - -DRK3588_ENABLE_RKNN=ON \ - -DRK_ZLMK_API_LIB_PATH=$PWD/third_party/rknpu2/examples/3rdparty/zlmediakit/aarch64/libmk_api.so \ - -DRK_ZLMEDIAKIT_INCLUDE_DIR=$PWD/third_party/rknpu2/examples/3rdparty/zlmediakit/include + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TESTS=OFF \ + -DBUILD_SAMPLES=ON \ + -DRK3588_ENABLE_FFMPEG=ON \ + -DRK3588_ENABLE_MPP=ON \ + -DRK_MPP_LIB_PATH=/usr/lib/aarch64-linux-gnu/librockchip_mpp.so.0 \ + -DRK3588_ENABLE_RGA=ON \ + -DRK3588_ENABLE_ZLMEDIAKIT=ON \ + -DRK3588_ENABLE_RKNN=ON \ + -DRK_ZLMK_API_LIB_PATH=$PWD/third_party/rknpu2/examples/3rdparty/zlmediakit/aarch64/libmk_api.so \ + -DRK_ZLMEDIAKIT_INCLUDE_DIR=$PWD/third_party/rknpu2/examples/3rdparty/zlmediakit/include cmake --build build -j$(nproc) diff --git a/plugins/ai_yolo/ai_yolo_node.cpp b/plugins/ai_yolo/ai_yolo_node.cpp index 0197c78..6f397c4 100644 --- a/plugins/ai_yolo/ai_yolo_node.cpp +++ b/plugins/ai_yolo/ai_yolo_node.cpp @@ -33,6 +33,7 @@ const int kAnchor1[6] = {30, 61, 62, 45, 59, 119}; const int kAnchor2[6] = {116, 90, 156, 198, 373, 326}; enum class YoloVersion { V5, V8 }; +enum class V8BoxFormat { Auto, CxCyWh, XyXy }; const char* kCocoLabels[kObjClassNum] = { "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", @@ -238,6 +239,68 @@ struct V8LayoutInfo { bool channels_first = true; // true: CxN, false: NxC }; +float ScoreBoxCandidate(float x, float y, float w, float h, int model_w, int model_h) { + float s = 0.0f; + if (w > 0.0f && h > 0.0f) s += 3.0f; + if (w <= model_w * 1.2f) s += 1.0f; + if (h <= model_h * 1.2f) s += 1.0f; + if (x >= -model_w * 0.1f) s += 1.0f; + if (y >= -model_h * 0.1f) s += 1.0f; + if ((x + w) <= model_w * 1.2f) s += 1.0f; + if ((y + h) <= model_h * 1.2f) s += 1.0f; + return s; +} + +bool SeemsNormalized(float a, float b, float c, float d) { + auto in_range = [](float v) { return v >= -0.05f && v <= 2.5f; }; + return in_range(a) && in_range(b) && in_range(c) && in_range(d); +} + +void DecodeV8Box(float a, float b, float c, float d, int model_w, int model_h, V8BoxFormat fmt, + float& out_x, float& out_y, float& out_w, float& out_h) { + if (SeemsNormalized(a, b, c, d)) { + a *= static_cast(model_w); + b *= static_cast(model_h); + c *= static_cast(model_w); + d *= static_cast(model_h); + } + + auto decode_cxcywh = [&](float& x, float& y, float& w, float& h) { + x = a - c / 2.0f; + y = b - d / 2.0f; + w = c; + h = d; + }; + auto decode_xyxy = [&](float& x, float& y, float& w, float& h) { + x = a; + y = b; + w = c - a; + h = d - b; + }; + + if (fmt == V8BoxFormat::CxCyWh) { + decode_cxcywh(out_x, out_y, out_w, out_h); + return; + } + if (fmt == V8BoxFormat::XyXy) { + decode_xyxy(out_x, out_y, out_w, out_h); + return; + } + + float x1 = 0.0f, y1 = 0.0f, w1 = 0.0f, h1 = 0.0f; + float x2 = 0.0f, y2 = 0.0f, w2 = 0.0f, h2 = 0.0f; + decode_cxcywh(x1, y1, w1, h1); + decode_xyxy(x2, y2, w2, h2); + + const float s1 = ScoreBoxCandidate(x1, y1, w1, h1, model_w, model_h); + const float s2 = ScoreBoxCandidate(x2, y2, w2, h2, model_w, model_h); + if (s2 > s1) { + out_x = x2; out_y = y2; out_w = w2; out_h = h2; + } else { + out_x = x1; out_y = y1; out_w = w1; out_h = h1; + } +} + V8LayoutInfo ResolveV8Layout(const std::vector& dims, size_t byte_size, rknn_tensor_type type, int num_classes, int model_h, int model_w) { @@ -319,7 +382,7 @@ int ProcessOutputV8(float* output, int num_boxes, int num_classes, int model_h, int model_w, std::vector& boxes, std::vector& obj_probs, std::vector& class_ids, float conf_thresh, - bool channels_first) { + bool channels_first, V8BoxFormat box_format) { int valid_count = 0; const int num_channels = 4 + num_classes; @@ -336,13 +399,12 @@ int ProcessOutputV8(float* output, int num_boxes, int num_classes, } if (max_score >= conf_thresh) { - float cx = channels_first ? output[0 * num_boxes + i] : output[i * num_channels + 0]; - float cy = channels_first ? output[1 * num_boxes + i] : output[i * num_channels + 1]; - float w = channels_first ? output[2 * num_boxes + i] : output[i * num_channels + 2]; - float h = channels_first ? output[3 * num_boxes + i] : output[i * num_channels + 3]; - - float x1 = cx - w / 2.0f; - float y1 = cy - h / 2.0f; + const float a = channels_first ? output[0 * num_boxes + i] : output[i * num_channels + 0]; + const float b = channels_first ? output[1 * num_boxes + i] : output[i * num_channels + 1]; + const float c = channels_first ? output[2 * num_boxes + i] : output[i * num_channels + 2]; + const float d = channels_first ? output[3 * num_boxes + i] : output[i * num_channels + 3]; + float x1 = 0.0f, y1 = 0.0f, w = 0.0f, h = 0.0f; + DecodeV8Box(a, b, c, d, model_w, model_h, box_format, x1, y1, w, h); boxes.push_back(x1); boxes.push_back(y1); @@ -361,7 +423,7 @@ int ProcessOutputV8Int8(int8_t* output, int num_boxes, int num_classes, int model_h, int model_w, std::vector& boxes, std::vector& obj_probs, std::vector& class_ids, float conf_thresh, - int32_t zp, float scale, bool channels_first) { + int32_t zp, float scale, bool channels_first, V8BoxFormat box_format) { int valid_count = 0; int8_t thresh_i8 = QuantizeF32ToAffine(conf_thresh, zp, scale); const int num_channels = 4 + num_classes; @@ -379,18 +441,17 @@ int ProcessOutputV8Int8(int8_t* output, int num_boxes, int num_classes, } if (max_score_i8 >= thresh_i8) { - float cx = DequantizeAffineToF32( + float a = DequantizeAffineToF32( channels_first ? output[0 * num_boxes + i] : output[i * num_channels + 0], zp, scale); - float cy = DequantizeAffineToF32( + float b = DequantizeAffineToF32( channels_first ? output[1 * num_boxes + i] : output[i * num_channels + 1], zp, scale); - float w = DequantizeAffineToF32( + float c = DequantizeAffineToF32( channels_first ? output[2 * num_boxes + i] : output[i * num_channels + 2], zp, scale); - float h = DequantizeAffineToF32( + float d = DequantizeAffineToF32( channels_first ? output[3 * num_boxes + i] : output[i * num_channels + 3], zp, scale); float max_score = DequantizeAffineToF32(max_score_i8, zp, scale); - - float x1 = cx - w / 2.0f; - float y1 = cy - h / 2.0f; + float x1 = 0.0f, y1 = 0.0f, w = 0.0f, h = 0.0f; + DecodeV8Box(a, b, c, d, model_w, model_h, box_format, x1, y1, w, h); boxes.push_back(x1); boxes.push_back(y1); @@ -420,6 +481,16 @@ public: model_input_w_ = config.ValueOr("model_w", 640); model_input_h_ = config.ValueOr("model_h", 640); num_classes_ = config.ValueOr("num_classes", 80); + { + const std::string bf = config.ValueOr("v8_box_format", "auto"); + if (bf == "xyxy") { + v8_box_format_ = V8BoxFormat::XyXy; + } else if (bf == "cxcywh") { + v8_box_format_ = V8BoxFormat::CxCyWh; + } else { + v8_box_format_ = V8BoxFormat::Auto; + } + } if (const SimpleJson* dbg = config.Find("debug"); dbg && dbg->IsObject()) { stats_log_ = dbg->ValueOr("stats", stats_log_); @@ -658,13 +729,26 @@ private: model_input_h_, model_input_w_); const int num_boxes = layout.num_boxes; if (num_boxes <= 0) return; + if (debug_det_ && processed_ < 5) { + std::string dims_s; + for (size_t di = 0; di < outputs[0].dims.size(); ++di) { + dims_s += (di == 0 ? "[" : ","); + dims_s += std::to_string(outputs[0].dims[di]); + } + dims_s += "]"; + LogInfo("[ai_yolo] v8 out type=" + std::to_string(static_cast(outputs[0].type)) + + " size=" + std::to_string(outputs[0].size) + + " dims=" + dims_s + + " num_boxes=" + std::to_string(num_boxes) + + " layout=" + std::string(layout.channels_first ? "CxN" : "NxC")); + } if (outputs[0].type == RKNN_TENSOR_FLOAT32) { valid_count = ProcessOutputV8(reinterpret_cast(const_cast(outputs[0].data)), num_boxes, num_classes_, model_input_h_, model_input_w_, boxes, obj_probs, class_ids, conf_thresh_, - layout.channels_first); + layout.channels_first, v8_box_format_); } else if (outputs[0].type == RKNN_TENSOR_FLOAT16) { // Convert FP16 to FP32 size_t num_elements = outputs[0].size / sizeof(uint16_t); @@ -677,14 +761,14 @@ private: num_boxes, num_classes_, model_input_h_, model_input_w_, boxes, obj_probs, class_ids, conf_thresh_, - layout.channels_first); + layout.channels_first, v8_box_format_); } else { valid_count = ProcessOutputV8Int8(reinterpret_cast(const_cast(outputs[0].data)), num_boxes, num_classes_, model_input_h_, model_input_w_, boxes, obj_probs, class_ids, conf_thresh_, outputs[0].zp, outputs[0].scale, - layout.channels_first); + layout.channels_first, v8_box_format_); } } @@ -780,13 +864,26 @@ private: model_input_h_, model_input_w_); const int num_boxes = layout.num_boxes; if (num_boxes <= 0) return; + if (debug_det_ && processed_ < 5) { + std::string dims_s; + for (size_t di = 0; di < outputs[0].dims.size(); ++di) { + dims_s += (di == 0 ? "[" : ","); + dims_s += std::to_string(outputs[0].dims[di]); + } + dims_s += "]"; + LogInfo("[ai_yolo] v8 out(type copy) type=" + std::to_string(static_cast(outputs[0].type)) + + " size=" + std::to_string(outputs[0].data.size()) + + " dims=" + dims_s + + " num_boxes=" + std::to_string(num_boxes) + + " layout=" + std::string(layout.channels_first ? "CxN" : "NxC")); + } if (outputs[0].type == RKNN_TENSOR_FLOAT32) { valid_count = ProcessOutputV8(reinterpret_cast(outputs[0].data.data()), num_boxes, num_classes_, model_input_h_, model_input_w_, boxes, obj_probs, class_ids, conf_thresh_, - layout.channels_first); + layout.channels_first, v8_box_format_); } else if (outputs[0].type == RKNN_TENSOR_FLOAT16) { // Convert FP16 to FP32 size_t num_elements = outputs[0].data.size() / sizeof(uint16_t); @@ -799,14 +896,14 @@ private: num_boxes, num_classes_, model_input_h_, model_input_w_, boxes, obj_probs, class_ids, conf_thresh_, - layout.channels_first); + layout.channels_first, v8_box_format_); } else { valid_count = ProcessOutputV8Int8(reinterpret_cast(outputs[0].data.data()), num_boxes, num_classes_, model_input_h_, model_input_w_, boxes, obj_probs, class_ids, conf_thresh_, outputs[0].zp, outputs[0].scale, - layout.channels_first); + layout.channels_first, v8_box_format_); } } @@ -876,6 +973,7 @@ private: int model_input_w_ = 640; int model_input_h_ = 640; int num_classes_ = 80; + V8BoxFormat v8_box_format_ = V8BoxFormat::Auto; YoloVersion yolo_version_ = YoloVersion::V8; bool auto_detect_version_ = false; std::set class_filter_;