diff --git a/configs/test_cam1_ppe11_yolo_debug.json b/configs/test_cam1_ppe11_yolo_debug.json new file mode 100644 index 0000000..0bf2b33 --- /dev/null +++ b/configs/test_cam1_ppe11_yolo_debug.json @@ -0,0 +1,129 @@ +{ + "queue": { + "size": 8, + "strategy": "drop_oldest" + }, + "graphs": [ + { + "name": "cam_ppe11_yolo_debug", + "nodes": [ + { + "id": "in_cam1", + "type": "input_rtsp", + "role": "source", + "enable": true, + "url": "rtsp://10.0.0.49:8554/cam", + "fps": 30, + "width": 1280, + "height": 720, + "use_mpp": true, + "use_ffmpeg": false, + "force_tcp": true, + "reconnect_sec": 5, + "reconnect_backoff_max_sec": 30 + }, + { + "id": "pre_cam1", + "type": "preprocess", + "role": "filter", + "enable": true, + "dst_w": 768, + "dst_h": 768, + "dst_format": "rgb", + "dst_packed": true, + "keep_ratio": false, + "rga_gate": "cam_ppe11_yolo_debug", + "use_rga": true + }, + { + "id": "yolo_cam1", + "type": "ai_yolo", + "role": "filter", + "enable": true, + "model_path": "./models/best-768.rknn", + "model_version": "v8", + "model_w": 768, + "model_h": 768, + "num_classes": 11, + "conf": 0.45, + "nms": 0.45, + "class_filter": [3, 6], + "debug": { + "stats": true, + "stats_interval": 30, + "detections": true + } + }, + { + "id": "osd_cam1", + "type": "osd", + "role": "filter", + "enable": true, + "draw_bbox": true, + "draw_text": true, + "draw_face_det": false, + "draw_face_bbox": false, + "line_width": 2, + "font_scale": 1, + "use_rga_bbox": false, + "labels": [ + "helmet", + "gloves", + "vest", + "boots", + "goggles", + "none", + "Person", + "no_helmet", + "no_goggle", + "no_gloves", + "no_boots" + ] + }, + { + "id": "post_cam1", + "type": "preprocess", + "role": "filter", + "enable": true, + "dst_w": 1280, + "dst_h": 720, + "dst_format": "nv12", + "keep_ratio": false, + "rga_gate": "cam_ppe11_yolo_debug", + "use_rga": true + }, + { + "id": "pub_cam1", + "type": "publish", + "role": "sink", + "enable": true, + "codec": "h264", + "fps": 30, + "gop": 60, + "bitrate_kbps": 2000, + "use_mpp": true, + "use_ffmpeg_mux": true, + "outputs": [ + { + "proto": "hls", + "path": "./web/hls/cam1_debug/index.m3u8", + "segment_sec": 2 + }, + { + "proto": "rtsp_server", + "port": 8556, + "path": "/live/cam1_debug" + } + ] + } + ], + "edges": [ + ["in_cam1", "pre_cam1"], + ["pre_cam1", "yolo_cam1"], + ["yolo_cam1", "osd_cam1"], + ["osd_cam1", "post_cam1"], + ["post_cam1", "pub_cam1"] + ] + } + ] +} diff --git a/plugins/ai_yolo/ai_yolo_node.cpp b/plugins/ai_yolo/ai_yolo_node.cpp index 47c9e9c..0197c78 100644 --- a/plugins/ai_yolo/ai_yolo_node.cpp +++ b/plugins/ai_yolo/ai_yolo_node.cpp @@ -212,18 +212,123 @@ int ProcessFeatureMapV5(int8_t* input, const int* anchor, int grid_h, int grid_w return valid_count; } +uint32_t TensorTypeSizeBytes(rknn_tensor_type t) { + switch (t) { + case RKNN_TENSOR_INT8: + case RKNN_TENSOR_UINT8: + return 1; + case RKNN_TENSOR_FLOAT16: + return 2; + case RKNN_TENSOR_FLOAT32: + return 4; + default: + return 1; + } +} + +int DefaultV8NumBoxes(int model_h, int model_w) { + if (model_h <= 0 || model_w <= 0) return 0; + return (model_h / 8) * (model_w / 8) + + (model_h / 16) * (model_w / 16) + + (model_h / 32) * (model_w / 32); +} + +struct V8LayoutInfo { + int num_boxes = 0; + bool channels_first = true; // true: CxN, false: NxC +}; + +V8LayoutInfo ResolveV8Layout(const std::vector& dims, size_t byte_size, + rknn_tensor_type type, int num_classes, + int model_h, int model_w) { + V8LayoutInfo info; + const int num_channels = 4 + num_classes; + if (num_channels <= 0) return info; + + const uint32_t elem_bytes = TensorTypeSizeBytes(type); + const size_t total_elems = elem_bytes > 0 ? (byte_size / elem_bytes) : 0; + const size_t max_boxes_from_data = static_cast(num_channels) > 0 + ? (total_elems / static_cast(num_channels)) + : 0; + + int ch_idx = -1; + for (size_t i = 0; i < dims.size(); ++i) { + if (dims[i] == static_cast(num_channels)) { + ch_idx = static_cast(i); + break; + } + } + + if (ch_idx >= 0 && total_elems >= static_cast(num_channels)) { + info.num_boxes = static_cast(max_boxes_from_data); + + int prev_non1 = 1; + for (int i = ch_idx - 1; i >= 0; --i) { + if (dims[static_cast(i)] > 1U) { + prev_non1 = static_cast(dims[static_cast(i)]); + break; + } + } + int next_non1 = 1; + for (size_t i = static_cast(ch_idx + 1); i < dims.size(); ++i) { + if (dims[i] > 1U) { + next_non1 = static_cast(dims[i]); + break; + } + } + + if (next_non1 > 1 && prev_non1 <= 1) { + info.channels_first = true; + } else if (prev_non1 > 1 && next_non1 <= 1) { + info.channels_first = false; + } else if (next_non1 > 1 && prev_non1 > 1) { + info.channels_first = next_non1 >= prev_non1; + } else { + info.channels_first = true; + } + } else if (dims.size() >= 3) { + // Compatibility with old rank-3 assumptions. + if (dims[1] == static_cast(num_channels)) { + info.num_boxes = static_cast(dims[2]); + info.channels_first = true; + } else if (dims[2] == static_cast(num_channels)) { + info.num_boxes = static_cast(dims[1]); + info.channels_first = false; + } + } + + if (info.num_boxes <= 0 && max_boxes_from_data > 0) { + info.num_boxes = static_cast(max_boxes_from_data); + } + if (info.num_boxes <= 0) { + info.num_boxes = DefaultV8NumBoxes(model_h, model_w); + } + if (info.num_boxes <= 0) { + info.num_boxes = 8400; + } + + if (max_boxes_from_data > 0 && static_cast(info.num_boxes) > max_boxes_from_data) { + info.num_boxes = static_cast(max_boxes_from_data); + } + if (info.num_boxes < 0) info.num_boxes = 0; + return info; +} + // YOLOv8 output processing (anchor-free, single output tensor) int ProcessOutputV8(float* output, int num_boxes, int num_classes, int model_h, int model_w, std::vector& boxes, std::vector& obj_probs, - std::vector& class_ids, float conf_thresh) { + std::vector& class_ids, float conf_thresh, + bool channels_first) { int valid_count = 0; + const int num_channels = 4 + num_classes; for (int i = 0; i < num_boxes; ++i) { float max_score = 0.0f; int max_cls_id = 0; for (int c = 0; c < num_classes; ++c) { - float score = output[(4 + c) * num_boxes + i]; + float score = channels_first ? output[(4 + c) * num_boxes + i] + : output[i * num_channels + (4 + c)]; if (score > max_score) { max_score = score; max_cls_id = c; @@ -231,10 +336,10 @@ int ProcessOutputV8(float* output, int num_boxes, int num_classes, } if (max_score >= conf_thresh) { - float cx = output[0 * num_boxes + i]; - float cy = output[1 * num_boxes + i]; - float w = output[2 * num_boxes + i]; - float h = output[3 * num_boxes + i]; + float cx = channels_first ? output[0 * num_boxes + i] : output[i * num_channels + 0]; + float cy = channels_first ? output[1 * num_boxes + i] : output[i * num_channels + 1]; + float w = channels_first ? output[2 * num_boxes + i] : output[i * num_channels + 2]; + float h = channels_first ? output[3 * num_boxes + i] : output[i * num_channels + 3]; float x1 = cx - w / 2.0f; float y1 = cy - h / 2.0f; @@ -256,15 +361,17 @@ int ProcessOutputV8Int8(int8_t* output, int num_boxes, int num_classes, int model_h, int model_w, std::vector& boxes, std::vector& obj_probs, std::vector& class_ids, float conf_thresh, - int32_t zp, float scale) { + int32_t zp, float scale, bool channels_first) { int valid_count = 0; int8_t thresh_i8 = QuantizeF32ToAffine(conf_thresh, zp, scale); + const int num_channels = 4 + num_classes; for (int i = 0; i < num_boxes; ++i) { int8_t max_score_i8 = -128; int max_cls_id = 0; for (int c = 0; c < num_classes; ++c) { - int8_t score = output[(4 + c) * num_boxes + i]; + int8_t score = channels_first ? output[(4 + c) * num_boxes + i] + : output[i * num_channels + (4 + c)]; if (score > max_score_i8) { max_score_i8 = score; max_cls_id = c; @@ -272,10 +379,14 @@ int ProcessOutputV8Int8(int8_t* output, int num_boxes, int num_classes, } if (max_score_i8 >= thresh_i8) { - float cx = DequantizeAffineToF32(output[0 * num_boxes + i], zp, scale); - float cy = DequantizeAffineToF32(output[1 * num_boxes + i], zp, scale); - float w = DequantizeAffineToF32(output[2 * num_boxes + i], zp, scale); - float h = DequantizeAffineToF32(output[3 * num_boxes + i], zp, scale); + float cx = DequantizeAffineToF32( + channels_first ? output[0 * num_boxes + i] : output[i * num_channels + 0], zp, scale); + float cy = DequantizeAffineToF32( + channels_first ? output[1 * num_boxes + i] : output[i * num_channels + 1], zp, scale); + float w = DequantizeAffineToF32( + channels_first ? output[2 * num_boxes + i] : output[i * num_channels + 2], zp, scale); + float h = DequantizeAffineToF32( + channels_first ? output[3 * num_boxes + i] : output[i * num_channels + 3], zp, scale); float max_score = DequantizeAffineToF32(max_score_i8, zp, scale); float x1 = cx - w / 2.0f; @@ -539,29 +650,21 @@ private: outputs[2].zp, outputs[2].scale); valid_count = cnt0 + cnt1 + cnt2; } else { - if (outputs.empty()) return; + if (outputs.empty()) return; if (!outputs[0].data || outputs[0].size == 0) return; - int num_boxes = 0; - int num_channels = 4 + num_classes_; - - if (outputs[0].dims.size() >= 3) { - if (outputs[0].dims[1] == static_cast(num_channels)) { - num_boxes = static_cast(outputs[0].dims[2]); - } else if (outputs[0].dims[2] == static_cast(num_channels)) { - num_boxes = static_cast(outputs[0].dims[1]); - } else { - num_boxes = 8400; - } - } else { - num_boxes = static_cast(outputs[0].size) / num_channels; - } + const V8LayoutInfo layout = ResolveV8Layout(outputs[0].dims, outputs[0].size, + outputs[0].type, num_classes_, + model_input_h_, model_input_w_); + const int num_boxes = layout.num_boxes; + if (num_boxes <= 0) return; if (outputs[0].type == RKNN_TENSOR_FLOAT32) { valid_count = ProcessOutputV8(reinterpret_cast(const_cast(outputs[0].data)), num_boxes, num_classes_, model_input_h_, model_input_w_, - boxes, obj_probs, class_ids, conf_thresh_); + boxes, obj_probs, class_ids, conf_thresh_, + layout.channels_first); } else if (outputs[0].type == RKNN_TENSOR_FLOAT16) { // Convert FP16 to FP32 size_t num_elements = outputs[0].size / sizeof(uint16_t); @@ -573,13 +676,15 @@ private: valid_count = ProcessOutputV8(fp32_buffer_.data(), num_boxes, num_classes_, model_input_h_, model_input_w_, - boxes, obj_probs, class_ids, conf_thresh_); + boxes, obj_probs, class_ids, conf_thresh_, + layout.channels_first); } else { valid_count = ProcessOutputV8Int8(reinterpret_cast(const_cast(outputs[0].data)), num_boxes, num_classes_, model_input_h_, model_input_w_, boxes, obj_probs, class_ids, conf_thresh_, - outputs[0].zp, outputs[0].scale); + outputs[0].zp, outputs[0].scale, + layout.channels_first); } } @@ -670,33 +775,38 @@ private: } else { if (outputs.empty()) return; - int num_boxes = 0; - int num_channels = 4 + num_classes_; + const V8LayoutInfo layout = ResolveV8Layout(outputs[0].dims, outputs[0].data.size(), + outputs[0].type, num_classes_, + model_input_h_, model_input_w_); + const int num_boxes = layout.num_boxes; + if (num_boxes <= 0) return; - if (outputs[0].dims.size() >= 3) { - if (outputs[0].dims[1] == static_cast(num_channels)) { - num_boxes = outputs[0].dims[2]; - } else if (outputs[0].dims[2] == static_cast(num_channels)) { - num_boxes = outputs[0].dims[1]; - } else { - num_boxes = 8400; - } - } else { - num_boxes = outputs[0].data.size() / num_channels; - } - - if (outputs[0].type == RKNN_TENSOR_FLOAT32 || - outputs[0].type == RKNN_TENSOR_FLOAT16) { + if (outputs[0].type == RKNN_TENSOR_FLOAT32) { valid_count = ProcessOutputV8(reinterpret_cast(outputs[0].data.data()), num_boxes, num_classes_, model_input_h_, model_input_w_, - boxes, obj_probs, class_ids, conf_thresh_); + boxes, obj_probs, class_ids, conf_thresh_, + layout.channels_first); + } else if (outputs[0].type == RKNN_TENSOR_FLOAT16) { + // Convert FP16 to FP32 + size_t num_elements = outputs[0].data.size() / sizeof(uint16_t); + fp32_buffer_.resize(num_elements); + const uint16_t* fp16_data = reinterpret_cast(outputs[0].data.data()); + for (size_t i = 0; i < num_elements; ++i) { + fp32_buffer_[i] = Fp16ToFp32(fp16_data[i]); + } + valid_count = ProcessOutputV8(fp32_buffer_.data(), + num_boxes, num_classes_, + model_input_h_, model_input_w_, + boxes, obj_probs, class_ids, conf_thresh_, + layout.channels_first); } else { valid_count = ProcessOutputV8Int8(reinterpret_cast(outputs[0].data.data()), num_boxes, num_classes_, model_input_h_, model_input_w_, boxes, obj_probs, class_ids, conf_thresh_, - outputs[0].zp, outputs[0].scale); + outputs[0].zp, outputs[0].scale, + layout.channels_first); } } diff --git a/third_party/rknpu2/examples/3rdparty/mpp/Android/arm64-v8a/libmpp.so b/third_party/rknpu2/examples/3rdparty/mpp/Android/arm64-v8a/libmpp.so new file mode 100644 index 0000000..15d262c Binary files /dev/null and b/third_party/rknpu2/examples/3rdparty/mpp/Android/arm64-v8a/libmpp.so differ diff --git a/third_party/rknpu2/examples/3rdparty/mpp/Android/armeabi-v7a/libmpp.so b/third_party/rknpu2/examples/3rdparty/mpp/Android/armeabi-v7a/libmpp.so new file mode 100644 index 0000000..88f76a8 Binary files /dev/null and b/third_party/rknpu2/examples/3rdparty/mpp/Android/armeabi-v7a/libmpp.so differ diff --git a/third_party/rknpu2/examples/3rdparty/mpp/Linux/aarch64/librockchip_mpp.so b/third_party/rknpu2/examples/3rdparty/mpp/Linux/aarch64/librockchip_mpp.so new file mode 100644 index 0000000..c8dad6d --- /dev/null +++ b/third_party/rknpu2/examples/3rdparty/mpp/Linux/aarch64/librockchip_mpp.so @@ -0,0 +1 @@ +librockchip_mpp.so.1 \ No newline at end of file diff --git a/third_party/rknpu2/examples/3rdparty/mpp/Linux/armhf/librockchip_mpp.so b/third_party/rknpu2/examples/3rdparty/mpp/Linux/armhf/librockchip_mpp.so new file mode 100644 index 0000000..c8dad6d --- /dev/null +++ b/third_party/rknpu2/examples/3rdparty/mpp/Linux/armhf/librockchip_mpp.so @@ -0,0 +1 @@ +librockchip_mpp.so.1 \ No newline at end of file diff --git a/third_party/rknpu2/examples/3rdparty/rga/RK356X/lib/Android/arm64-v8a/librga.so b/third_party/rknpu2/examples/3rdparty/rga/RK356X/lib/Android/arm64-v8a/librga.so new file mode 100644 index 0000000..74bd4de Binary files /dev/null and b/third_party/rknpu2/examples/3rdparty/rga/RK356X/lib/Android/arm64-v8a/librga.so differ diff --git a/third_party/rknpu2/examples/3rdparty/rga/RK356X/lib/Android/armeabi-v7a/librga.so b/third_party/rknpu2/examples/3rdparty/rga/RK356X/lib/Android/armeabi-v7a/librga.so new file mode 100644 index 0000000..e018d30 Binary files /dev/null and b/third_party/rknpu2/examples/3rdparty/rga/RK356X/lib/Android/armeabi-v7a/librga.so differ diff --git a/third_party/rknpu2/examples/3rdparty/rga/RK356X/lib/Linux/aarch64/librga.so b/third_party/rknpu2/examples/3rdparty/rga/RK356X/lib/Linux/aarch64/librga.so new file mode 100644 index 0000000..9223831 Binary files /dev/null and b/third_party/rknpu2/examples/3rdparty/rga/RK356X/lib/Linux/aarch64/librga.so differ diff --git a/third_party/rknpu2/examples/3rdparty/rga/RK3588/lib/Android/arm64-v8a/librga.so b/third_party/rknpu2/examples/3rdparty/rga/RK3588/lib/Android/arm64-v8a/librga.so new file mode 100644 index 0000000..74bd4de Binary files /dev/null and b/third_party/rknpu2/examples/3rdparty/rga/RK3588/lib/Android/arm64-v8a/librga.so differ diff --git a/third_party/rknpu2/examples/3rdparty/rga/RK3588/lib/Android/armeabi-v7a/librga.so b/third_party/rknpu2/examples/3rdparty/rga/RK3588/lib/Android/armeabi-v7a/librga.so new file mode 100644 index 0000000..e018d30 Binary files /dev/null and b/third_party/rknpu2/examples/3rdparty/rga/RK3588/lib/Android/armeabi-v7a/librga.so differ diff --git a/third_party/rknpu2/examples/3rdparty/rga/RK3588/lib/Linux/aarch64/librga.so b/third_party/rknpu2/examples/3rdparty/rga/RK3588/lib/Linux/aarch64/librga.so new file mode 100644 index 0000000..9223831 Binary files /dev/null and b/third_party/rknpu2/examples/3rdparty/rga/RK3588/lib/Linux/aarch64/librga.so differ diff --git a/third_party/rknpu2/examples/3rdparty/rga/RV110X/lib/Linux/armhf/librga.so b/third_party/rknpu2/examples/3rdparty/rga/RV110X/lib/Linux/armhf/librga.so new file mode 100644 index 0000000..2242beb Binary files /dev/null and b/third_party/rknpu2/examples/3rdparty/rga/RV110X/lib/Linux/armhf/librga.so differ diff --git a/third_party/rknpu2/examples/3rdparty/rk_mpi_mmz/lib/Android/arm64-v8a/libmpimmz.so b/third_party/rknpu2/examples/3rdparty/rk_mpi_mmz/lib/Android/arm64-v8a/libmpimmz.so new file mode 100644 index 0000000..ea8d308 Binary files /dev/null and b/third_party/rknpu2/examples/3rdparty/rk_mpi_mmz/lib/Android/arm64-v8a/libmpimmz.so differ diff --git a/third_party/rknpu2/examples/3rdparty/rk_mpi_mmz/lib/Android/armeabi-v7a/libmpimmz.so b/third_party/rknpu2/examples/3rdparty/rk_mpi_mmz/lib/Android/armeabi-v7a/libmpimmz.so new file mode 100644 index 0000000..ac3ff98 Binary files /dev/null and b/third_party/rknpu2/examples/3rdparty/rk_mpi_mmz/lib/Android/armeabi-v7a/libmpimmz.so differ diff --git a/third_party/rknpu2/examples/3rdparty/rk_mpi_mmz/lib/Linux/aarch64/libmpimmz.so b/third_party/rknpu2/examples/3rdparty/rk_mpi_mmz/lib/Linux/aarch64/libmpimmz.so new file mode 100644 index 0000000..f7c0de6 Binary files /dev/null and b/third_party/rknpu2/examples/3rdparty/rk_mpi_mmz/lib/Linux/aarch64/libmpimmz.so differ diff --git a/third_party/rknpu2/examples/3rdparty/rk_mpi_mmz/lib/Linux/armhf/libmpimmz.so b/third_party/rknpu2/examples/3rdparty/rk_mpi_mmz/lib/Linux/armhf/libmpimmz.so new file mode 100644 index 0000000..3216de4 Binary files /dev/null and b/third_party/rknpu2/examples/3rdparty/rk_mpi_mmz/lib/Linux/armhf/libmpimmz.so differ