From 60ccab8e67e899f65fc6fe2e1f4436a871e1cb83 Mon Sep 17 00:00:00 2001 From: sladro Date: Fri, 27 Feb 2026 19:39:08 +0800 Subject: [PATCH] Refactor YOLO node output processing and enhance output management in AI scheduler --- configs/test_cam1_ppe11_yolo_debug.json | 7 +- include/ai_scheduler.h | 1 + plugins/ai_yolo/ai_yolo_node.cpp | 103 +++++++++++++----------- plugins/preprocess/preprocess_node.cpp | 90 ++++++++++++++++++++- src/ai_scheduler.cpp | 22 ++++- 5 files changed, 170 insertions(+), 53 deletions(-) diff --git a/configs/test_cam1_ppe11_yolo_debug.json b/configs/test_cam1_ppe11_yolo_debug.json index a35d5b1..526a8ca 100644 --- a/configs/test_cam1_ppe11_yolo_debug.json +++ b/configs/test_cam1_ppe11_yolo_debug.json @@ -50,7 +50,7 @@ "model_h": 768, "num_classes": 11, "v8_box_format": "cxcywh", - "v8_cls_activation": "sigmoid", + "v8_cls_activation": "auto", "conf": 0.25, "nms": 0.45, "class_filter": [], @@ -114,6 +114,11 @@ "proto": "hls", "path": "./web/hls/cam1_debug/index.m3u8", "segment_sec": 2 + }, + { + "proto": "rtsp_server", + "port": 8556, + "path": "/live/cam1_debug" } ] } diff --git a/include/ai_scheduler.h b/include/ai_scheduler.h index dc605a3..1850d00 100644 --- a/include/ai_scheduler.h +++ b/include/ai_scheduler.h @@ -108,6 +108,7 @@ public: #if defined(RK3588_ENABLE_RKNN) std::shared_ptr keepalive; // keeps the selected ModelContext alive std::unique_lock infer_lock; // holds ModelContext::infer_mutex for that context + std::shared_ptr output_guard; // releases RKNN borrowed outputs before infer_lock unlocks #endif BorrowedInferResult() = default; diff --git a/plugins/ai_yolo/ai_yolo_node.cpp b/plugins/ai_yolo/ai_yolo_node.cpp index 92b3f56..0910018 100644 --- a/plugins/ai_yolo/ai_yolo_node.cpp +++ b/plugins/ai_yolo/ai_yolo_node.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -70,39 +71,25 @@ inline float Sigmoid(float x) { return 1.0f / (1.0f + std::exp(-x)); } -// FP16 (half) to FP32 conversion -// IEEE 754 half-precision: 1 sign bit, 5 exponent bits, 10 mantissa bits +// FP16 (half) to FP32 conversion. +// Uses arithmetic reconstruction to avoid undefined behavior on subnormals. inline float Fp16ToFp32(uint16_t h) { - uint32_t sign = (h >> 15) & 0x1; - uint32_t exp = (h >> 10) & 0x1F; - uint32_t mant = h & 0x3FF; - - uint32_t f; + const int sign = (h & 0x8000) ? -1 : 1; + const int exp = (h >> 10) & 0x1F; + const int mant = h & 0x03FF; + if (exp == 0) { - // Zero or subnormal - if (mant == 0) { - f = (sign << 31); // Signed zero - } else { - // Subnormal: convert to normal - exp = 1; - while ((mant & 0x400) == 0) { - mant <<= 1; - exp--; - } - mant &= 0x3FF; - f = (sign << 31) | ((exp + 112) << 23) | (mant << 13); - } - } else if (exp == 0x1F) { - // Infinity or NaN - f = (sign << 31) | (0xFF << 23) | (mant << 13); - } else { - // Normal number - f = (sign << 31) | ((exp + 112) << 23) | (mant << 13); + if (mant == 0) return sign < 0 ? -0.0f : 0.0f; + // subnormal: mant * 2^-24 + return static_cast(sign) * std::ldexp(static_cast(mant), -24); } - - float result; - memcpy(&result, &f, sizeof(float)); - return result; + if (exp == 0x1F) { + if (mant == 0) return sign < 0 ? -INFINITY : INFINITY; + return std::numeric_limits::quiet_NaN(); + } + // normal: (mant + 1024) * 2^(exp-25) + return static_cast(sign) * + std::ldexp(static_cast(mant + 1024), exp - 25); } float CalculateIoU(float x1_min, float y1_min, float x1_max, float y1_max, @@ -431,8 +418,15 @@ int ProcessOutputV8(float* output, int num_boxes, int num_classes, const float b = channels_first ? output[1 * num_boxes + i] : output[i * num_channels + 1]; const float c = channels_first ? output[2 * num_boxes + i] : output[i * num_channels + 2]; const float d = channels_first ? output[3 * num_boxes + i] : output[i * num_channels + 3]; + if (!std::isfinite(a) || !std::isfinite(b) || !std::isfinite(c) || !std::isfinite(d)) { + continue; + } float x1 = 0.0f, y1 = 0.0f, w = 0.0f, h = 0.0f; DecodeV8Box(a, b, c, d, model_w, model_h, box_format, x1, y1, w, h); + if (!std::isfinite(x1) || !std::isfinite(y1) || !std::isfinite(w) || !std::isfinite(h)) { + continue; + } + if (w <= 1e-3f || h <= 1e-3f) continue; boxes.push_back(x1); boxes.push_back(y1); @@ -478,8 +472,15 @@ int ProcessOutputV8Int8(int8_t* output, int num_boxes, int num_classes, float d = DequantizeAffineToF32( channels_first ? output[3 * num_boxes + i] : output[i * num_channels + 3], zp, scale); float max_score = DequantizeAffineToF32(max_score_i8, zp, scale); + if (!std::isfinite(a) || !std::isfinite(b) || !std::isfinite(c) || !std::isfinite(d)) { + continue; + } float x1 = 0.0f, y1 = 0.0f, w = 0.0f, h = 0.0f; DecodeV8Box(a, b, c, d, model_w, model_h, box_format, x1, y1, w, h); + if (!std::isfinite(x1) || !std::isfinite(y1) || !std::isfinite(w) || !std::isfinite(h)) { + continue; + } + if (w <= 1e-3f || h <= 1e-3f) continue; boxes.push_back(x1); boxes.push_back(y1); @@ -782,14 +783,17 @@ private: } if (outputs[0].type == RKNN_TENSOR_FLOAT32) { + const bool apply_sigmoid = ResolveV8ApplySigmoid( + reinterpret_cast(const_cast(outputs[0].data)), + num_boxes, num_classes_, layout.channels_first, v8_cls_activation_); + if (debug_det_ && processed_ < 5) { + LogInfo("[ai_yolo] v8 cls activation=" + std::string(apply_sigmoid ? "sigmoid" : "none")); + } valid_count = ProcessOutputV8(reinterpret_cast(const_cast(outputs[0].data)), num_boxes, num_classes_, model_input_h_, model_input_w_, boxes, obj_probs, class_ids, conf_thresh_, - layout.channels_first, v8_box_format_, - ResolveV8ApplySigmoid(reinterpret_cast(const_cast(outputs[0].data)), - num_boxes, num_classes_, layout.channels_first, - v8_cls_activation_)); + layout.channels_first, v8_box_format_, apply_sigmoid); } else if (outputs[0].type == RKNN_TENSOR_FLOAT16) { // Convert FP16 to FP32 size_t num_elements = outputs[0].size / sizeof(uint16_t); @@ -798,14 +802,16 @@ private: for (size_t i = 0; i < num_elements; ++i) { fp32_buffer_[i] = Fp16ToFp32(fp16_data[i]); } + const bool apply_sigmoid = ResolveV8ApplySigmoid( + fp32_buffer_.data(), num_boxes, num_classes_, layout.channels_first, v8_cls_activation_); + if (debug_det_ && processed_ < 5) { + LogInfo("[ai_yolo] v8 cls activation=" + std::string(apply_sigmoid ? "sigmoid" : "none")); + } valid_count = ProcessOutputV8(fp32_buffer_.data(), num_boxes, num_classes_, model_input_h_, model_input_w_, boxes, obj_probs, class_ids, conf_thresh_, - layout.channels_first, v8_box_format_, - ResolveV8ApplySigmoid(fp32_buffer_.data(), - num_boxes, num_classes_, layout.channels_first, - v8_cls_activation_)); + layout.channels_first, v8_box_format_, apply_sigmoid); } else { valid_count = ProcessOutputV8Int8(reinterpret_cast(const_cast(outputs[0].data)), num_boxes, num_classes_, @@ -927,14 +933,17 @@ private: } if (outputs[0].type == RKNN_TENSOR_FLOAT32) { + const bool apply_sigmoid = ResolveV8ApplySigmoid( + reinterpret_cast(outputs[0].data.data()), + num_boxes, num_classes_, layout.channels_first, v8_cls_activation_); + if (debug_det_ && processed_ < 5) { + LogInfo("[ai_yolo] v8 cls activation(copy)=" + std::string(apply_sigmoid ? "sigmoid" : "none")); + } valid_count = ProcessOutputV8(reinterpret_cast(outputs[0].data.data()), num_boxes, num_classes_, model_input_h_, model_input_w_, boxes, obj_probs, class_ids, conf_thresh_, - layout.channels_first, v8_box_format_, - ResolveV8ApplySigmoid(reinterpret_cast(outputs[0].data.data()), - num_boxes, num_classes_, layout.channels_first, - v8_cls_activation_)); + layout.channels_first, v8_box_format_, apply_sigmoid); } else if (outputs[0].type == RKNN_TENSOR_FLOAT16) { // Convert FP16 to FP32 size_t num_elements = outputs[0].data.size() / sizeof(uint16_t); @@ -943,14 +952,16 @@ private: for (size_t i = 0; i < num_elements; ++i) { fp32_buffer_[i] = Fp16ToFp32(fp16_data[i]); } + const bool apply_sigmoid = ResolveV8ApplySigmoid( + fp32_buffer_.data(), num_boxes, num_classes_, layout.channels_first, v8_cls_activation_); + if (debug_det_ && processed_ < 5) { + LogInfo("[ai_yolo] v8 cls activation(copy)=" + std::string(apply_sigmoid ? "sigmoid" : "none")); + } valid_count = ProcessOutputV8(fp32_buffer_.data(), num_boxes, num_classes_, model_input_h_, model_input_w_, boxes, obj_probs, class_ids, conf_thresh_, - layout.channels_first, v8_box_format_, - ResolveV8ApplySigmoid(fp32_buffer_.data(), - num_boxes, num_classes_, layout.channels_first, - v8_cls_activation_)); + layout.channels_first, v8_box_format_, apply_sigmoid); } else { valid_count = ProcessOutputV8Int8(reinterpret_cast(outputs[0].data.data()), num_boxes, num_classes_, diff --git a/plugins/preprocess/preprocess_node.cpp b/plugins/preprocess/preprocess_node.cpp index 36538c7..c1fa691 100644 --- a/plugins/preprocess/preprocess_node.cpp +++ b/plugins/preprocess/preprocess_node.cpp @@ -3,6 +3,7 @@ #include #include +#include "face/face_result.h" #include "hw/i_image_processor.h" #include "node.h" #include "utils/logger.h" @@ -19,6 +20,41 @@ PixelFormat ParseFormat(const std::string& s) { return PixelFormat::UNKNOWN; } +inline float ClampFloat(float v, float lo, float hi) { + return std::max(lo, std::min(v, hi)); +} + +void ScaleRect(Rect& r, float sx, float sy, int out_w, int out_h) { + if (out_w <= 0 || out_h <= 0) { + r = Rect{}; + return; + } + const float fw = static_cast(out_w); + const float fh = static_cast(out_h); + + const float x = ClampFloat(r.x * sx, 0.0f, fw); + const float y = ClampFloat(r.y * sy, 0.0f, fh); + float w = std::max(0.0f, r.w * sx); + float h = std::max(0.0f, r.h * sy); + + if (x + w > fw) w = std::max(0.0f, fw - x); + if (y + h > fh) h = std::max(0.0f, fh - y); + + r.x = x; + r.y = y; + r.w = w; + r.h = h; +} + +void ScalePoint(Point2f& p, float sx, float sy, int out_w, int out_h) { + if (out_w <= 0 || out_h <= 0) { + p = Point2f{}; + return; + } + p.x = ClampFloat(p.x * sx, 0.0f, static_cast(out_w)); + p.y = ClampFloat(p.y * sy, 0.0f, static_cast(out_h)); +} + } // namespace class PreprocessNode : public INode { @@ -138,9 +174,55 @@ public: auto out_frame = std::make_shared(out); out_frame->pts = frame->pts; out_frame->frame_id = frame->frame_id; - out_frame->det = frame->det; - out_frame->face_det = frame->face_det; - out_frame->face_recog = frame->face_recog; + if (frame->det) { + auto det = std::make_shared(*frame->det); + if (frame->width > 0 && frame->height > 0) { + const float sx = static_cast(out_w) / static_cast(frame->width); + const float sy = static_cast(out_h) / static_cast(frame->height); + for (auto& it : det->items) { + ScaleRect(it.bbox, sx, sy, out_w, out_h); + } + } + det->img_w = out_w; + det->img_h = out_h; + out_frame->det = std::move(det); + } + if (frame->face_det) { + auto face_det = std::make_shared(*frame->face_det); + if (frame->width > 0 && frame->height > 0) { + const float sx = static_cast(out_w) / static_cast(frame->width); + const float sy = static_cast(out_h) / static_cast(frame->height); + for (auto& it : face_det->faces) { + ScaleRect(it.bbox, sx, sy, out_w, out_h); + if (it.has_landmarks) { + for (auto& lm : it.landmarks) { + ScalePoint(lm, sx, sy, out_w, out_h); + } + } + } + } + face_det->img_w = out_w; + face_det->img_h = out_h; + out_frame->face_det = std::move(face_det); + } + if (frame->face_recog) { + auto face_recog = std::make_shared(*frame->face_recog); + if (frame->width > 0 && frame->height > 0) { + const float sx = static_cast(out_w) / static_cast(frame->width); + const float sy = static_cast(out_h) / static_cast(frame->height); + for (auto& it : face_recog->items) { + ScaleRect(it.bbox, sx, sy, out_w, out_h); + if (it.has_landmarks) { + for (auto& lm : it.landmarks) { + ScalePoint(lm, sx, sy, out_w, out_h); + } + } + } + } + face_recog->img_w = out_w; + face_recog->img_h = out_h; + out_frame->face_recog = std::move(face_recog); + } out_frame->user_meta = frame->user_meta; PushToDownstream(out_frame); @@ -170,7 +252,7 @@ private: if (frame->width == out_w && frame->height == out_h) return; if (!frame->det && !frame->face_det && !frame->face_recog) return; warned_meta_resize_ = true; - LogWarn("[preprocess] resized frame but forwarded det/face meta without coordinate scaling; ensure det/recog/osd use same resolution (id=" + id_ + ")"); + LogInfo("[preprocess] resized frame and scaled det/face meta to destination resolution (id=" + id_ + ")"); } void ProcessPassthrough(FramePtr frame) { diff --git a/src/ai_scheduler.cpp b/src/ai_scheduler.cpp index ed08d7e..75e6b17 100644 --- a/src/ai_scheduler.cpp +++ b/src/ai_scheduler.cpp @@ -77,6 +77,17 @@ uint32_t TensorTypeSizeBytes(rknn_tensor_type t) { } } +struct BorrowedOutputsGuard { + std::shared_ptr ctx_keepalive; + rknn_context ctx = 0; + std::vector outputs; + + ~BorrowedOutputsGuard() { + if (!ctx_keepalive || ctx == 0 || outputs.empty()) return; + rknn_outputs_release(ctx, static_cast(outputs.size()), outputs.data()); + } +}; + } // namespace #endif @@ -652,10 +663,17 @@ AiScheduler::BorrowedInferResult AiScheduler::InferBorrowed(ModelHandle handle, } } - rknn_outputs_release(ctx->ctx, ctx->n_output, outputs.data()); - result.success = true; total_inferences_.fetch_add(1); + + // Keep RKNN outputs valid for the full lifetime of BorrowedInferResult. + // Release happens when this guard is destroyed with the result object. + auto out_guard = std::make_shared(); + out_guard->ctx_keepalive = ctx; + out_guard->ctx = ctx->ctx; + out_guard->outputs = std::move(outputs); + result.output_guard = out_guard; + return result; #else