Doni/safesight-edge
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor YOLO node output processing and enhance output management in AI scheduler

Browse Source
This commit is contained in:
sladro 2026-02-27 19:39:08 +08:00
parent 6a70c6a805
commit 60ccab8e67
5 changed files with 170 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
configs/test_cam1_ppe11_yolo_debug.json
View File

@ -50,7 +50,7 @@
"model_h": 768,
"num_classes": 11,
"v8_box_format": "cxcywh",
"v8_cls_activation": "sigmoid",
"v8_cls_activation": "auto",
"conf": 0.25,
"nms": 0.45,
"class_filter": [],
@ -114,6 +114,11 @@
"proto": "hls",
"path": "./web/hls/cam1_debug/index.m3u8",
"segment_sec": 2
},
{
"proto": "rtsp_server",
"port": 8556,
"path": "/live/cam1_debug"
}
]
}

1
include/ai_scheduler.h
View File

@ -108,6 +108,7 @@ public:
#if defined(RK3588_ENABLE_RKNN)
std::shared_ptr<void> keepalive; // keeps the selected ModelContext alive
std::unique_lock<std::mutex> infer_lock; // holds ModelContext::infer_mutex for that context
std::shared_ptr<void> output_guard; // releases RKNN borrowed outputs before infer_lock unlocks
#endif
BorrowedInferResult() = default;

103
plugins/ai_yolo/ai_yolo_node.cpp
View File

@ -4,6 +4,7 @@
#include <cmath>
#include <cstddef>
#include <cstring>
#include <limits>
#include <memory>
#include <set>
#include <thread>
@ -70,39 +71,25 @@ inline float Sigmoid(float x) {
return 1.0f / (1.0f + std::exp(-x));
}
// FP16 (half) to FP32 conversion
// IEEE 754 half-precision: 1 sign bit, 5 exponent bits, 10 mantissa bits
// FP16 (half) to FP32 conversion.
// Uses arithmetic reconstruction to avoid undefined behavior on subnormals.
inline float Fp16ToFp32(uint16_t h) {
uint32_t sign = (h >> 15) & 0x1;
uint32_t exp = (h >> 10) & 0x1F;
uint32_t mant = h & 0x3FF;
uint32_t f;
const int sign = (h & 0x8000) ? -1 : 1;
const int exp = (h >> 10) & 0x1F;
const int mant = h & 0x03FF;
if (exp == 0) {
// Zero or subnormal
if (mant == 0) {
f = (sign << 31); // Signed zero
} else {
// Subnormal: convert to normal
exp = 1;
while ((mant & 0x400) == 0) {
mant <<= 1;
exp--;
}
mant &= 0x3FF;
f = (sign << 31) | ((exp + 112) << 23) | (mant << 13);
}
} else if (exp == 0x1F) {
// Infinity or NaN
f = (sign << 31) | (0xFF << 23) | (mant << 13);
} else {
// Normal number
f = (sign << 31) | ((exp + 112) << 23) | (mant << 13);
if (mant == 0) return sign < 0 ? -0.0f : 0.0f;
// subnormal: mant * 2^-24
return static_cast<float>(sign) * std::ldexp(static_cast<float>(mant), -24);
}
float result;
memcpy(&result, &f, sizeof(float));
return result;
if (exp == 0x1F) {
if (mant == 0) return sign < 0 ? -INFINITY : INFINITY;
return std::numeric_limits<float>::quiet_NaN();
}
// normal: (mant + 1024) * 2^(exp-25)
return static_cast<float>(sign) *
std::ldexp(static_cast<float>(mant + 1024), exp - 25);
}
float CalculateIoU(float x1_min, float y1_min, float x1_max, float y1_max,
@ -431,8 +418,15 @@ int ProcessOutputV8(float* output, int num_boxes, int num_classes,
const float b = channels_first ? output[1 * num_boxes + i] : output[i * num_channels + 1];
const float c = channels_first ? output[2 * num_boxes + i] : output[i * num_channels + 2];
const float d = channels_first ? output[3 * num_boxes + i] : output[i * num_channels + 3];
if (!std::isfinite(a) || !std::isfinite(b) || !std::isfinite(c) || !std::isfinite(d)) {
continue;
}
float x1 = 0.0f, y1 = 0.0f, w = 0.0f, h = 0.0f;
DecodeV8Box(a, b, c, d, model_w, model_h, box_format, x1, y1, w, h);
if (!std::isfinite(x1) || !std::isfinite(y1) || !std::isfinite(w) || !std::isfinite(h)) {
continue;
}
if (w <= 1e-3f || h <= 1e-3f) continue;
boxes.push_back(x1);
boxes.push_back(y1);
@ -478,8 +472,15 @@ int ProcessOutputV8Int8(int8_t* output, int num_boxes, int num_classes,
float d = DequantizeAffineToF32(
channels_first ? output[3 * num_boxes + i] : output[i * num_channels + 3], zp, scale);
float max_score = DequantizeAffineToF32(max_score_i8, zp, scale);
if (!std::isfinite(a) || !std::isfinite(b) || !std::isfinite(c) || !std::isfinite(d)) {
continue;
}
float x1 = 0.0f, y1 = 0.0f, w = 0.0f, h = 0.0f;
DecodeV8Box(a, b, c, d, model_w, model_h, box_format, x1, y1, w, h);
if (!std::isfinite(x1) || !std::isfinite(y1) || !std::isfinite(w) || !std::isfinite(h)) {
continue;
}
if (w <= 1e-3f || h <= 1e-3f) continue;
boxes.push_back(x1);
boxes.push_back(y1);
@ -782,14 +783,17 @@ private:
}
if (outputs[0].type == RKNN_TENSOR_FLOAT32) {
const bool apply_sigmoid = ResolveV8ApplySigmoid(
reinterpret_cast<float*>(const_cast<uint8_t*>(outputs[0].data)),
num_boxes, num_classes_, layout.channels_first, v8_cls_activation_);
if (debug_det_ && processed_ < 5) {
LogInfo("[ai_yolo] v8 cls activation=" + std::string(apply_sigmoid ? "sigmoid" : "none"));
}
valid_count = ProcessOutputV8(reinterpret_cast<float*>(const_cast<uint8_t*>(outputs[0].data)),
num_boxes, num_classes_,
model_input_h_, model_input_w_,
boxes, obj_probs, class_ids, conf_thresh_,
layout.channels_first, v8_box_format_,
ResolveV8ApplySigmoid(reinterpret_cast<float*>(const_cast<uint8_t*>(outputs[0].data)),
num_boxes, num_classes_, layout.channels_first,
v8_cls_activation_));
layout.channels_first, v8_box_format_, apply_sigmoid);
} else if (outputs[0].type == RKNN_TENSOR_FLOAT16) {
// Convert FP16 to FP32
size_t num_elements = outputs[0].size / sizeof(uint16_t);
@ -798,14 +802,16 @@ private:
for (size_t i = 0; i < num_elements; ++i) {
fp32_buffer_[i] = Fp16ToFp32(fp16_data[i]);
}
const bool apply_sigmoid = ResolveV8ApplySigmoid(
fp32_buffer_.data(), num_boxes, num_classes_, layout.channels_first, v8_cls_activation_);
if (debug_det_ && processed_ < 5) {
LogInfo("[ai_yolo] v8 cls activation=" + std::string(apply_sigmoid ? "sigmoid" : "none"));
}
valid_count = ProcessOutputV8(fp32_buffer_.data(),
num_boxes, num_classes_,
model_input_h_, model_input_w_,
boxes, obj_probs, class_ids, conf_thresh_,
layout.channels_first, v8_box_format_,
ResolveV8ApplySigmoid(fp32_buffer_.data(),
num_boxes, num_classes_, layout.channels_first,
v8_cls_activation_));
layout.channels_first, v8_box_format_, apply_sigmoid);
} else {
valid_count = ProcessOutputV8Int8(reinterpret_cast<int8_t*>(const_cast<uint8_t*>(outputs[0].data)),
num_boxes, num_classes_,
@ -927,14 +933,17 @@ private:
}
if (outputs[0].type == RKNN_TENSOR_FLOAT32) {
const bool apply_sigmoid = ResolveV8ApplySigmoid(
reinterpret_cast<float*>(outputs[0].data.data()),
num_boxes, num_classes_, layout.channels_first, v8_cls_activation_);
if (debug_det_ && processed_ < 5) {
LogInfo("[ai_yolo] v8 cls activation(copy)=" + std::string(apply_sigmoid ? "sigmoid" : "none"));
}
valid_count = ProcessOutputV8(reinterpret_cast<float*>(outputs[0].data.data()),
num_boxes, num_classes_,
model_input_h_, model_input_w_,
boxes, obj_probs, class_ids, conf_thresh_,
layout.channels_first, v8_box_format_,
ResolveV8ApplySigmoid(reinterpret_cast<float*>(outputs[0].data.data()),
num_boxes, num_classes_, layout.channels_first,
v8_cls_activation_));
layout.channels_first, v8_box_format_, apply_sigmoid);
} else if (outputs[0].type == RKNN_TENSOR_FLOAT16) {
// Convert FP16 to FP32
size_t num_elements = outputs[0].data.size() / sizeof(uint16_t);
@ -943,14 +952,16 @@ private:
for (size_t i = 0; i < num_elements; ++i) {
fp32_buffer_[i] = Fp16ToFp32(fp16_data[i]);
}
const bool apply_sigmoid = ResolveV8ApplySigmoid(
fp32_buffer_.data(), num_boxes, num_classes_, layout.channels_first, v8_cls_activation_);
if (debug_det_ && processed_ < 5) {
LogInfo("[ai_yolo] v8 cls activation(copy)=" + std::string(apply_sigmoid ? "sigmoid" : "none"));
}
valid_count = ProcessOutputV8(fp32_buffer_.data(),
num_boxes, num_classes_,
model_input_h_, model_input_w_,
boxes, obj_probs, class_ids, conf_thresh_,
layout.channels_first, v8_box_format_,
ResolveV8ApplySigmoid(fp32_buffer_.data(),
num_boxes, num_classes_, layout.channels_first,
v8_cls_activation_));
layout.channels_first, v8_box_format_, apply_sigmoid);
} else {
valid_count = ProcessOutputV8Int8(reinterpret_cast<int8_t*>(outputs[0].data.data()),
num_boxes, num_classes_,

90
plugins/preprocess/preprocess_node.cpp
View File

@ -3,6 +3,7 @@
#include <string>
#include <vector>
#include "face/face_result.h"
#include "hw/i_image_processor.h"
#include "node.h"
#include "utils/logger.h"
@ -19,6 +20,41 @@ PixelFormat ParseFormat(const std::string& s) {
return PixelFormat::UNKNOWN;
}
inline float ClampFloat(float v, float lo, float hi) {
return std::max(lo, std::min(v, hi));
}
void ScaleRect(Rect& r, float sx, float sy, int out_w, int out_h) {
if (out_w <= 0 || out_h <= 0) {
r = Rect{};
return;
}
const float fw = static_cast<float>(out_w);
const float fh = static_cast<float>(out_h);
const float x = ClampFloat(r.x * sx, 0.0f, fw);
const float y = ClampFloat(r.y * sy, 0.0f, fh);
float w = std::max(0.0f, r.w * sx);
float h = std::max(0.0f, r.h * sy);
if (x + w > fw) w = std::max(0.0f, fw - x);
if (y + h > fh) h = std::max(0.0f, fh - y);
r.x = x;
r.y = y;
r.w = w;
r.h = h;
}
void ScalePoint(Point2f& p, float sx, float sy, int out_w, int out_h) {
if (out_w <= 0 || out_h <= 0) {
p = Point2f{};
return;
}
p.x = ClampFloat(p.x * sx, 0.0f, static_cast<float>(out_w));
p.y = ClampFloat(p.y * sy, 0.0f, static_cast<float>(out_h));
}
} // namespace
class PreprocessNode : public INode {
@ -138,9 +174,55 @@ public:
auto out_frame = std::make_shared<Frame>(out);
out_frame->pts = frame->pts;
out_frame->frame_id = frame->frame_id;
out_frame->det = frame->det;
out_frame->face_det = frame->face_det;
out_frame->face_recog = frame->face_recog;
if (frame->det) {
auto det = std::make_shared<DetectionResult>(*frame->det);
if (frame->width > 0 && frame->height > 0) {
const float sx = static_cast<float>(out_w) / static_cast<float>(frame->width);
const float sy = static_cast<float>(out_h) / static_cast<float>(frame->height);
for (auto& it : det->items) {
ScaleRect(it.bbox, sx, sy, out_w, out_h);
}
}
det->img_w = out_w;
det->img_h = out_h;
out_frame->det = std::move(det);
}
if (frame->face_det) {
auto face_det = std::make_shared<FaceDetResult>(*frame->face_det);
if (frame->width > 0 && frame->height > 0) {
const float sx = static_cast<float>(out_w) / static_cast<float>(frame->width);
const float sy = static_cast<float>(out_h) / static_cast<float>(frame->height);
for (auto& it : face_det->faces) {
ScaleRect(it.bbox, sx, sy, out_w, out_h);
if (it.has_landmarks) {
for (auto& lm : it.landmarks) {
ScalePoint(lm, sx, sy, out_w, out_h);
}
}
}
}
face_det->img_w = out_w;
face_det->img_h = out_h;
out_frame->face_det = std::move(face_det);
}
if (frame->face_recog) {
auto face_recog = std::make_shared<FaceRecogResult>(*frame->face_recog);
if (frame->width > 0 && frame->height > 0) {
const float sx = static_cast<float>(out_w) / static_cast<float>(frame->width);
const float sy = static_cast<float>(out_h) / static_cast<float>(frame->height);
for (auto& it : face_recog->items) {
ScaleRect(it.bbox, sx, sy, out_w, out_h);
if (it.has_landmarks) {
for (auto& lm : it.landmarks) {
ScalePoint(lm, sx, sy, out_w, out_h);
}
}
}
}
face_recog->img_w = out_w;
face_recog->img_h = out_h;
out_frame->face_recog = std::move(face_recog);
}
out_frame->user_meta = frame->user_meta;
PushToDownstream(out_frame);
@ -170,7 +252,7 @@ private:
if (frame->width == out_w && frame->height == out_h) return;
if (!frame->det && !frame->face_det && !frame->face_recog) return;
warned_meta_resize_ = true;
LogWarn("[preprocess] resized frame but forwarded det/face meta without coordinate scaling; ensure det/recog/osd use same resolution (id=" + id_ + ")");
LogInfo("[preprocess] resized frame and scaled det/face meta to destination resolution (id=" + id_ + ")");
}
void ProcessPassthrough(FramePtr frame) {

22
src/ai_scheduler.cpp
View File

@ -77,6 +77,17 @@ uint32_t TensorTypeSizeBytes(rknn_tensor_type t) {
}
}
struct BorrowedOutputsGuard {
std::shared_ptr<void> ctx_keepalive;
rknn_context ctx = 0;
std::vector<rknn_output> outputs;
~BorrowedOutputsGuard() {
if (!ctx_keepalive || ctx == 0 || outputs.empty()) return;
rknn_outputs_release(ctx, static_cast<uint32_t>(outputs.size()), outputs.data());
}
};
} // namespace
#endif
@ -652,10 +663,17 @@ AiScheduler::BorrowedInferResult AiScheduler::InferBorrowed(ModelHandle handle,
}
}
rknn_outputs_release(ctx->ctx, ctx->n_output, outputs.data());
result.success = true;
total_inferences_.fetch_add(1);
// Keep RKNN outputs valid for the full lifetime of BorrowedInferResult.
// Release happens when this guard is destroyed with the result object.
auto out_guard = std::make_shared<BorrowedOutputsGuard>();
out_guard->ctx_keepalive = ctx;
out_guard->ctx = ctx->ctx;
out_guard->outputs = std::move(outputs);
result.output_guard = out_guard;
return result;
#else