Doni/safesight-edge
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add V8 box format support and enhance output processing in YOLO node

Browse Source
This commit is contained in:
sladro 2026-02-27 19:16:10 +08:00
parent 35d00925fc
commit 84408e8a86
3 changed files with 136 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
configs/test_cam1_ppe11_yolo_debug.json
View File

@ -1,4 +1,8 @@
{
"global": {
"metrics_port": 9001,
"web_root": "web"
},
"queue": {
"size": 8,
"strategy": "drop_oldest"
@ -45,6 +49,7 @@
"model_w": 768,
"model_h": 768,
"num_classes": 11,
"v8_box_format": "xyxy",
"conf": 0.45,
"nms": 0.45,
"class_filter": [3, 6],

21
docs/命令.md
View File

@ -6,16 +6,17 @@
ffmpeg -f dshow -video_size 1280x720 -vcodec mjpeg -i video="1080P USB Camera" -c:v libx264 -preset ultrafast -pix_fmt yuv420p -f rtsp rtsp://localhost:8554/cam
cmake -S . -B build \
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_TESTS=OFF \
-DBUILD_SAMPLES=ON \
-DRK3588_ENABLE_FFMPEG=ON \
-DRK3588_ENABLE_MPP=ON \
-DRK3588_ENABLE_RGA=ON \
-DRK3588_ENABLE_ZLMEDIAKIT=ON \
-DRK3588_ENABLE_RKNN=ON \
-DRK_ZLMK_API_LIB_PATH=$PWD/third_party/rknpu2/examples/3rdparty/zlmediakit/aarch64/libmk_api.so \
-DRK_ZLMEDIAKIT_INCLUDE_DIR=$PWD/third_party/rknpu2/examples/3rdparty/zlmediakit/include
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_TESTS=OFF \
-DBUILD_SAMPLES=ON \
-DRK3588_ENABLE_FFMPEG=ON \
-DRK3588_ENABLE_MPP=ON \
-DRK_MPP_LIB_PATH=/usr/lib/aarch64-linux-gnu/librockchip_mpp.so.0 \
-DRK3588_ENABLE_RGA=ON \
-DRK3588_ENABLE_ZLMEDIAKIT=ON \
-DRK3588_ENABLE_RKNN=ON \
-DRK_ZLMK_API_LIB_PATH=$PWD/third_party/rknpu2/examples/3rdparty/zlmediakit/aarch64/libmk_api.so \
-DRK_ZLMEDIAKIT_INCLUDE_DIR=$PWD/third_party/rknpu2/examples/3rdparty/zlmediakit/include
cmake --build build -j$(nproc)

142
plugins/ai_yolo/ai_yolo_node.cpp
View File

@ -33,6 +33,7 @@ const int kAnchor1[6] = {30, 61, 62, 45, 59, 119};
const int kAnchor2[6] = {116, 90, 156, 198, 373, 326};
enum class YoloVersion { V5, V8 };
enum class V8BoxFormat { Auto, CxCyWh, XyXy };
const char* kCocoLabels[kObjClassNum] = {
"person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat",
@ -238,6 +239,68 @@ struct V8LayoutInfo {
bool channels_first = true; // true: CxN, false: NxC
};
float ScoreBoxCandidate(float x, float y, float w, float h, int model_w, int model_h) {
float s = 0.0f;
if (w > 0.0f && h > 0.0f) s += 3.0f;
if (w <= model_w * 1.2f) s += 1.0f;
if (h <= model_h * 1.2f) s += 1.0f;
if (x >= -model_w * 0.1f) s += 1.0f;
if (y >= -model_h * 0.1f) s += 1.0f;
if ((x + w) <= model_w * 1.2f) s += 1.0f;
if ((y + h) <= model_h * 1.2f) s += 1.0f;
return s;
}
bool SeemsNormalized(float a, float b, float c, float d) {
auto in_range = [](float v) { return v >= -0.05f && v <= 2.5f; };
return in_range(a) && in_range(b) && in_range(c) && in_range(d);
}
void DecodeV8Box(float a, float b, float c, float d, int model_w, int model_h, V8BoxFormat fmt,
float& out_x, float& out_y, float& out_w, float& out_h) {
if (SeemsNormalized(a, b, c, d)) {
a *= static_cast<float>(model_w);
b *= static_cast<float>(model_h);
c *= static_cast<float>(model_w);
d *= static_cast<float>(model_h);
}
auto decode_cxcywh = [&](float& x, float& y, float& w, float& h) {
x = a - c / 2.0f;
y = b - d / 2.0f;
w = c;
h = d;
};
auto decode_xyxy = [&](float& x, float& y, float& w, float& h) {
x = a;
y = b;
w = c - a;
h = d - b;
};
if (fmt == V8BoxFormat::CxCyWh) {
decode_cxcywh(out_x, out_y, out_w, out_h);
return;
}
if (fmt == V8BoxFormat::XyXy) {
decode_xyxy(out_x, out_y, out_w, out_h);
return;
}
float x1 = 0.0f, y1 = 0.0f, w1 = 0.0f, h1 = 0.0f;
float x2 = 0.0f, y2 = 0.0f, w2 = 0.0f, h2 = 0.0f;
decode_cxcywh(x1, y1, w1, h1);
decode_xyxy(x2, y2, w2, h2);
const float s1 = ScoreBoxCandidate(x1, y1, w1, h1, model_w, model_h);
const float s2 = ScoreBoxCandidate(x2, y2, w2, h2, model_w, model_h);
if (s2 > s1) {
out_x = x2; out_y = y2; out_w = w2; out_h = h2;
} else {
out_x = x1; out_y = y1; out_w = w1; out_h = h1;
}
}
V8LayoutInfo ResolveV8Layout(const std::vector<uint32_t>& dims, size_t byte_size,
rknn_tensor_type type, int num_classes,
int model_h, int model_w) {
@ -319,7 +382,7 @@ int ProcessOutputV8(float* output, int num_boxes, int num_classes,
int model_h, int model_w,
std::vector<float>& boxes, std::vector<float>& obj_probs,
std::vector<int>& class_ids, float conf_thresh,
bool channels_first) {
bool channels_first, V8BoxFormat box_format) {
int valid_count = 0;
const int num_channels = 4 + num_classes;
@ -336,13 +399,12 @@ int ProcessOutputV8(float* output, int num_boxes, int num_classes,
}
if (max_score >= conf_thresh) {
float cx = channels_first ? output[0 * num_boxes + i] : output[i * num_channels + 0];
float cy = channels_first ? output[1 * num_boxes + i] : output[i * num_channels + 1];
float w = channels_first ? output[2 * num_boxes + i] : output[i * num_channels + 2];
float h = channels_first ? output[3 * num_boxes + i] : output[i * num_channels + 3];
float x1 = cx - w / 2.0f;
float y1 = cy - h / 2.0f;
const float a = channels_first ? output[0 * num_boxes + i] : output[i * num_channels + 0];
const float b = channels_first ? output[1 * num_boxes + i] : output[i * num_channels + 1];
const float c = channels_first ? output[2 * num_boxes + i] : output[i * num_channels + 2];
const float d = channels_first ? output[3 * num_boxes + i] : output[i * num_channels + 3];
float x1 = 0.0f, y1 = 0.0f, w = 0.0f, h = 0.0f;
DecodeV8Box(a, b, c, d, model_w, model_h, box_format, x1, y1, w, h);
boxes.push_back(x1);
boxes.push_back(y1);
@ -361,7 +423,7 @@ int ProcessOutputV8Int8(int8_t* output, int num_boxes, int num_classes,
int model_h, int model_w,
std::vector<float>& boxes, std::vector<float>& obj_probs,
std::vector<int>& class_ids, float conf_thresh,
int32_t zp, float scale, bool channels_first) {
int32_t zp, float scale, bool channels_first, V8BoxFormat box_format) {
int valid_count = 0;
int8_t thresh_i8 = QuantizeF32ToAffine(conf_thresh, zp, scale);
const int num_channels = 4 + num_classes;
@ -379,18 +441,17 @@ int ProcessOutputV8Int8(int8_t* output, int num_boxes, int num_classes,
}
if (max_score_i8 >= thresh_i8) {
float cx = DequantizeAffineToF32(
float a = DequantizeAffineToF32(
channels_first ? output[0 * num_boxes + i] : output[i * num_channels + 0], zp, scale);
float cy = DequantizeAffineToF32(
float b = DequantizeAffineToF32(
channels_first ? output[1 * num_boxes + i] : output[i * num_channels + 1], zp, scale);
float w = DequantizeAffineToF32(
float c = DequantizeAffineToF32(
channels_first ? output[2 * num_boxes + i] : output[i * num_channels + 2], zp, scale);
float h = DequantizeAffineToF32(
float d = DequantizeAffineToF32(
channels_first ? output[3 * num_boxes + i] : output[i * num_channels + 3], zp, scale);
float max_score = DequantizeAffineToF32(max_score_i8, zp, scale);
float x1 = cx - w / 2.0f;
float y1 = cy - h / 2.0f;
float x1 = 0.0f, y1 = 0.0f, w = 0.0f, h = 0.0f;
DecodeV8Box(a, b, c, d, model_w, model_h, box_format, x1, y1, w, h);
boxes.push_back(x1);
boxes.push_back(y1);
@ -420,6 +481,16 @@ public:
model_input_w_ = config.ValueOr<int>("model_w", 640);
model_input_h_ = config.ValueOr<int>("model_h", 640);
num_classes_ = config.ValueOr<int>("num_classes", 80);
{
const std::string bf = config.ValueOr<std::string>("v8_box_format", "auto");
if (bf == "xyxy") {
v8_box_format_ = V8BoxFormat::XyXy;
} else if (bf == "cxcywh") {
v8_box_format_ = V8BoxFormat::CxCyWh;
} else {
v8_box_format_ = V8BoxFormat::Auto;
}
}
if (const SimpleJson* dbg = config.Find("debug"); dbg && dbg->IsObject()) {
stats_log_ = dbg->ValueOr<bool>("stats", stats_log_);
@ -658,13 +729,26 @@ private:
model_input_h_, model_input_w_);
const int num_boxes = layout.num_boxes;
if (num_boxes <= 0) return;
if (debug_det_ && processed_ < 5) {
std::string dims_s;
for (size_t di = 0; di < outputs[0].dims.size(); ++di) {
dims_s += (di == 0 ? "[" : ",");
dims_s += std::to_string(outputs[0].dims[di]);
}
dims_s += "]";
LogInfo("[ai_yolo] v8 out type=" + std::to_string(static_cast<int>(outputs[0].type)) +
" size=" + std::to_string(outputs[0].size) +
" dims=" + dims_s +
" num_boxes=" + std::to_string(num_boxes) +
" layout=" + std::string(layout.channels_first ? "CxN" : "NxC"));
}
if (outputs[0].type == RKNN_TENSOR_FLOAT32) {
valid_count = ProcessOutputV8(reinterpret_cast<float*>(const_cast<uint8_t*>(outputs[0].data)),
num_boxes, num_classes_,
model_input_h_, model_input_w_,
boxes, obj_probs, class_ids, conf_thresh_,
layout.channels_first);
layout.channels_first, v8_box_format_);
} else if (outputs[0].type == RKNN_TENSOR_FLOAT16) {
// Convert FP16 to FP32
size_t num_elements = outputs[0].size / sizeof(uint16_t);
@ -677,14 +761,14 @@ private:
num_boxes, num_classes_,
model_input_h_, model_input_w_,
boxes, obj_probs, class_ids, conf_thresh_,
layout.channels_first);
layout.channels_first, v8_box_format_);
} else {
valid_count = ProcessOutputV8Int8(reinterpret_cast<int8_t*>(const_cast<uint8_t*>(outputs[0].data)),
num_boxes, num_classes_,
model_input_h_, model_input_w_,
boxes, obj_probs, class_ids, conf_thresh_,
outputs[0].zp, outputs[0].scale,
layout.channels_first);
layout.channels_first, v8_box_format_);
}
}
@ -780,13 +864,26 @@ private:
model_input_h_, model_input_w_);
const int num_boxes = layout.num_boxes;
if (num_boxes <= 0) return;
if (debug_det_ && processed_ < 5) {
std::string dims_s;
for (size_t di = 0; di < outputs[0].dims.size(); ++di) {
dims_s += (di == 0 ? "[" : ",");
dims_s += std::to_string(outputs[0].dims[di]);
}
dims_s += "]";
LogInfo("[ai_yolo] v8 out(type copy) type=" + std::to_string(static_cast<int>(outputs[0].type)) +
" size=" + std::to_string(outputs[0].data.size()) +
" dims=" + dims_s +
" num_boxes=" + std::to_string(num_boxes) +
" layout=" + std::string(layout.channels_first ? "CxN" : "NxC"));
}
if (outputs[0].type == RKNN_TENSOR_FLOAT32) {
valid_count = ProcessOutputV8(reinterpret_cast<float*>(outputs[0].data.data()),
num_boxes, num_classes_,
model_input_h_, model_input_w_,
boxes, obj_probs, class_ids, conf_thresh_,
layout.channels_first);
layout.channels_first, v8_box_format_);
} else if (outputs[0].type == RKNN_TENSOR_FLOAT16) {
// Convert FP16 to FP32
size_t num_elements = outputs[0].data.size() / sizeof(uint16_t);
@ -799,14 +896,14 @@ private:
num_boxes, num_classes_,
model_input_h_, model_input_w_,
boxes, obj_probs, class_ids, conf_thresh_,
layout.channels_first);
layout.channels_first, v8_box_format_);
} else {
valid_count = ProcessOutputV8Int8(reinterpret_cast<int8_t*>(outputs[0].data.data()),
num_boxes, num_classes_,
model_input_h_, model_input_w_,
boxes, obj_probs, class_ids, conf_thresh_,
outputs[0].zp, outputs[0].scale,
layout.channels_first);
layout.channels_first, v8_box_format_);
}
}
@ -876,6 +973,7 @@ private:
int model_input_w_ = 640;
int model_input_h_ = 640;
int num_classes_ = 80;
V8BoxFormat v8_box_format_ = V8BoxFormat::Auto;
YoloVersion yolo_version_ = YoloVersion::V8;
bool auto_detect_version_ = false;
std::set<int> class_filter_;