diff --git a/configs/sample_cam1_ppe12.json b/configs/sample_cam1_ppe12.json index fd3f420..f3ef51c 100644 --- a/configs/sample_cam1_ppe12.json +++ b/configs/sample_cam1_ppe12.json @@ -47,49 +47,14 @@ "model_w": 768, "model_h": 768, "num_classes": 11, - "conf": 0.35, + "conf": 0.25, "nms": 0.45, "debug": { "stats": true, "stats_interval": 30, "detections": true }, - "class_filter": [3, 6] - }, - { - "id": "face_det_cam1", - "type": "ai_face_det", - "role": "filter", - "enable": true, - "model_path": "./models/RetinaFace_mobile320.rknn", - "conf": 0.7, - "nms": 0.4, - "max_faces": 10, - "output_landmarks": true, - "input_format": "rgb" - }, - { - "id": "face_recog_cam1", - "type": "ai_face_recog", - "role": "filter", - "enable": true, - "model_path": "./models/mobilefacenet_arcface.rknn", - "align": true, - "emit_embedding": false, - "max_faces": 10, - "input_format": "rgb", - "input_dtype": "uint8", - "threshold": { - "accept": 0.45, - "margin": 0.05 - }, - "gallery": { - "backend": "sqlite", - "path": "./models/face_gallery.db", - "load_on_start": true, - "expected_dim": 512, - "dtype": "auto" - } + "class_filter": [3, 6, 10] }, { "id": "trk_cam1", @@ -99,7 +64,7 @@ "mode": "bytetrack_lite", "per_class": true, "state_key": "cam_ppe11_detection", - "track_classes": [3, 6], + "track_classes": [3, 6, 10], "ignore_classes": [], "allowed_models": ["yolov8"], "high_th": 0.5, @@ -110,17 +75,20 @@ "max_tracks": 128 }, { - "id": "pre_face_cam1", - "type": "preprocess", + "id": "logic_cam1", + "type": "logic_gate", "role": "filter", "enable": true, - "dst_w": 0, - "dst_h": 0, - "dst_format": "rgb", - "dst_packed": true, - "keep_ratio": false, - "rga_gate": "cam_ppe11_detection", - "use_rga": true + "mode": "ppe_boots_check", + "anchor_class": 6, + "boots_class": 3, + "color_check": { + "enable": true, + "method": "hsv", + "dark_threshold": 80, + "roi_expand": 1.0 + }, + "debug": true }, { "id": "osd_cam1", @@ -134,7 +102,7 @@ "line_width": 2, "font_scale": 1, "use_rga_bbox": false, - "labels": ["helmet", "gloves", "vest", "boots", "goggles", "none", "Person", "no_helmet", "no_goggle", "no_gloves", "no_boots"] + "labels": ["helmet", "gloves", "vest", "boots", "goggles", "none", "Person", "no_helmet", "no_goggle", "no_gloves", "no_boots", "violation"] }, { "id": "post_cam1", @@ -153,6 +121,7 @@ "type": "publish", "role": "filter", "enable": true, + "queue": {"size": 2, "policy": "drop_oldest"}, "codec": "h264", "fps": 30, "gop": 60, @@ -177,29 +146,23 @@ "type": "alarm", "role": "sink", "enable": true, - "eval_fps": 10, - "labels": ["helmet", "gloves", "vest", "boots", "goggles", "none", "Person", "no_helmet", "no_goggle", "no_gloves", "no_boots"], + "eval_fps": 0, + "labels": ["helmet", "gloves", "vest", "boots", "goggles", "none", "Person", "no_helmet", "no_goggle", "no_gloves", "no_boots", "violation"], "rules": [ { - "name": "ppe_violation", - "class_ids": [3, 6], - "roi": { - "x": 0.0, - "y": 0.0, - "w": 1.0, - "h": 1.0 - }, - "min_score": 0.4, - "min_box_area_ratio": 0.02, - "require_track_id": true, - "min_duration_ms": 1500, - "min_hits": 3, - "hit_window_ms": 1500, + "name": "non_compliant_boots", + "class_ids": [10], + "roi": {"x": 0.0, "y": 0.0, "w": 1.0, "h": 1.0}, + "min_score": 0.1, + "min_box_area_ratio": 0.005, + "require_track_id": false, + "min_duration_ms": 500, + "min_hits": 1, + "hit_window_ms": 500, "cooldown_ms": 5000, "per_track_cooldown_ms": 5000 } ], - "face_rules": [], "actions": { "log": { "enable": true, @@ -233,19 +196,12 @@ "secret_key": "minioadmin" } }, - "http": { - "enable": false, - "url": "http://127.0.0.1:8080/api/alarm", - "timeout_ms": 3000, - "include_media_url": true, - "method": "POST" - }, "external_api": { "enable": true, "getTokenUrl": "http://10.0.0.49:8080/api/getToken", "putMessageUrl": "http://10.0.0.49:8080/api/putMessage", "tenantCode": "32", - "channelNo": "${vod_channelNo}", + "channelNo": "cam1", "timeout_ms": 3000, "include_media_url": true, "token_header": "X-Access-Token", @@ -253,82 +209,17 @@ "token_cache_sec": 1200 } } - }, - { - "id": "alarm_face_cam1", - "type": "alarm", - "role": "sink", - "enable": true, - "eval_fps": 5, - "labels": [], - "rules": [], - "face_rules": [ - { - "name": "unknown_face", - "type": "unknown", - "cooldown_ms": 7000, - "min_sim": 0.35, - "min_hits": 2, - "hit_window_ms": 1500, - "min_face_area_ratio": 0.01, - "min_face_aspect": 0.6, - "max_face_aspect": 1.6 - }, - { - "name": "known_person", - "type": "person", - "cooldown_ms": 7000, - "min_sim": 0.6, - "min_hits": 2, - "hit_window_ms": 1500, - "min_face_area_ratio": 0.01, - "min_face_aspect": 0.6, - "max_face_aspect": 1.6 - } - ], - "actions": { - "log": { - "enable": false, - "level": "info" - }, - "snapshot": { - "enable": true, - "format": "jpg", - "quality": 85, - "upload": { - "type": "minio", - "endpoint": "http://10.0.0.49:9000", - "bucket": "myminio", - "region": "us-east-1", - "access_key": "minioadmin", - "secret_key": "minioadmin" - } - }, - "clip": { - "enable": false - }, - "http": { - "enable": true, - "url": "http://127.0.0.1:8080/api/alarm", - "timeout_ms": 3000, - "include_media_url": true, - "method": "POST" - } - } } ], "edges": [ ["in_cam1", "pre_cam1"], - ["in_cam1", "pre_face_cam1"], ["pre_cam1", "yolo_cam1"], ["yolo_cam1", "trk_cam1"], - ["trk_cam1", "osd_cam1"], + ["trk_cam1", "logic_cam1"], + ["logic_cam1", "osd_cam1"], ["osd_cam1", "post_cam1"], ["post_cam1", "pub_cam1"], - ["pub_cam1", "alarm_cam1"], - ["pre_face_cam1", "face_det_cam1"], - ["face_det_cam1", "face_recog_cam1"], - ["face_recog_cam1", "alarm_face_cam1"] + ["pub_cam1", "alarm_cam1"] ] } ] diff --git a/configs/sample_logic_gate_ppe.json b/configs/sample_logic_gate_ppe.json new file mode 100644 index 0000000..28f6c47 --- /dev/null +++ b/configs/sample_logic_gate_ppe.json @@ -0,0 +1,120 @@ +{ + "queue": { + "size": 8, + "strategy": "drop_oldest" + }, + "graphs": [ + { + "name": "ppe_logic_gate_demo", + "nodes": [ + { + "id": "in", + "type": "input_rtsp", + "role": "source", + "enable": true, + "url": "rtsp://10.0.0.49:8554/cam", + "fps": 30, + "width": 1280, + "height": 720, + "use_mpp": true, + "force_tcp": true + }, + { + "id": "pre", + "type": "preprocess", + "role": "filter", + "enable": true, + "dst_w": 768, + "dst_h": 768, + "dst_format": "rgb", + "resize_mode": "stretch", + "use_rga": true + }, + { + "id": "yolo", + "type": "ai_yolo", + "role": "filter", + "enable": true, + "infer_fps": 10, + "model_path": "./models/best-768.rknn", + "model_version": "v8", + "model_w": 768, + "model_h": 768, + "num_classes": 11, + "conf": 0.35, + "nms": 0.45, + "class_filter": [3, 6] + }, + { + "id": "trk", + "type": "tracker", + "role": "filter", + "enable": true, + "mode": "bytetrack_lite", + "per_class": true, + "track_classes": [3, 6], + "max_age_ms": 1500, + "min_hits": 2 + }, + { + "id": "logic", + "type": "logic_gate", + "role": "filter", + "enable": true, + "mode": "ppe_boots_check", + "anchor_class": 6, + "boots_class": 3, + "color_check": { + "enable": true, + "method": "hsv", + "dark_threshold": 80, + "roi_expand": 1.0 + }, + "debug": true + }, + { + "id": "osd", + "type": "osd", + "role": "filter", + "enable": true, + "draw_bbox": true, + "draw_text": true, + "labels": ["helmet", "gloves", "vest", "boots", "goggles", "none", "Person", "no_helmet", "no_goggle", "no_gloves", "no_boots"] + }, + { + "id": "post", + "type": "preprocess", + "role": "filter", + "enable": true, + "dst_w": 1280, + "dst_h": 720, + "dst_format": "nv12", + "resize_mode": "stretch", + "use_rga": true + }, + { + "id": "pub", + "type": "publish", + "role": "filter", + "enable": true, + "codec": "h264", + "fps": 30, + "bitrate_kbps": 2000, + "use_mpp": true, + "outputs": [ + {"proto": "rtsp_server", "port": 8555, "path": "/live/cam1"} + ] + } + ], + "edges": [ + ["in", "pre"], + ["pre", "yolo"], + ["yolo", "trk"], + ["trk", "logic"], + ["logic", "osd"], + ["osd", "post"], + ["post", "pub"] + ] + } + ] +} diff --git a/docs/config_guide.md b/docs/config_guide.md index 4bda0cf..4a35d1b 100644 --- a/docs/config_guide.md +++ b/docs/config_guide.md @@ -361,6 +361,55 @@ } ``` +#### logic_gate（逻辑门 - PPE合规检测） + +用于空间关联检测和颜色分析，典型场景：检测人是否穿劳保鞋及颜色是否合规。 + +```json +{ + "id": "logic_ppe", + "type": "logic_gate", + "role": "filter", + "enable": true, + "mode": "ppe_boots_check", // 工作模式 + "anchor_class": 6, // 基准类别：人(Person) + "boots_class": 3, // 目标类别：鞋(boots) + "color_check": { + "enable": true, // 启用颜色检测 + "method": "hsv", // 颜色空间：hsv/rgb/brightness + "dark_threshold": 80, // 深色阈值（0-255） + "roi_expand": 1.0 // 检测框扩大系数 + }, + "debug": false // 调试输出 +} +``` + +**颜色阈值调整建议：** + +| 阈值 | 效果 | 适用场景 | +|------|------|----------| +| **60** | 很严格，只有纯黑才算深色 | 环境光很暗 | +| **80** (默认) | 适中，黑色/深灰通过 | 正常室内光 | +| **100** | 较宽松，灰色也算深色 | 环境光很亮 | +| **120** | 很宽松，浅灰也算深色 | 强光/反光环境 | + +**判断逻辑：** +``` +is_dark = (平均亮度 < dark_threshold) && (深色像素比例 > 60%) +``` + +**使用示例流程：** +``` +input → preprocess → ai_yolo(Person+boots) → tracker → logic_gate → osd → publish + ↓ + 判断：人+鞋同时存在？ + ↓ + 是 → 检查鞋颜色 + ↓ + 深色 → 合规 + 浅色 → 告警 +``` + --- ## 4. 连接关系 (edges) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 03eec0d..a92d5bc 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -407,6 +407,20 @@ set_target_properties(alarm PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${RK_PLUGIN_OUTPUT_DIR} ) +# logic_gate plugin (PPE compliance logic with spatial and color analysis) +add_library(logic_gate SHARED + logic_gate/logic_gate_node.cpp + logic_gate/spatial_matcher.cpp + logic_gate/color_analyzer.cpp +) +target_include_directories(logic_gate PRIVATE ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/third_party) +target_link_libraries(logic_gate PRIVATE project_options Threads::Threads) +set_target_properties(logic_gate PROPERTIES + OUTPUT_NAME "logic_gate" + LIBRARY_OUTPUT_DIRECTORY ${RK_PLUGIN_OUTPUT_DIR} + RUNTIME_OUTPUT_DIRECTORY ${RK_PLUGIN_OUTPUT_DIR} +) + # storage plugin (continuous recording with segment management) add_library(storage SHARED storage/storage_node.cpp @@ -461,7 +475,7 @@ if(RK3588_ENABLE_ZLMEDIAKIT AND RK_ZLMK_API_LIB) ) endif() -install(TARGETS input_rtsp input_file publish preprocess ai_yolo ai_face_det ai_face_recog tracker gate osd alarm storage ai_scheduler +install(TARGETS input_rtsp input_file publish preprocess ai_yolo ai_face_det ai_face_recog tracker gate osd alarm logic_gate storage ai_scheduler LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}/rk3588-media-server/plugins RUNTIME DESTINATION ${CMAKE_INSTALL_LIBDIR}/rk3588-media-server/plugins ) diff --git a/plugins/alarm/alarm_node.cpp b/plugins/alarm/alarm_node.cpp index a022093..b3b8306 100644 --- a/plugins/alarm/alarm_node.cpp +++ b/plugins/alarm/alarm_node.cpp @@ -422,6 +422,18 @@ public: { std::lock_guard lock(mu_); + // 调试：检查 frame 和 det + if (frame->det) { + int no_boots_count = 0; + for (const auto& d : frame->det->items) { + if (d.cls_id == 10) no_boots_count++; + } + if (no_boots_count > 0 || processed_frames_ % 30 == 0) { + LogInfo("[alarm] frame received, dets=" + std::to_string(frame->det->items.size()) + + " no_boots=" + std::to_string(no_boots_count)); + } + } + if (eval_interval_ms_ > 0 && frame->pts > 0) { const int64_t pts_ms = static_cast(frame->pts / 1000ULL); if (last_eval_pts_ms_ > 0 && (pts_ms - last_eval_pts_ms_) < eval_interval_ms_) { @@ -432,7 +444,10 @@ public: } auto result = rule_engine_.Evaluate(frame); - if (result.matched) TriggerAlarm(result, frame); + if (result.matched) { + LogInfo("[alarm] RULE MATCHED: " + result.rule_name); + TriggerAlarm(result, frame); + } EvaluateFaceRulesLocked(frame); ++processed_frames_; diff --git a/plugins/logic_gate/color_analyzer.cpp b/plugins/logic_gate/color_analyzer.cpp new file mode 100644 index 0000000..19dcdcf --- /dev/null +++ b/plugins/logic_gate/color_analyzer.cpp @@ -0,0 +1,239 @@ +#include "color_analyzer.h" +#include "utils/logger.h" +#include +#include +#include +#include + +namespace rk3588 { + +ColorAnalyzer::ColorAnalyzer(const ColorConfig& config) : config_(config) {} + +ColorResult ColorAnalyzer::Analyze(const Frame& frame, const Rect& bbox) { + ColorResult result; + + // 扩展检测框 + Rect expanded = bbox; + float cx = bbox.x + bbox.w * 0.5f; + float cy = bbox.y + bbox.h * 0.5f; + expanded.w = bbox.w * config_.roi_expand; + expanded.h = bbox.h * config_.roi_expand; + expanded.x = cx - expanded.w * 0.5f; + expanded.y = cy - expanded.h * 0.5f; + + // 裁剪区域 + int crop_w = 0, crop_h = 0; + auto cropped = CropRegion(frame, expanded, crop_w, crop_h); + + if (cropped.empty() || crop_w == 0 || crop_h == 0) { + LogWarn("[ColorAnalyzer] Failed to crop region, fmt=" + + std::to_string(static_cast(frame.format)) + + " w=" + std::to_string(frame.width) + + " h=" + std::to_string(frame.height)); + return result; + } + + // 保存调试图像（用于检查裁剪区域） + if (config_.debug_output) { + static int save_count = 0; + if (save_count++ % 30 == 0) { // 每30帧保存一次 + SaveDebugImage(cropped.data(), crop_w, crop_h, "/tmp/boot_debug_" + std::to_string(save_count) + ".ppm"); + LogInfo("[ColorAnalyzer] Saved debug image: /tmp/boot_debug_" + std::to_string(save_count) + ".ppm"); + } + } + + // 根据方法进行分析 + switch (config_.method) { + case ColorMethod::HSV: + result = AnalyzeHSV(cropped.data(), crop_w, crop_h, 3); + break; + case ColorMethod::RGB: + result = AnalyzeRGB(cropped.data(), crop_w, crop_h, 3); + break; + case ColorMethod::BRIGHTNESS: + result = AnalyzeBrightness(cropped.data(), crop_w, crop_h, 3); + break; + default: + result = AnalyzeHSV(cropped.data(), crop_w, crop_h, 3); + break; + } + + // 判断是否为劳保鞋颜色 + result.is_dark = IsSafetyBootsColor(result); + + if (config_.debug_output) { + LogInfo("[ColorAnalyzer] Brightness=" + std::to_string(result.brightness) + + " is_dark=" + (result.is_dark ? "true" : "false") + + " dark_ratio=" + std::to_string(result.dark_ratio) + + " region=" + std::to_string(static_cast(expanded.x)) + "," + + std::to_string(static_cast(expanded.y)) + "," + + std::to_string(static_cast(expanded.w)) + "x" + + std::to_string(static_cast(expanded.h))); + } + + return result; +} + +void ColorAnalyzer::SaveDebugImage(const uint8_t* data, int w, int h, const std::string& filename) { + // 保存为 PPM 格式 (简单文本格式) + std::ofstream file(filename, std::ios::binary); + if (!file) return; + + file << "P6\n" << w << " " << h << "\n255\n"; + for (int i = 0; i < w * h * 3; i++) { + file.put(data[i]); + } + file.close(); +} + +bool ColorAnalyzer::IsSafetyBootsColor(const ColorResult& result) { + // 判断标准：亮度低于阈值，且深色像素比例高 + bool brightness_ok = result.brightness < config_.dark_threshold; + bool ratio_ok = result.dark_ratio > 0.6f; // 60%以上像素为深色 + + return brightness_ok && ratio_ok; +} + +std::vector ColorAnalyzer::CropRegion(const Frame& frame, const Rect& bbox, + int& out_w, int& out_h) { + std::vector result; + + // 检查格式 - 只支持RGB24 + if (frame.format != PixelFormat::RGB && frame.format != PixelFormat::BGR) { + LogWarn("[ColorAnalyzer] Unsupported pixel format: " + std::to_string(static_cast(frame.format)) + + ", expected RGB24/BGR24"); + return result; + } + + // 确保框在有效范围内 + int x = static_cast(std::max(0.0f, bbox.x)); + int y = static_cast(std::max(0.0f, bbox.y)); + int w = static_cast(bbox.w); + int h = static_cast(bbox.h); + + x = std::min(x, frame.width - 1); + y = std::min(y, frame.height - 1); + w = std::min(w, frame.width - x); + h = std::min(h, frame.height - y); + + if (w <= 0 || h <= 0) { + return result; + } + + out_w = w; + out_h = h; + + // RGB格式 + int channels = 3; + result.resize(w * h * channels); + + // 裁剪（逐行拷贝） + for (int row = 0; row < h; ++row) { + const uint8_t* src_row = frame.data + ((y + row) * frame.width + x) * channels; + uint8_t* dst_row = result.data() + row * w * channels; + std::memcpy(dst_row, src_row, w * channels); + } + + return result; +} + +ColorResult ColorAnalyzer::AnalyzeHSV(const uint8_t* data, int w, int h, int channels) { + ColorResult result; + + long long total_v = 0; + long long total_s = 0; + int dark_pixels = 0; + int total_pixels = w * h; + + for (int i = 0; i < total_pixels; ++i) { + uint8_t r = data[i * channels]; + uint8_t g = data[i * channels + 1]; + uint8_t b = data[i * channels + 2]; + + float hsv_h, hsv_s, hsv_v; + RGBtoHSV(r, g, b, hsv_h, hsv_s, hsv_v); + + total_v += static_cast(hsv_v * 255); + total_s += static_cast(hsv_s * 255); + + // V < 0.3 认为是深色 + if (hsv_v < 0.3f) { + dark_pixels++; + } + } + + result.avg_v = total_v / static_cast(total_pixels); + result.avg_s = total_s / static_cast(total_pixels); + result.brightness = result.avg_v; // V通道作为亮度 + result.dark_ratio = dark_pixels / static_cast(total_pixels); + result.light_ratio = 1.0f - result.dark_ratio; + result.confidence = result.dark_ratio; + + return result; +} + +ColorResult ColorAnalyzer::AnalyzeRGB(const uint8_t* data, int w, int h, int channels) { + ColorResult result; + + long long total_brightness = 0; + int dark_pixels = 0; + int total_pixels = w * h; + + for (int i = 0; i < total_pixels; ++i) { + uint8_t r = data[i * channels]; + uint8_t g = data[i * channels + 1]; + uint8_t b = data[i * channels + 2]; + + // 简单亮度公式 + int brightness = (r * 299 + g * 587 + b * 114) / 1000; + total_brightness += brightness; + + if (brightness < config_.dark_threshold) { + dark_pixels++; + } + } + + result.brightness = total_brightness / static_cast(total_pixels); + result.dark_ratio = dark_pixels / static_cast(total_pixels); + result.light_ratio = 1.0f - result.dark_ratio; + result.confidence = result.dark_ratio; + + return result; +} + +ColorResult ColorAnalyzer::AnalyzeBrightness(const uint8_t* data, int w, int h, int channels) { + // 与RGB方法类似，但只关注亮度 + return AnalyzeRGB(data, w, h, channels); +} + +void ColorAnalyzer::RGBtoHSV(uint8_t r, uint8_t g, uint8_t b, float& h, float& s, float& v) { + float rf = r / 255.0f; + float gf = g / 255.0f; + float bf = b / 255.0f; + + float max_val = std::max({rf, gf, bf}); + float min_val = std::min({rf, gf, bf}); + float delta = max_val - min_val; + + // V + v = max_val; + + // S + s = (max_val > 0.0f) ? (delta / max_val) : 0.0f; + + // H + if (delta > 0.0f) { + if (max_val == rf) { + h = 60.0f * (fmod(((gf - bf) / delta), 6.0f)); + } else if (max_val == gf) { + h = 60.0f * (((bf - rf) / delta) + 2.0f); + } else { + h = 60.0f * (((rf - gf) / delta) + 4.0f); + } + if (h < 0.0f) h += 360.0f; + } else { + h = 0.0f; + } +} + +} // namespace rk3588 diff --git a/plugins/logic_gate/color_analyzer.h b/plugins/logic_gate/color_analyzer.h new file mode 100644 index 0000000..235e16d --- /dev/null +++ b/plugins/logic_gate/color_analyzer.h @@ -0,0 +1,71 @@ +#pragma once + +#include "frame/frame.h" + +namespace rk3588 { + +// 颜色分析方法 +enum class ColorMethod { + HSV, // HSV颜色空间 + RGB, // RGB颜色空间 + BRIGHTNESS, // 亮度（灰度） + LAB // LAB颜色空间（更准的亮度） +}; + +// 颜色分析配置 +struct ColorConfig { + ColorMethod method = ColorMethod::HSV; + int dark_threshold = 80; // 深色阈值（亮度0-255） + float roi_expand = 1.0f; // ROI扩大系数 + bool debug_output = false; // 是否输出调试信息 +}; + +// 颜色分析结果 +struct ColorResult { + bool is_dark = false; // 是否为深色 + float brightness = 0.0f; // 平均亮度（0-255） + float confidence = 0.0f; // 置信度 + + // HSV统计 + float avg_h = 0.0f; + float avg_s = 0.0f; + float avg_v = 0.0f; + + // 颜色分布 + float dark_ratio = 0.0f; // 深色像素比例 + float light_ratio = 0.0f; // 浅色像素比例 +}; + +class ColorAnalyzer { +public: + ColorAnalyzer(const ColorConfig& config); + + // 分析检测框内的颜色 + ColorResult Analyze(const Frame& frame, const Rect& bbox); + + // 判断是否为劳保鞋颜色（深色） + bool IsSafetyBootsColor(const ColorResult& result); + + // 保存调试图像 + void SaveDebugImage(const uint8_t* data, int w, int h, const std::string& filename); + +private: + ColorConfig config_; + + // 裁剪图像区域 + std::vector CropRegion(const Frame& frame, const Rect& bbox, int& out_w, int& out_h); + + // HSV分析方法 + ColorResult AnalyzeHSV(const uint8_t* data, int w, int h, int channels); + + // RGB分析方法 + ColorResult AnalyzeRGB(const uint8_t* data, int w, int h, int channels); + + // 亮度分析方法 + ColorResult AnalyzeBrightness(const uint8_t* data, int w, int h, int channels); + + // RGB转HSV + void RGBtoHSV(uint8_t r, uint8_t g, uint8_t b, float& h, float& s, float& v); +}; + +} // namespace rk3588 diff --git a/plugins/logic_gate/logic_gate_node.cpp b/plugins/logic_gate/logic_gate_node.cpp new file mode 100644 index 0000000..08daa54 --- /dev/null +++ b/plugins/logic_gate/logic_gate_node.cpp @@ -0,0 +1,192 @@ +#include +#include +#include +#include +#include +#include + +#include "node.h" +#include "frame/frame.h" +#include "utils/logger.h" +#include "utils/simple_json.h" + +#include "spatial_matcher.h" +#include "color_analyzer.h" + +namespace rk3588 { + +// 违规类型 +enum class ViolationType { + NONE, + MISSING_BOOTS, // 没有鞋 + WRONG_COLOR_BOOTS, // 鞋颜色不对 +}; + +// 违规信息 +struct ViolationInfo { + ViolationType type = ViolationType::NONE; + std::string description; + int track_id = -1; + float confidence = 0.0f; + Rect violation_region; +}; + +// Logic Gate 配置 +struct LogicGateConfig { + std::string mode = "ppe_boots_check"; + int anchor_class = 6; // Person + int boots_class = 3; // boots + + ColorConfig color; + bool enable_color_check = true; + + std::string violation_key = "ppe_violation"; + bool pass_through = true; + bool debug = false; +}; + +class LogicGateNode : public INode { +public: + LogicGateNode() = default; + ~LogicGateNode() override = default; + + std::string Id() const override { return id_; } + std::string Type() const override { return "logic_gate"; } + + bool Init(const SimpleJson& config, const NodeContext& ctx) override { + id_ = config.ValueOr("id", "logic_gate"); + config_ = ParseConfig(config); + output_queues_ = ctx.output_queues; + + // 初始化颜色分析器 + if (config_.enable_color_check) { + color_analyzer_ = std::make_unique(config_.color); + } + + LogInfo("[LogicGateNode] Initialized, id=" + id_ + " mode=" + config_.mode + + " output_queues=" + std::to_string(output_queues_.size())); + return true; + } + + bool Start() override { + LogInfo("[LogicGateNode] started id=" + id_); + return true; + } + + void Stop() override { + } + + NodeStatus Process(FramePtr frame) override { + if (!frame || !frame->det) { + PushToDownstream(frame); + return NodeStatus::OK; + } + + if (config_.mode == "ppe_boots_check") { + ProcessPpeBootsCheck(frame); + } + + PushToDownstream(frame); + return NodeStatus::OK; + } + +private: + std::string id_; + LogicGateConfig config_; + std::unique_ptr color_analyzer_; + std::vector>> output_queues_; + + LogicGateConfig ParseConfig(const SimpleJson& cfg) { + LogicGateConfig config; + + config.mode = cfg.ValueOr("mode", "ppe_boots_check"); + config.anchor_class = cfg.ValueOr("anchor_class", 6); + config.boots_class = cfg.ValueOr("boots_class", 3); + config.violation_key = cfg.ValueOr("violation_key", "ppe_violation"); + config.pass_through = cfg.ValueOr("pass_through", true); + config.debug = cfg.ValueOr("debug", false); + config.enable_color_check = cfg.ValueOr("enable_color_check", true); + + // 解析颜色配置 + if (const SimpleJson* color = cfg.Find("color_check")) { + std::string method = color->ValueOr("method", "hsv"); + if (method == "hsv") config.color.method = ColorMethod::HSV; + else if (method == "rgb") config.color.method = ColorMethod::RGB; + else if (method == "brightness") config.color.method = ColorMethod::BRIGHTNESS; + + config.color.dark_threshold = color->ValueOr("dark_threshold", 80); + config.color.roi_expand = color->ValueOr("roi_expand", 1.0f); + config.color.debug_output = config.debug; + } + + return config; + } + + void PushToDownstream(FramePtr frame) { + for (auto& q : output_queues_) { + if (q) q->Push(frame); + } + } + + void ProcessPpeBootsCheck(FramePtr frame) { + const auto& detections = frame->det->items; + + // 收集所有人和鞋 + std::vector persons; + std::vector boots; + + for (const auto& det : detections) { + if (det.cls_id == config_.anchor_class) { + persons.push_back(det); + } else if (det.cls_id == config_.boots_class) { + boots.push_back(det); + } + } + + if (config_.debug) { + LogInfo("[LogicGateNode] Persons=" + std::to_string(persons.size()) + + " Boots=" + std::to_string(boots.size())); + } + + // 简化逻辑：必须同时检测到人和鞋，才开始判断 + if (persons.empty() || boots.empty()) { + return; + } + + std::vector violations; + + // 对每只鞋进行颜色检查 + for (const auto& boot : boots) { + if (config_.enable_color_check && color_analyzer_) { + auto color_result = color_analyzer_->Analyze(*frame, boot.bbox); + + if (config_.debug) { + LogInfo("[LogicGateNode] Boot brightness=" + + std::to_string(color_result.brightness) + + " is_dark=" + (color_result.is_dark ? "true" : "false")); + } + + if (!color_result.is_dark) { + // 颜色不对，添加 no_boots 检测框 + Detection no_boots_det; + no_boots_det.cls_id = 10; // no_boots + no_boots_det.track_id = std::max(0, boot.track_id); + no_boots_det.score = std::max(0.5f, color_result.confidence); + no_boots_det.bbox = boot.bbox; + + frame->det->items.push_back(no_boots_det); + + if (config_.debug) { + LogInfo("[LogicGateNode] VIOLATION: Non-compliant boots color (brightness=" + + std::to_string(static_cast(color_result.brightness)) + + ") added no_boots(cls=10) track_id=" + std::to_string(no_boots_det.track_id)); + } + } + } + } + } +}; + +REGISTER_NODE(LogicGateNode, "logic_gate"); + +} // namespace rk3588 diff --git a/plugins/logic_gate/spatial_matcher.cpp b/plugins/logic_gate/spatial_matcher.cpp new file mode 100644 index 0000000..b26e994 --- /dev/null +++ b/plugins/logic_gate/spatial_matcher.cpp @@ -0,0 +1,106 @@ +#include "spatial_matcher.h" +#include +#include + +namespace rk3588 { + +SpatialMatcher::SpatialMatcher(const SpatialConfig& config) : config_(config) {} + +SpatialMatchResult SpatialMatcher::Match(const Detection& anchor, const Detection& target, + int img_w, int img_h) { + SpatialMatchResult result; + + // 根据关系类型进行匹配 + switch (config_.relation) { + case SpatialRelation::BELOW: { + // 计算脚下区域 + result.foot_region = CalculateFootRegion(anchor.bbox, img_w, img_h); + + // 计算目标与脚下区域的IOU + result.iou = CalculateIoU(result.foot_region, target.bbox); + result.matched = (result.iou >= config_.iou_threshold); + break; + } + + case SpatialRelation::INSIDE: { + result.iou = CalculateIoU(anchor.bbox, target.bbox); + result.matched = (result.iou >= config_.iou_threshold); + break; + } + + case SpatialRelation::OVERLAP: { + result.iou = CalculateIoU(anchor.bbox, target.bbox); + result.matched = (result.iou > 0); // 只要有重叠就算匹配 + break; + } + + case SpatialRelation::NEAR: { + result.distance = CalculateDistance(anchor.bbox, target.bbox, img_w, img_h); + result.matched = (result.distance <= config_.max_distance); + break; + } + + default: + break; + } + + return result; +} + +Rect SpatialMatcher::CalculateFootRegion(const Rect& person_bbox, int img_w, int img_h) { + // 脚下区域：人体框底部向下扩展 foot_ratio 倍高度 + float foot_height = person_bbox.h * config_.foot_ratio; + + Rect foot; + foot.x = person_bbox.x - person_bbox.w * (config_.expand_ratio - 1.0f) * 0.5f; + foot.y = person_bbox.y + person_bbox.h; // 从人体底部开始 + foot.w = person_bbox.w * config_.expand_ratio; + foot.h = foot_height * config_.expand_ratio; + + // 确保在图像范围内 + foot.x = std::max(0.0f, foot.x); + foot.y = std::max(0.0f, foot.y); + foot.w = std::min(static_cast(img_w) - foot.x, foot.w); + foot.h = std::min(static_cast(img_h) - foot.y, foot.h); + + return foot; +} + +float SpatialMatcher::CalculateIoU(const Rect& a, const Rect& b) { + // 计算交集 + float x1 = std::max(a.x, b.x); + float y1 = std::max(a.y, b.y); + float x2 = std::min(a.x + a.w, b.x + b.w); + float y2 = std::min(a.y + a.h, b.y + b.h); + + if (x2 <= x1 || y2 <= y1) { + return 0.0f; + } + + float intersection = (x2 - x1) * (y2 - y1); + float area_a = a.w * a.h; + float area_b = b.w * b.h; + float union_area = area_a + area_b - intersection; + + return union_area > 0 ? intersection / union_area : 0.0f; +} + +float SpatialMatcher::CalculateDistance(const Rect& anchor, const Rect& target, int img_w, int img_h) { + // 计算中心点距离（归一化） + float cx1 = (anchor.x + anchor.w * 0.5f) / img_w; + float cy1 = (anchor.y + anchor.h * 0.5f) / img_h; + float cx2 = (target.x + target.w * 0.5f) / img_w; + float cy2 = (target.y + target.h * 0.5f) / img_h; + + float dx = cx1 - cx2; + float dy = cy1 - cy2; + + return std::sqrt(dx * dx + dy * dy); +} + +bool SpatialMatcher::IsBelow(const Rect& anchor, const Rect& target, int img_w, int img_h) { + // 简单判断：目标的顶部是否在锚点底部的下方 + return target.y >= (anchor.y + anchor.h * 0.8f); +} + +} // namespace rk3588 diff --git a/plugins/logic_gate/spatial_matcher.h b/plugins/logic_gate/spatial_matcher.h new file mode 100644 index 0000000..0d1e734 --- /dev/null +++ b/plugins/logic_gate/spatial_matcher.h @@ -0,0 +1,58 @@ +#pragma once + +#include "frame/frame.h" +#include + +namespace rk3588 { + +// 空间关系类型 +enum class SpatialRelation { + BELOW, // 目标在锚点下方 + ABOVE, // 目标在锚点上方 + INSIDE, // 目标在锚点内部 + OVERLAP, // 目标与锚点重叠 + NEAR // 目标在锚点附近 +}; + +// 空间匹配配置 +struct SpatialConfig { + SpatialRelation relation = SpatialRelation::BELOW; + float iou_threshold = 0.3f; // IOU阈值 + float foot_ratio = 0.3f; // 脚下区域比例（人体高度） + float expand_ratio = 1.0f; // 框扩大系数 + float max_distance = 0.5f; // 最大归一化距离（NEAR模式用） +}; + +// 匹配结果 +struct SpatialMatchResult { + bool matched = false; + float iou = 0.0f; + float distance = 0.0f; // 归一化距离 + Rect foot_region; // 脚下区域（用于可视化） +}; + +class SpatialMatcher { +public: + SpatialMatcher(const SpatialConfig& config); + + // 检查目标是否在锚点的指定空间位置 + SpatialMatchResult Match(const Detection& anchor, const Detection& target, + int img_w, int img_h); + + // 计算脚下区域 + Rect CalculateFootRegion(const Rect& person_bbox, int img_w, int img_h); + +private: + SpatialConfig config_; + + // 计算两个框的IOU + float CalculateIoU(const Rect& a, const Rect& b); + + // 计算归一化中心点距离 + float CalculateDistance(const Rect& anchor, const Rect& target, int img_w, int img_h); + + // 检查目标是否在锚点下方 + bool IsBelow(const Rect& anchor, const Rect& target, int img_w, int img_h); +}; + +} // namespace rk3588 diff --git a/plugins/osd/osd_node.cpp b/plugins/osd/osd_node.cpp index d3dfed1..822058d 100644 --- a/plugins/osd/osd_node.cpp +++ b/plugins/osd/osd_node.cpp @@ -393,6 +393,7 @@ public: return false; } output_queues_ = ctx.output_queues; + LogInfo("[osd] Init: id=" + id_ + " output_queues=" + std::to_string(output_queues_.size())); return true; } diff --git a/plugins/preprocess/preprocess_node.cpp b/plugins/preprocess/preprocess_node.cpp index 85cda1f..cbe7fa6 100644 --- a/plugins/preprocess/preprocess_node.cpp +++ b/plugins/preprocess/preprocess_node.cpp @@ -327,6 +327,10 @@ public: void Stop() override {} NodeStatus Process(FramePtr frame) override { + static int process_count = 0; + if (id_.find("post") != std::string::npos && process_count++ % 30 == 0) { + LogInfo("[preprocess] post_cam1 Process called, frame=" + std::to_string(frame ? 1 : 0)); + } if (!frame) return NodeStatus::DROP; if (!image_processor_) return NodeStatus::ERROR; diff --git a/plugins/publish/publish_node.cpp b/plugins/publish/publish_node.cpp index 4725b23..a03ccf8 100644 --- a/plugins/publish/publish_node.cpp +++ b/plugins/publish/publish_node.cpp @@ -637,6 +637,7 @@ public: id_ = config.ValueOr("id", "publish"); input_queue_ = ctx.input_queue; output_queues_ = ctx.output_queues; + LogInfo("[publish] Init: id=" + id_ + " output_queues=" + std::to_string(output_queues_.size())); codec_ = config.ValueOr("codec", "h264"); fps_ = config.ValueOr("fps", 25); gop_ = config.ValueOr("gop", 50); @@ -805,7 +806,14 @@ public: } NodeStatus Process(FramePtr frame) override { - if (!frame) return NodeStatus::DROP; + static int process_count = 0; + if (process_count++ % 30 == 0) { + LogInfo("[publish] Process called, frame=" + std::to_string(frame ? 1 : 0)); + } + if (!frame) { + LogWarn("[publish] received null frame"); + return NodeStatus::DROP; + } { std::lock_guard lock(mu_); @@ -821,13 +829,33 @@ public: #endif } + static int frame_count = 0; + if (frame_count++ % 30 == 0) { + int no_boots = 0; + if (frame->det) { + for (const auto& d : frame->det->items) { + if (d.cls_id == 10) no_boots++; + } + } + LogInfo("[publish] pushing frame to downstream, dets=" + + std::to_string(frame->det ? frame->det->items.size() : 0) + + " no_boots=" + std::to_string(no_boots) + + " queues=" + std::to_string(output_queues_.size())); + } + PushToDownstream(frame); return NodeStatus::OK; } private: void PushToDownstream(const FramePtr& frame) { - if (output_queues_.empty()) return; + if (output_queues_.empty()) { + static int log_count = 0; + if (log_count++ % 100 == 0) { + LogInfo("[publish] no output queues"); + } + return; + } for (auto& q : output_queues_) { if (q) q->Push(frame); }