diff --git a/configs/person_shoe_two_stage_recommended_alarm.json b/configs/person_shoe_two_stage_recommended_alarm.json new file mode 100644 index 0000000..ba2ad21 --- /dev/null +++ b/configs/person_shoe_two_stage_recommended_alarm.json @@ -0,0 +1,228 @@ +{ + "queue": { + "size": 8, + "strategy": "drop_oldest" + }, + "graphs": [ + { + "name": "person_shoe_two_stage_recommended_alarm", + "executor": { + "batch_size": 2, + "run_budget": 8 + }, + "nodes": [ + { + "id": "in", + "type": "input_rtsp", + "role": "source", + "enable": true, + "url": "rtsp://10.0.0.49:8554/cam", + "fps": 30, + "width": 1920, + "height": 1080, + "use_ffmpeg": true, + "use_mpp": false, + "force_tcp": true + }, + { + "id": "pre_rgb", + "type": "preprocess", + "role": "filter", + "enable": true, + "dst_w": 1920, + "dst_h": 1080, + "dst_format": "rgb", + "dst_packed": true, + "resize_mode": "stretch", + "rga_gate": "person_shoe_two_stage_recommended_alarm", + "use_rga": true + }, + { + "id": "person_det", + "type": "ai_yolo", + "role": "filter", + "enable": true, + "cpu_affinity": [4], + "use_rga": true, + "rga_gate": "person_shoe_two_stage_recommended_alarm", + "rga_max_inflight": 4, + "dst_packed": true, + "use_dma_input": true, + "infer_fps": 2, + "infer_phase_ms": 0, + "model_path": "./models/yolov8n-640.rknn", + "model_version": "v8", + "model_w": 640, + "model_h": 640, + "num_classes": 80, + "conf": 0.35, + "nms": 0.45, + "class_filter": [0], + "bbox_expand": { + "enable": true, + "class_id": 0, + "left": 0.06, + "right": 0.06, + "top": 0.04, + "bottom": 0.16 + } + }, + { + "id": "person_trk", + "type": "tracker", + "role": "filter", + "enable": true, + "cpu_affinity": [5], + "mode": "bytetrack_lite", + "per_class": true, + "track_classes": [0], + "high_th": 0.55, + "low_th": 0.10, + "iou_th": 0.3, + "max_age_ms": 900, + "min_hits": 2 + }, + { + "id": "shoe_det", + "type": "ai_shoe_det", + "role": "filter", + "enable": true, + "cpu_affinity": [6], + "use_rga": true, + "rga_gate": "person_shoe_two_stage_recommended_alarm", + "rga_max_inflight": 4, + "dst_packed": true, + "use_dma_input": false, + "infer_fps": 2, + "infer_phase_ms": 150, + "model_path": "./models/shoe_detector_openimages_ppe_v1.rknn", + "model_w": 640, + "model_h": 640, + "conf": 0.15, + "nms": 0.45, + "v8_box_format": "cxcywh", + "append_detections": true, + "dynamic_roi": { + "enable": true, + "person_class_id": 0, + "shoe_class_id": 1, + "debug_roi_class_id": -1, + "max_rois": 3, + "min_person_height": 60, + "x_offset": -0.24, + "y_offset": 0.64, + "width_scale": 1.48, + "height_scale": 0.58 + } + }, + { + "id": "shoe_logic", + "type": "logic_gate", + "role": "filter", + "enable": true, + "cpu_affinity": [6], + "mode": "person_shoe_check", + "debug": false, + "person_shoe_check": { + "person_class": 0, + "shoe_class": 1, + "violation_class": 2, + "min_person_score": 0.30, + "min_shoe_score": 0.15, + "require_person_track_id": true, + "use_person_bbox_for_violation": true, + "attach_person_track_to_shoe": true, + "match_iou": 0.02, + "foot_region": { + "x_offset": -0.24, + "y_offset": 0.64, + "width_scale": 1.48, + "height_scale": 0.58 + } + } + }, + { + "id": "osd", + "type": "osd", + "role": "filter", + "enable": true, + "cpu_affinity": [7], + "draw_bbox": true, + "draw_text": false, + "use_rga_bbox": false, + "labels": ["person", "shoe", "no_shoe"] + }, + { + "id": "post", + "type": "preprocess", + "role": "filter", + "enable": true, + "cpu_affinity": [7], + "dst_w": 1920, + "dst_h": 1080, + "dst_format": "nv12", + "resize_mode": "stretch", + "rga_gate": "person_shoe_two_stage_recommended_alarm", + "use_rga": true + }, + { + "id": "pub", + "type": "publish", + "role": "filter", + "enable": true, + "cpu_affinity": [3], + "codec": "h264", + "fps": 30, + "bitrate_kbps": 2000, + "mpp_output_timeout_ms": 50, + "mpp_packet_wait_ms": 10, + "use_mpp": true, + "outputs": [ + {"proto": "rtsp_server", "port": 8555, "path": "/live/cam1"} + ] + }, + { + "id": "alarm", + "type": "alarm", + "role": "sink", + "enable": true, + "eval_fps": 2, + "labels": ["person", "shoe", "no_shoe"], + "rules": [ + { + "name": "missing_shoe", + "class_ids": [2], + "roi": {"x": 0.0, "y": 0.0, "w": 1.0, "h": 1.0}, + "min_score": 0.30, + "require_track_id": true, + "min_duration_ms": 800, + "min_hits": 2, + "hit_window_ms": 2000, + "cooldown_ms": 10000, + "per_track_cooldown_ms": 15000 + } + ], + "actions": { + "log": { + "enable": true, + "level": "info", + "include_detections": true, + "min_interval_ms": 2000 + } + } + } + ], + "edges": [ + ["in", "pre_rgb"], + ["pre_rgb", "person_det"], + ["person_det", "person_trk"], + ["person_trk", "shoe_det", {"queue": {"size": 16, "strategy": "drop_oldest"}}], + ["shoe_det", "shoe_logic", {"queue": {"size": 16, "strategy": "drop_oldest"}}], + ["shoe_logic", "osd", {"queue": {"size": 16, "strategy": "drop_oldest"}}], + ["osd", "post", {"queue": {"size": 32, "strategy": "drop_oldest"}}], + ["post", "pub", {"queue": {"size": 64, "strategy": "drop_oldest"}}], + ["pub", "alarm", {"queue": {"size": 64, "strategy": "drop_oldest"}}] + ] + } + ] +} diff --git a/docs/deployment.md b/docs/deployment.md index b64189f..df37bf9 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -159,6 +159,12 @@ sudo ./scripts/ops.sh ddr-restore ./build/media-server -c configs/person_shoe_two_stage_recommended.json ``` +如需直接启用“缺鞋”判定和按人冷却告警，可使用： + +```bash +./build/media-server -c configs/person_shoe_two_stage_recommended_alarm.json +``` + 说明： - 输入端：`use_ffmpeg: true`、`use_mpp: false` diff --git a/plugins/logic_gate/logic_gate_node.cpp b/plugins/logic_gate/logic_gate_node.cpp index ef31007..5cfe0cd 100644 --- a/plugins/logic_gate/logic_gate_node.cpp +++ b/plugins/logic_gate/logic_gate_node.cpp @@ -1,234 +1,305 @@ -#include -#include -#include -#include #include -#include +#include +#include +#include +#include -#include "node.h" +#include "color_analyzer.h" #include "frame/frame.h" +#include "node.h" #include "utils/logger.h" #include "utils/simple_json.h" -#include "spatial_matcher.h" -#include "color_analyzer.h" - namespace rk3588 { -// 违规类型 -enum class ViolationType { - NONE, - MISSING_BOOTS, // 没有鞋 - WRONG_COLOR_BOOTS, // 鞋颜色不对 +struct PersonShoeCheckConfig { + int person_class = 0; + int shoe_class = 1; + int violation_class = 2; + float min_person_score = 0.0f; + float min_shoe_score = 0.0f; + bool require_person_track_id = false; + bool use_person_bbox_for_violation = true; + bool attach_person_track_to_shoe = true; + float match_iou = 0.02f; + float x_offset = -0.20f; + float y_offset = 0.64f; + float width_scale = 1.48f; + float height_scale = 0.58f; }; -// 违规信息 -struct ViolationInfo { - ViolationType type = ViolationType::NONE; - std::string description; - int track_id = -1; - float confidence = 0.0f; - Rect violation_region; -}; - -// Logic Gate 配置 struct LogicGateConfig { std::string mode = "ppe_boots_check"; - int anchor_class = 6; // Person - int boots_class = 3; // boots - - ColorConfig color; + int anchor_class = 6; + int boots_class = 3; bool enable_color_check = true; - - std::string violation_key = "ppe_violation"; bool pass_through = true; bool debug = false; + std::string violation_key = "ppe_violation"; + ColorConfig color; + PersonShoeCheckConfig person_shoe; }; class LogicGateNode : public INode { public: - LogicGateNode() = default; - ~LogicGateNode() override = default; - std::string Id() const override { return id_; } std::string Type() const override { return "logic_gate"; } - + bool Init(const SimpleJson& config, const NodeContext& ctx) override { id_ = config.ValueOr("id", "logic_gate"); config_ = ParseConfig(config); output_queues_ = ctx.output_queues; - - // 初始化颜色分析器 + if (config_.enable_color_check) { color_analyzer_ = std::make_unique(config_.color); } - - LogInfo("[LogicGateNode] Initialized, id=" + id_ + " mode=" + config_.mode + - " output_queues=" + std::to_string(output_queues_.size())); + + LogInfo("[logic_gate] initialized id=" + id_ + " mode=" + config_.mode); return true; } - + bool Start() override { - LogInfo("[LogicGateNode] started id=" + id_); + LogInfo("[logic_gate] started id=" + id_); return true; } - - void Stop() override { - } - + + void Stop() override {} + NodeStatus Process(FramePtr frame) override { if (!frame || !frame->det) { PushToDownstream(frame); return NodeStatus::OK; } - - if (config_.mode == "ppe_boots_check") { + + if (config_.mode == "person_shoe_check") { + ProcessPersonShoeCheck(frame); + } else if (config_.mode == "ppe_boots_check") { ProcessPpeBootsCheck(frame); } - + PushToDownstream(frame); return NodeStatus::OK; } private: - std::string id_; - LogicGateConfig config_; - std::unique_ptr color_analyzer_; - std::vector>> output_queues_; - + static Rect ClipRect(const Rect& r, int img_w, int img_h) { + Rect out = r; + out.x = std::max(0.0f, out.x); + out.y = std::max(0.0f, out.y); + if (img_w > 0) out.w = std::min(out.w, static_cast(img_w) - out.x); + if (img_h > 0) out.h = std::min(out.h, static_cast(img_h) - out.y); + out.w = std::max(0.0f, out.w); + out.h = std::max(0.0f, out.h); + return out; + } + + static float IoU(const Rect& a, const Rect& b) { + const float x1 = std::max(a.x, b.x); + const float y1 = std::max(a.y, b.y); + const float x2 = std::min(a.x + a.w, b.x + b.w); + const float y2 = std::min(a.y + a.h, b.y + b.h); + if (x2 <= x1 || y2 <= y1) return 0.0f; + const float inter = (x2 - x1) * (y2 - y1); + const float area_a = a.w * a.h; + const float area_b = b.w * b.h; + const float uni = area_a + area_b - inter; + return uni > 0.0f ? (inter / uni) : 0.0f; + } + + static bool CenterInside(const Rect& outer, const Rect& inner) { + const float cx = inner.x + inner.w * 0.5f; + const float cy = inner.y + inner.h * 0.5f; + return cx >= outer.x && cx <= (outer.x + outer.w) && + cy >= outer.y && cy <= (outer.y + outer.h); + } + LogicGateConfig ParseConfig(const SimpleJson& cfg) { LogicGateConfig config; - - config.mode = cfg.ValueOr("mode", "ppe_boots_check"); - config.anchor_class = cfg.ValueOr("anchor_class", 6); - config.boots_class = cfg.ValueOr("boots_class", 3); - config.violation_key = cfg.ValueOr("violation_key", "ppe_violation"); - config.pass_through = cfg.ValueOr("pass_through", true); - config.debug = cfg.ValueOr("debug", false); - // 解析颜色配置 + config.mode = cfg.ValueOr("mode", config.mode); + config.anchor_class = cfg.ValueOr("anchor_class", config.anchor_class); + config.boots_class = cfg.ValueOr("boots_class", config.boots_class); + config.violation_key = cfg.ValueOr("violation_key", config.violation_key); + config.pass_through = cfg.ValueOr("pass_through", config.pass_through); + config.debug = cfg.ValueOr("debug", config.debug); + if (const SimpleJson* color = cfg.Find("color_check")) { config.enable_color_check = color->ValueOr("enable", true); - std::string method = color->ValueOr("method", "hsv"); + const std::string method = color->ValueOr("method", "hsv"); if (method == "hsv") config.color.method = ColorMethod::HSV; else if (method == "rgb") config.color.method = ColorMethod::RGB; else if (method == "brightness") config.color.method = ColorMethod::BRIGHTNESS; - config.color.dark_threshold = color->ValueOr("dark_threshold", 80); config.color.roi_expand = color->ValueOr("roi_expand", 1.0f); config.color.debug_output = config.debug; } - + + if (const SimpleJson* ps = cfg.Find("person_shoe_check"); ps && ps->IsObject()) { + config.person_shoe.person_class = ps->ValueOr("person_class", config.person_shoe.person_class); + config.person_shoe.shoe_class = ps->ValueOr("shoe_class", config.person_shoe.shoe_class); + config.person_shoe.violation_class = ps->ValueOr("violation_class", config.person_shoe.violation_class); + config.person_shoe.min_person_score = ps->ValueOr("min_person_score", config.person_shoe.min_person_score); + config.person_shoe.min_shoe_score = ps->ValueOr("min_shoe_score", config.person_shoe.min_shoe_score); + config.person_shoe.require_person_track_id = ps->ValueOr("require_person_track_id", config.person_shoe.require_person_track_id); + config.person_shoe.use_person_bbox_for_violation = ps->ValueOr( + "use_person_bbox_for_violation", config.person_shoe.use_person_bbox_for_violation); + config.person_shoe.attach_person_track_to_shoe = ps->ValueOr( + "attach_person_track_to_shoe", config.person_shoe.attach_person_track_to_shoe); + config.person_shoe.match_iou = ps->ValueOr("match_iou", config.person_shoe.match_iou); + if (const SimpleJson* fr = ps->Find("foot_region"); fr && fr->IsObject()) { + config.person_shoe.x_offset = fr->ValueOr("x_offset", config.person_shoe.x_offset); + config.person_shoe.y_offset = fr->ValueOr("y_offset", config.person_shoe.y_offset); + config.person_shoe.width_scale = fr->ValueOr("width_scale", config.person_shoe.width_scale); + config.person_shoe.height_scale = fr->ValueOr("height_scale", config.person_shoe.height_scale); + } + } + return config; } - - void PushToDownstream(FramePtr frame) { + + void PushToDownstream(const FramePtr& frame) { for (auto& q : output_queues_) { if (q) q->Push(frame); } } - - // 将检测坐标（相对于原始图像）映射到当前帧坐标 - Rect MapDetCoordToFrame(const Rect& det_bbox, FramePtr frame) { - if (!frame->transform_meta || !frame->transform_meta->valid) { - return det_bbox; // 无变换信息，直接使用 - } - + + Rect MapDetCoordToFrame(const Rect& det_bbox, const FramePtr& frame) const { + if (!frame->transform_meta || !frame->transform_meta->valid) return det_bbox; const auto& meta = *frame->transform_meta; - if (meta.src_w <= 0 || meta.src_h <= 0 || frame->width <= 0 || frame->height <= 0) { - return det_bbox; - } - - // 计算缩放因子：检测坐标是基于 src_w x src_h 的 - float scale_x = static_cast(frame->width) / meta.src_w; - float scale_y = static_cast(frame->height) / meta.src_h; - + if (meta.src_w <= 0 || meta.src_h <= 0 || frame->width <= 0 || frame->height <= 0) return det_bbox; + + const float scale_x = static_cast(frame->width) / static_cast(meta.src_w); + const float scale_y = static_cast(frame->height) / static_cast(meta.src_h); Rect mapped; mapped.x = det_bbox.x * scale_x; mapped.y = det_bbox.y * scale_y; mapped.w = det_bbox.w * scale_x; mapped.h = det_bbox.h * scale_y; - return mapped; } - - void ProcessPpeBootsCheck(FramePtr frame) { - const auto& detections = frame->det->items; - - // 收集所有人和鞋 - std::vector persons; - std::vector boots; - - for (const auto& det : detections) { - if (det.cls_id == config_.anchor_class) { - persons.push_back(det); - } else if (det.cls_id == config_.boots_class) { - boots.push_back(det); + + Rect BuildFootRegion(const Rect& person_bbox, int img_w, int img_h) const { + Rect roi; + roi.x = person_bbox.x + person_bbox.w * config_.person_shoe.x_offset; + roi.y = person_bbox.y + person_bbox.h * config_.person_shoe.y_offset; + roi.w = person_bbox.w * config_.person_shoe.width_scale; + roi.h = person_bbox.h * config_.person_shoe.height_scale; + return ClipRect(roi, img_w, img_h); + } + + void ProcessPersonShoeCheck(const FramePtr& frame) { + auto& items = frame->det->items; + const int img_w = frame->det->img_w > 0 ? frame->det->img_w : frame->width; + const int img_h = frame->det->img_h > 0 ? frame->det->img_h : frame->height; + + std::vector person_indices; + std::vector shoe_indices; + person_indices.reserve(items.size()); + shoe_indices.reserve(items.size()); + for (size_t i = 0; i < items.size(); ++i) { + if (items[i].cls_id == config_.person_shoe.person_class) { + person_indices.push_back(i); + } else if (items[i].cls_id == config_.person_shoe.shoe_class) { + shoe_indices.push_back(i); } } - + if (config_.debug) { - LogInfo("[LogicGateNode] Persons=" + std::to_string(persons.size()) + - " Boots=" + std::to_string(boots.size()) + - " Frame=" + std::to_string(frame->width) + "x" + std::to_string(frame->height)); - if (frame->transform_meta && frame->transform_meta->valid) { - LogInfo("[LogicGateNode] TransformMeta: src=" + std::to_string(frame->transform_meta->src_w) + - "x" + std::to_string(frame->transform_meta->src_h)); + LogInfo("[logic_gate] person_shoe_check persons=" + std::to_string(person_indices.size()) + + " shoes=" + std::to_string(shoe_indices.size())); + } + + std::vector shoe_used(shoe_indices.size(), false); + std::vector appended; + + for (size_t pi = 0; pi < person_indices.size(); ++pi) { + Detection& person = items[person_indices[pi]]; + if (person.score < config_.person_shoe.min_person_score) continue; + if (config_.person_shoe.require_person_track_id && person.track_id < 0) continue; + + const Rect foot_region = BuildFootRegion(person.bbox, img_w, img_h); + int best_shoe_local = -1; + float best_match_score = 0.0f; + + for (size_t si = 0; si < shoe_indices.size(); ++si) { + if (shoe_used[si]) continue; + Detection& shoe = items[shoe_indices[si]]; + if (shoe.score < config_.person_shoe.min_shoe_score) continue; + + const float iou = IoU(foot_region, shoe.bbox); + const bool center_inside = CenterInside(foot_region, shoe.bbox); + if (!center_inside && iou < config_.person_shoe.match_iou) continue; + + const float match_score = (center_inside ? 1.0f : 0.0f) + iou + shoe.score * 0.1f; + if (best_shoe_local < 0 || match_score > best_match_score) { + best_shoe_local = static_cast(si); + best_match_score = match_score; + } + } + + if (best_shoe_local >= 0) { + shoe_used[static_cast(best_shoe_local)] = true; + Detection& matched_shoe = items[shoe_indices[static_cast(best_shoe_local)]]; + if (config_.person_shoe.attach_person_track_to_shoe && + matched_shoe.track_id < 0 && person.track_id >= 0) { + matched_shoe.track_id = person.track_id; + } + continue; + } + + Detection no_shoe; + no_shoe.cls_id = config_.person_shoe.violation_class; + no_shoe.track_id = person.track_id; + no_shoe.score = std::max(0.5f, person.score); + no_shoe.bbox = config_.person_shoe.use_person_bbox_for_violation ? person.bbox : foot_region; + appended.push_back(no_shoe); + + if (config_.debug) { + LogInfo("[logic_gate] no_shoe track_id=" + std::to_string(no_shoe.track_id) + + " bbox=(" + std::to_string(static_cast(no_shoe.bbox.x)) + "," + + std::to_string(static_cast(no_shoe.bbox.y)) + "," + + std::to_string(static_cast(no_shoe.bbox.w)) + "," + + std::to_string(static_cast(no_shoe.bbox.h)) + ")"); } } - - // 简化逻辑：必须同时检测到人和鞋，才开始判断 - if (persons.empty() || boots.empty()) { - return; + + if (!appended.empty()) { + items.insert(items.end(), appended.begin(), appended.end()); } - - std::vector violations; - - // 对每只鞋进行颜色检查 + } + + void ProcessPpeBootsCheck(const FramePtr& frame) { + auto& detections = frame->det->items; + std::vector boots; + boots.reserve(detections.size()); + for (const auto& det : detections) { + if (det.cls_id == config_.boots_class) boots.push_back(det); + } + for (const auto& boot : boots) { - if (config_.enable_color_check && color_analyzer_) { - // 将检测坐标映射到当前帧坐标 - Rect mapped_bbox = MapDetCoordToFrame(boot.bbox, frame); - - if (config_.debug) { - LogInfo("[LogicGateNode] Boot bbox: [" + std::to_string(static_cast(boot.bbox.x)) + - "," + std::to_string(static_cast(boot.bbox.y)) + - " " + std::to_string(static_cast(boot.bbox.w)) + - "x" + std::to_string(static_cast(boot.bbox.h)) + - "] -> Mapped: [" + std::to_string(static_cast(mapped_bbox.x)) + - "," + std::to_string(static_cast(mapped_bbox.y)) + - " " + std::to_string(static_cast(mapped_bbox.w)) + - "x" + std::to_string(static_cast(mapped_bbox.h)) + "]"); - } - - auto color_result = color_analyzer_->Analyze(*frame, mapped_bbox); - - if (config_.debug) { - LogInfo("[LogicGateNode] Boot brightness=" + - std::to_string(color_result.brightness) + - " is_dark=" + (color_result.is_dark ? "true" : "false")); - } - - if (!color_result.is_dark) { - // 颜色不对，添加 no_boots 检测框 - Detection no_boots_det; - no_boots_det.cls_id = 10; // no_boots - no_boots_det.track_id = std::max(0, boot.track_id); - no_boots_det.score = std::max(0.5f, color_result.confidence); - no_boots_det.bbox = boot.bbox; - - frame->det->items.push_back(no_boots_det); - - if (config_.debug) { - LogInfo("[LogicGateNode] VIOLATION: Non-compliant boots color (brightness=" + - std::to_string(static_cast(color_result.brightness)) + - ") added no_boots(cls=10) track_id=" + std::to_string(no_boots_det.track_id)); - } - } + if (!config_.enable_color_check || !color_analyzer_) continue; + const Rect mapped_bbox = MapDetCoordToFrame(boot.bbox, frame); + const auto color_result = color_analyzer_->Analyze(*frame, mapped_bbox); + if (color_result.is_dark) continue; + + Detection no_boots_det; + no_boots_det.cls_id = 10; + no_boots_det.track_id = std::max(0, boot.track_id); + no_boots_det.score = std::max(0.5f, color_result.confidence); + no_boots_det.bbox = boot.bbox; + detections.push_back(no_boots_det); + + if (config_.debug) { + LogInfo("[logic_gate] color violation track_id=" + std::to_string(no_boots_det.track_id)); } } } + + std::string id_; + LogicGateConfig config_; + std::unique_ptr color_analyzer_; + std::vector>> output_queues_; }; REGISTER_NODE(LogicGateNode, "logic_gate"); -} // namespace rk3588 +} // namespace rk3588