Doni/OrangePi3588Media
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add person shoe violation logic and alarm pipeline

Browse Source
This commit is contained in:
tian 2026-03-15 03:19:58 +08:00
parent e2b9a3cd42
commit 45842a4a52
3 changed files with 455 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

228
configs/person_shoe_two_stage_recommended_alarm.json Normal file
View File

@ -0,0 +1,228 @@
{
"queue": {
"size": 8,
"strategy": "drop_oldest"
},
"graphs": [
{
"name": "person_shoe_two_stage_recommended_alarm",
"executor": {
"batch_size": 2,
"run_budget": 8
},
"nodes": [
{
"id": "in",
"type": "input_rtsp",
"role": "source",
"enable": true,
"url": "rtsp://10.0.0.49:8554/cam",
"fps": 30,
"width": 1920,
"height": 1080,
"use_ffmpeg": true,
"use_mpp": false,
"force_tcp": true
},
{
"id": "pre_rgb",
"type": "preprocess",
"role": "filter",
"enable": true,
"dst_w": 1920,
"dst_h": 1080,
"dst_format": "rgb",
"dst_packed": true,
"resize_mode": "stretch",
"rga_gate": "person_shoe_two_stage_recommended_alarm",
"use_rga": true
},
{
"id": "person_det",
"type": "ai_yolo",
"role": "filter",
"enable": true,
"cpu_affinity": [4],
"use_rga": true,
"rga_gate": "person_shoe_two_stage_recommended_alarm",
"rga_max_inflight": 4,
"dst_packed": true,
"use_dma_input": true,
"infer_fps": 2,
"infer_phase_ms": 0,
"model_path": "./models/yolov8n-640.rknn",
"model_version": "v8",
"model_w": 640,
"model_h": 640,
"num_classes": 80,
"conf": 0.35,
"nms": 0.45,
"class_filter": [0],
"bbox_expand": {
"enable": true,
"class_id": 0,
"left": 0.06,
"right": 0.06,
"top": 0.04,
"bottom": 0.16
}
},
{
"id": "person_trk",
"type": "tracker",
"role": "filter",
"enable": true,
"cpu_affinity": [5],
"mode": "bytetrack_lite",
"per_class": true,
"track_classes": [0],
"high_th": 0.55,
"low_th": 0.10,
"iou_th": 0.3,
"max_age_ms": 900,
"min_hits": 2
},
{
"id": "shoe_det",
"type": "ai_shoe_det",
"role": "filter",
"enable": true,
"cpu_affinity": [6],
"use_rga": true,
"rga_gate": "person_shoe_two_stage_recommended_alarm",
"rga_max_inflight": 4,
"dst_packed": true,
"use_dma_input": false,
"infer_fps": 2,
"infer_phase_ms": 150,
"model_path": "./models/shoe_detector_openimages_ppe_v1.rknn",
"model_w": 640,
"model_h": 640,
"conf": 0.15,
"nms": 0.45,
"v8_box_format": "cxcywh",
"append_detections": true,
"dynamic_roi": {
"enable": true,
"person_class_id": 0,
"shoe_class_id": 1,
"debug_roi_class_id": -1,
"max_rois": 3,
"min_person_height": 60,
"x_offset": -0.24,
"y_offset": 0.64,
"width_scale": 1.48,
"height_scale": 0.58
}
},
{
"id": "shoe_logic",
"type": "logic_gate",
"role": "filter",
"enable": true,
"cpu_affinity": [6],
"mode": "person_shoe_check",
"debug": false,
"person_shoe_check": {
"person_class": 0,
"shoe_class": 1,
"violation_class": 2,
"min_person_score": 0.30,
"min_shoe_score": 0.15,
"require_person_track_id": true,
"use_person_bbox_for_violation": true,
"attach_person_track_to_shoe": true,
"match_iou": 0.02,
"foot_region": {
"x_offset": -0.24,
"y_offset": 0.64,
"width_scale": 1.48,
"height_scale": 0.58
}
}
},
{
"id": "osd",
"type": "osd",
"role": "filter",
"enable": true,
"cpu_affinity": [7],
"draw_bbox": true,
"draw_text": false,
"use_rga_bbox": false,
"labels": ["person", "shoe", "no_shoe"]
},
{
"id": "post",
"type": "preprocess",
"role": "filter",
"enable": true,
"cpu_affinity": [7],
"dst_w": 1920,
"dst_h": 1080,
"dst_format": "nv12",
"resize_mode": "stretch",
"rga_gate": "person_shoe_two_stage_recommended_alarm",
"use_rga": true
},
{
"id": "pub",
"type": "publish",
"role": "filter",
"enable": true,
"cpu_affinity": [3],
"codec": "h264",
"fps": 30,
"bitrate_kbps": 2000,
"mpp_output_timeout_ms": 50,
"mpp_packet_wait_ms": 10,
"use_mpp": true,
"outputs": [
{"proto": "rtsp_server", "port": 8555, "path": "/live/cam1"}
]
},
{
"id": "alarm",
"type": "alarm",
"role": "sink",
"enable": true,
"eval_fps": 2,
"labels": ["person", "shoe", "no_shoe"],
"rules": [
{
"name": "missing_shoe",
"class_ids": [2],
"roi": {"x": 0.0, "y": 0.0, "w": 1.0, "h": 1.0},
"min_score": 0.30,
"require_track_id": true,
"min_duration_ms": 800,
"min_hits": 2,
"hit_window_ms": 2000,
"cooldown_ms": 10000,
"per_track_cooldown_ms": 15000
}
],
"actions": {
"log": {
"enable": true,
"level": "info",
"include_detections": true,
"min_interval_ms": 2000
}
}
}
],
"edges": [
["in", "pre_rgb"],
["pre_rgb", "person_det"],
["person_det", "person_trk"],
["person_trk", "shoe_det", {"queue": {"size": 16, "strategy": "drop_oldest"}}],
["shoe_det", "shoe_logic", {"queue": {"size": 16, "strategy": "drop_oldest"}}],
["shoe_logic", "osd", {"queue": {"size": 16, "strategy": "drop_oldest"}}],
["osd", "post", {"queue": {"size": 32, "strategy": "drop_oldest"}}],
["post", "pub", {"queue": {"size": 64, "strategy": "drop_oldest"}}],
["pub", "alarm", {"queue": {"size": 64, "strategy": "drop_oldest"}}]
]
}
]
}

6
docs/deployment.md
View File

@ -159,6 +159,12 @@ sudo ./scripts/ops.sh ddr-restore
./build/media-server -c configs/person_shoe_two_stage_recommended.json
```
如需直接启用“缺鞋”判定和按人冷却告警，可使用：
```bash
./build/media-server -c configs/person_shoe_two_stage_recommended_alarm.json
```
说明：
- 输入端：`use_ffmpeg: true`、`use_mpp: false`

371
plugins/logic_gate/logic_gate_node.cpp
View File

@ -1,234 +1,305 @@
#include <map>
#include <vector>
#include <memory>
#include <chrono>
#include <algorithm>
#include <cstring>
#include <cmath>
#include <memory>
#include <string>
#include <vector>
#include "node.h"
#include "color_analyzer.h"
#include "frame/frame.h"
#include "node.h"
#include "utils/logger.h"
#include "utils/simple_json.h"
#include "spatial_matcher.h"
#include "color_analyzer.h"
namespace rk3588 {
// 违规类型
enum class ViolationType {
NONE,
MISSING_BOOTS, // 没有鞋
WRONG_COLOR_BOOTS, // 鞋颜色不对
struct PersonShoeCheckConfig {
int person_class = 0;
int shoe_class = 1;
int violation_class = 2;
float min_person_score = 0.0f;
float min_shoe_score = 0.0f;
bool require_person_track_id = false;
bool use_person_bbox_for_violation = true;
bool attach_person_track_to_shoe = true;
float match_iou = 0.02f;
float x_offset = -0.20f;
float y_offset = 0.64f;
float width_scale = 1.48f;
float height_scale = 0.58f;
};
// 违规信息
struct ViolationInfo {
ViolationType type = ViolationType::NONE;
std::string description;
int track_id = -1;
float confidence = 0.0f;
Rect violation_region;
};
// Logic Gate 配置
struct LogicGateConfig {
std::string mode = "ppe_boots_check";
int anchor_class = 6; // Person
int boots_class = 3; // boots
ColorConfig color;
int anchor_class = 6;
int boots_class = 3;
bool enable_color_check = true;
std::string violation_key = "ppe_violation";
bool pass_through = true;
bool debug = false;
std::string violation_key = "ppe_violation";
ColorConfig color;
PersonShoeCheckConfig person_shoe;
};
class LogicGateNode : public INode {
public:
LogicGateNode() = default;
~LogicGateNode() override = default;
std::string Id() const override { return id_; }
std::string Type() const override { return "logic_gate"; }
bool Init(const SimpleJson& config, const NodeContext& ctx) override {
id_ = config.ValueOr<std::string>("id", "logic_gate");
config_ = ParseConfig(config);
output_queues_ = ctx.output_queues;
// 初始化颜色分析器
if (config_.enable_color_check) {
color_analyzer_ = std::make_unique<ColorAnalyzer>(config_.color);
}
LogInfo("[LogicGateNode] Initialized, id=" + id_ + " mode=" + config_.mode +
" output_queues=" + std::to_string(output_queues_.size()));
LogInfo("[logic_gate] initialized id=" + id_ + " mode=" + config_.mode);
return true;
}
bool Start() override {
LogInfo("[LogicGateNode] started id=" + id_);
LogInfo("[logic_gate] started id=" + id_);
return true;
}
void Stop() override {
}
void Stop() override {}
NodeStatus Process(FramePtr frame) override {
if (!frame || !frame->det) {
PushToDownstream(frame);
return NodeStatus::OK;
}
if (config_.mode == "ppe_boots_check") {
if (config_.mode == "person_shoe_check") {
ProcessPersonShoeCheck(frame);
} else if (config_.mode == "ppe_boots_check") {
ProcessPpeBootsCheck(frame);
}
PushToDownstream(frame);
return NodeStatus::OK;
}
private:
std::string id_;
LogicGateConfig config_;
std::unique_ptr<ColorAnalyzer> color_analyzer_;
std::vector<std::shared_ptr<SpscQueue<FramePtr>>> output_queues_;
static Rect ClipRect(const Rect& r, int img_w, int img_h) {
Rect out = r;
out.x = std::max(0.0f, out.x);
out.y = std::max(0.0f, out.y);
if (img_w > 0) out.w = std::min(out.w, static_cast<float>(img_w) - out.x);
if (img_h > 0) out.h = std::min(out.h, static_cast<float>(img_h) - out.y);
out.w = std::max(0.0f, out.w);
out.h = std::max(0.0f, out.h);
return out;
}
static float IoU(const Rect& a, const Rect& b) {
const float x1 = std::max(a.x, b.x);
const float y1 = std::max(a.y, b.y);
const float x2 = std::min(a.x + a.w, b.x + b.w);
const float y2 = std::min(a.y + a.h, b.y + b.h);
if (x2 <= x1 || y2 <= y1) return 0.0f;
const float inter = (x2 - x1) * (y2 - y1);
const float area_a = a.w * a.h;
const float area_b = b.w * b.h;
const float uni = area_a + area_b - inter;
return uni > 0.0f ? (inter / uni) : 0.0f;
}
static bool CenterInside(const Rect& outer, const Rect& inner) {
const float cx = inner.x + inner.w * 0.5f;
const float cy = inner.y + inner.h * 0.5f;
return cx >= outer.x && cx <= (outer.x + outer.w) &&
cy >= outer.y && cy <= (outer.y + outer.h);
}
LogicGateConfig ParseConfig(const SimpleJson& cfg) {
LogicGateConfig config;
config.mode = cfg.ValueOr<std::string>("mode", "ppe_boots_check");
config.anchor_class = cfg.ValueOr<int>("anchor_class", 6);
config.boots_class = cfg.ValueOr<int>("boots_class", 3);
config.violation_key = cfg.ValueOr<std::string>("violation_key", "ppe_violation");
config.pass_through = cfg.ValueOr<bool>("pass_through", true);
config.debug = cfg.ValueOr<bool>("debug", false);
// 解析颜色配置
config.mode = cfg.ValueOr<std::string>("mode", config.mode);
config.anchor_class = cfg.ValueOr<int>("anchor_class", config.anchor_class);
config.boots_class = cfg.ValueOr<int>("boots_class", config.boots_class);
config.violation_key = cfg.ValueOr<std::string>("violation_key", config.violation_key);
config.pass_through = cfg.ValueOr<bool>("pass_through", config.pass_through);
config.debug = cfg.ValueOr<bool>("debug", config.debug);
if (const SimpleJson* color = cfg.Find("color_check")) {
config.enable_color_check = color->ValueOr<bool>("enable", true);
std::string method = color->ValueOr<std::string>("method", "hsv");
const std::string method = color->ValueOr<std::string>("method", "hsv");
if (method == "hsv") config.color.method = ColorMethod::HSV;
else if (method == "rgb") config.color.method = ColorMethod::RGB;
else if (method == "brightness") config.color.method = ColorMethod::BRIGHTNESS;
config.color.dark_threshold = color->ValueOr<int>("dark_threshold", 80);
config.color.roi_expand = color->ValueOr<float>("roi_expand", 1.0f);
config.color.debug_output = config.debug;
}
if (const SimpleJson* ps = cfg.Find("person_shoe_check"); ps && ps->IsObject()) {
config.person_shoe.person_class = ps->ValueOr<int>("person_class", config.person_shoe.person_class);
config.person_shoe.shoe_class = ps->ValueOr<int>("shoe_class", config.person_shoe.shoe_class);
config.person_shoe.violation_class = ps->ValueOr<int>("violation_class", config.person_shoe.violation_class);
config.person_shoe.min_person_score = ps->ValueOr<float>("min_person_score", config.person_shoe.min_person_score);
config.person_shoe.min_shoe_score = ps->ValueOr<float>("min_shoe_score", config.person_shoe.min_shoe_score);
config.person_shoe.require_person_track_id = ps->ValueOr<bool>("require_person_track_id", config.person_shoe.require_person_track_id);
config.person_shoe.use_person_bbox_for_violation = ps->ValueOr<bool>(
"use_person_bbox_for_violation", config.person_shoe.use_person_bbox_for_violation);
config.person_shoe.attach_person_track_to_shoe = ps->ValueOr<bool>(
"attach_person_track_to_shoe", config.person_shoe.attach_person_track_to_shoe);
config.person_shoe.match_iou = ps->ValueOr<float>("match_iou", config.person_shoe.match_iou);
if (const SimpleJson* fr = ps->Find("foot_region"); fr && fr->IsObject()) {
config.person_shoe.x_offset = fr->ValueOr<float>("x_offset", config.person_shoe.x_offset);
config.person_shoe.y_offset = fr->ValueOr<float>("y_offset", config.person_shoe.y_offset);
config.person_shoe.width_scale = fr->ValueOr<float>("width_scale", config.person_shoe.width_scale);
config.person_shoe.height_scale = fr->ValueOr<float>("height_scale", config.person_shoe.height_scale);
}
}
return config;
}
void PushToDownstream(FramePtr frame) {
void PushToDownstream(const FramePtr& frame) {
for (auto& q : output_queues_) {
if (q) q->Push(frame);
}
}
// 将检测坐标（相对于原始图像）映射到当前帧坐标
Rect MapDetCoordToFrame(const Rect& det_bbox, FramePtr frame) {
if (!frame->transform_meta || !frame->transform_meta->valid) {
return det_bbox; // 无变换信息，直接使用
}
Rect MapDetCoordToFrame(const Rect& det_bbox, const FramePtr& frame) const {
if (!frame->transform_meta || !frame->transform_meta->valid) return det_bbox;
const auto& meta = *frame->transform_meta;
if (meta.src_w <= 0 || meta.src_h <= 0 || frame->width <= 0 || frame->height <= 0) {
return det_bbox;
}
// 计算缩放因子：检测坐标是基于 src_w x src_h 的
float scale_x = static_cast<float>(frame->width) / meta.src_w;
float scale_y = static_cast<float>(frame->height) / meta.src_h;
if (meta.src_w <= 0 || meta.src_h <= 0 || frame->width <= 0 || frame->height <= 0) return det_bbox;
const float scale_x = static_cast<float>(frame->width) / static_cast<float>(meta.src_w);
const float scale_y = static_cast<float>(frame->height) / static_cast<float>(meta.src_h);
Rect mapped;
mapped.x = det_bbox.x * scale_x;
mapped.y = det_bbox.y * scale_y;
mapped.w = det_bbox.w * scale_x;
mapped.h = det_bbox.h * scale_y;
return mapped;
}
void ProcessPpeBootsCheck(FramePtr frame) {
const auto& detections = frame->det->items;
// 收集所有人和鞋
std::vector<Detection> persons;
std::vector<Detection> boots;
for (const auto& det : detections) {
if (det.cls_id == config_.anchor_class) {
persons.push_back(det);
} else if (det.cls_id == config_.boots_class) {
boots.push_back(det);
Rect BuildFootRegion(const Rect& person_bbox, int img_w, int img_h) const {
Rect roi;
roi.x = person_bbox.x + person_bbox.w * config_.person_shoe.x_offset;
roi.y = person_bbox.y + person_bbox.h * config_.person_shoe.y_offset;
roi.w = person_bbox.w * config_.person_shoe.width_scale;
roi.h = person_bbox.h * config_.person_shoe.height_scale;
return ClipRect(roi, img_w, img_h);
}
void ProcessPersonShoeCheck(const FramePtr& frame) {
auto& items = frame->det->items;
const int img_w = frame->det->img_w > 0 ? frame->det->img_w : frame->width;
const int img_h = frame->det->img_h > 0 ? frame->det->img_h : frame->height;
std::vector<size_t> person_indices;
std::vector<size_t> shoe_indices;
person_indices.reserve(items.size());
shoe_indices.reserve(items.size());
for (size_t i = 0; i < items.size(); ++i) {
if (items[i].cls_id == config_.person_shoe.person_class) {
person_indices.push_back(i);
} else if (items[i].cls_id == config_.person_shoe.shoe_class) {
shoe_indices.push_back(i);
}
}
if (config_.debug) {
LogInfo("[LogicGateNode] Persons=" + std::to_string(persons.size()) +
" Boots=" + std::to_string(boots.size()) +
" Frame=" + std::to_string(frame->width) + "x" + std::to_string(frame->height));
if (frame->transform_meta && frame->transform_meta->valid) {
LogInfo("[LogicGateNode] TransformMeta: src=" + std::to_string(frame->transform_meta->src_w) +
"x" + std::to_string(frame->transform_meta->src_h));
LogInfo("[logic_gate] person_shoe_check persons=" + std::to_string(person_indices.size()) +
" shoes=" + std::to_string(shoe_indices.size()));
}
std::vector<bool> shoe_used(shoe_indices.size(), false);
std::vector<Detection> appended;
for (size_t pi = 0; pi < person_indices.size(); ++pi) {
Detection& person = items[person_indices[pi]];
if (person.score < config_.person_shoe.min_person_score) continue;
if (config_.person_shoe.require_person_track_id && person.track_id < 0) continue;
const Rect foot_region = BuildFootRegion(person.bbox, img_w, img_h);
int best_shoe_local = -1;
float best_match_score = 0.0f;
for (size_t si = 0; si < shoe_indices.size(); ++si) {
if (shoe_used[si]) continue;
Detection& shoe = items[shoe_indices[si]];
if (shoe.score < config_.person_shoe.min_shoe_score) continue;
const float iou = IoU(foot_region, shoe.bbox);
const bool center_inside = CenterInside(foot_region, shoe.bbox);
if (!center_inside && iou < config_.person_shoe.match_iou) continue;
const float match_score = (center_inside ? 1.0f : 0.0f) + iou + shoe.score * 0.1f;
if (best_shoe_local < 0 || match_score > best_match_score) {
best_shoe_local = static_cast<int>(si);
best_match_score = match_score;
}
}
if (best_shoe_local >= 0) {
shoe_used[static_cast<size_t>(best_shoe_local)] = true;
Detection& matched_shoe = items[shoe_indices[static_cast<size_t>(best_shoe_local)]];
if (config_.person_shoe.attach_person_track_to_shoe &&
matched_shoe.track_id < 0 && person.track_id >= 0) {
matched_shoe.track_id = person.track_id;
}
continue;
}
Detection no_shoe;
no_shoe.cls_id = config_.person_shoe.violation_class;
no_shoe.track_id = person.track_id;
no_shoe.score = std::max(0.5f, person.score);
no_shoe.bbox = config_.person_shoe.use_person_bbox_for_violation ? person.bbox : foot_region;
appended.push_back(no_shoe);
if (config_.debug) {
LogInfo("[logic_gate] no_shoe track_id=" + std::to_string(no_shoe.track_id) +
" bbox=(" + std::to_string(static_cast<int>(no_shoe.bbox.x)) + "," +
std::to_string(static_cast<int>(no_shoe.bbox.y)) + "," +
std::to_string(static_cast<int>(no_shoe.bbox.w)) + "," +
std::to_string(static_cast<int>(no_shoe.bbox.h)) + ")");
}
}
// 简化逻辑：必须同时检测到人和鞋，才开始判断
if (persons.empty() || boots.empty()) {
return;
if (!appended.empty()) {
items.insert(items.end(), appended.begin(), appended.end());
}
std::vector<ViolationInfo> violations;
// 对每只鞋进行颜色检查
}
void ProcessPpeBootsCheck(const FramePtr& frame) {
auto& detections = frame->det->items;
std::vector<Detection> boots;
boots.reserve(detections.size());
for (const auto& det : detections) {
if (det.cls_id == config_.boots_class) boots.push_back(det);
}
for (const auto& boot : boots) {
if (config_.enable_color_check && color_analyzer_) {
// 将检测坐标映射到当前帧坐标
Rect mapped_bbox = MapDetCoordToFrame(boot.bbox, frame);
if (config_.debug) {
LogInfo("[LogicGateNode] Boot bbox: [" + std::to_string(static_cast<int>(boot.bbox.x)) +
"," + std::to_string(static_cast<int>(boot.bbox.y)) +
" " + std::to_string(static_cast<int>(boot.bbox.w)) +
"x" + std::to_string(static_cast<int>(boot.bbox.h)) +
"] -> Mapped: [" + std::to_string(static_cast<int>(mapped_bbox.x)) +
"," + std::to_string(static_cast<int>(mapped_bbox.y)) +
" " + std::to_string(static_cast<int>(mapped_bbox.w)) +
"x" + std::to_string(static_cast<int>(mapped_bbox.h)) + "]");
}
auto color_result = color_analyzer_->Analyze(*frame, mapped_bbox);
if (config_.debug) {
LogInfo("[LogicGateNode] Boot brightness=" +
std::to_string(color_result.brightness) +
" is_dark=" + (color_result.is_dark ? "true" : "false"));
}
if (!color_result.is_dark) {
// 颜色不对，添加 no_boots 检测框
Detection no_boots_det;
no_boots_det.cls_id = 10; // no_boots
no_boots_det.track_id = std::max(0, boot.track_id);
no_boots_det.score = std::max(0.5f, color_result.confidence);
no_boots_det.bbox = boot.bbox;
frame->det->items.push_back(no_boots_det);
if (config_.debug) {
LogInfo("[LogicGateNode] VIOLATION: Non-compliant boots color (brightness=" +
std::to_string(static_cast<int>(color_result.brightness)) +
") added no_boots(cls=10) track_id=" + std::to_string(no_boots_det.track_id));
}
}
if (!config_.enable_color_check || !color_analyzer_) continue;
const Rect mapped_bbox = MapDetCoordToFrame(boot.bbox, frame);
const auto color_result = color_analyzer_->Analyze(*frame, mapped_bbox);
if (color_result.is_dark) continue;
Detection no_boots_det;
no_boots_det.cls_id = 10;
no_boots_det.track_id = std::max(0, boot.track_id);
no_boots_det.score = std::max(0.5f, color_result.confidence);
no_boots_det.bbox = boot.bbox;
detections.push_back(no_boots_det);
if (config_.debug) {
LogInfo("[logic_gate] color violation track_id=" + std::to_string(no_boots_det.track_id));
}
}
}
std::string id_;
LogicGateConfig config_;
std::unique_ptr<ColorAnalyzer> color_analyzer_;
std::vector<std::shared_ptr<SpscQueue<FramePtr>>> output_queues_;
};
REGISTER_NODE(LogicGateNode, "logic_gate");
} // namespace rk3588
} // namespace rk3588