From f275fc0e927aadb1d69df74b9337ce4577bdd452 Mon Sep 17 00:00:00 2001 From: sladro Date: Wed, 1 Apr 2026 10:17:01 +0800 Subject: [PATCH] feat: add temporal action recognition plugin --- plugins/CMakeLists.txt | 14 +- plugins/action_recog/action_recog_node.cpp | 254 +++++++++++++++++++++ plugins/action_recog/action_recog_node.h | 33 +++ plugins/region_event/region_event_node.cpp | 2 + tests/CMakeLists.txt | 4 + tests/test_action_recog.cpp | 128 +++++++++++ 6 files changed, 434 insertions(+), 1 deletion(-) create mode 100644 plugins/action_recog/action_recog_node.cpp create mode 100644 plugins/action_recog/action_recog_node.h create mode 100644 tests/test_action_recog.cpp diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index b677cb8..de183c8 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -470,6 +470,18 @@ set_target_properties(region_event PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${RK_PLUGIN_OUTPUT_DIR} ) +# action_recog plugin (temporal rule-based behavior events) +add_library(action_recog SHARED + action_recog/action_recog_node.cpp +) +target_include_directories(action_recog PRIVATE ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/third_party) +target_link_libraries(action_recog PRIVATE project_options Threads::Threads) +set_target_properties(action_recog PROPERTIES + OUTPUT_NAME "action_recog" + LIBRARY_OUTPUT_DIRECTORY ${RK_PLUGIN_OUTPUT_DIR} + RUNTIME_OUTPUT_DIRECTORY ${RK_PLUGIN_OUTPUT_DIR} +) + # storage plugin (continuous recording with segment management) add_library(storage SHARED storage/storage_node.cpp @@ -541,7 +553,7 @@ set_target_properties(ai_shoe_det PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${RK_PLUGIN_OUTPUT_DIR} ) -install(TARGETS input_rtsp input_file publish preprocess ai_yolo ai_face_det ai_scrfd ai_scrfd_sliding ai_face_recog tracker gate osd alarm logic_gate region_event storage ai_scheduler ai_shoe_det +install(TARGETS input_rtsp input_file publish preprocess ai_yolo ai_face_det ai_scrfd ai_scrfd_sliding ai_face_recog tracker gate osd alarm logic_gate region_event action_recog storage ai_scheduler ai_shoe_det LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}/rk3588-media-server/plugins RUNTIME DESTINATION ${CMAKE_INSTALL_LIBDIR}/rk3588-media-server/plugins ) diff --git a/plugins/action_recog/action_recog_node.cpp b/plugins/action_recog/action_recog_node.cpp new file mode 100644 index 0000000..3ab69df --- /dev/null +++ b/plugins/action_recog/action_recog_node.cpp @@ -0,0 +1,254 @@ +#include "action_recog_node.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "behavior/behavior_event.h" +#include "utils/logger.h" + +namespace rk3588 { +namespace { + +enum class ActionEventKind { + Fall, + Fight +}; + +struct ActionEventRule { + ActionEventKind kind = ActionEventKind::Fall; + uint64_t window_ms = 1000; + uint64_t activate_duration_ms = 0; + float min_drop_pixels = 0.0f; + float min_aspect_ratio_delta = 0.0f; + float proximity_pixels = 0.0f; + float min_motion_pixels = 0.0f; +}; + +struct TrackSample { + uint64_t pts = 0; + Rect bbox{}; +}; + +static bool ParseRules(const SimpleJson& config, std::vector& out, std::string& err) { + const SimpleJson* events = config.Find("events"); + if (!events || !events->IsArray()) { + err = "events must be array"; + return false; + } + + out.clear(); + for (const auto& ev : events->AsArray()) { + if (!ev.IsObject()) { + err = "event entry must be object"; + return false; + } + + ActionEventRule rule; + const std::string type = ev.ValueOr("type", ""); + rule.window_ms = static_cast(std::max(0, ev.ValueOr("window_ms", 1000))); + rule.activate_duration_ms = static_cast(std::max(0, ev.ValueOr("activate_duration_ms", 0))); + + if (type == "fall") { + rule.kind = ActionEventKind::Fall; + rule.min_drop_pixels = ev.ValueOr("min_drop_pixels", 0.0f); + rule.min_aspect_ratio_delta = ev.ValueOr("min_aspect_ratio_delta", 0.0f); + } else if (type == "fight") { + rule.kind = ActionEventKind::Fight; + rule.proximity_pixels = ev.ValueOr("proximity_pixels", 0.0f); + rule.min_motion_pixels = ev.ValueOr("min_motion_pixels", 0.0f); + } else { + err = "unsupported event type: " + type; + return false; + } + + out.push_back(std::move(rule)); + } + return true; +} + +static float CenterX(const Rect& rect) { + return rect.x + (rect.w * 0.5f); +} + +static float CenterY(const Rect& rect) { + return rect.y + (rect.h * 0.5f); +} + +static float CenterDistance(const Rect& lhs, const Rect& rhs) { + const float dx = CenterX(lhs) - CenterX(rhs); + const float dy = CenterY(lhs) - CenterY(rhs); + return std::sqrt((dx * dx) + (dy * dy)); +} + +static float MotionDistance(const Rect& lhs, const Rect& rhs) { + const float dx = CenterX(lhs) - CenterX(rhs); + const float dy = CenterY(lhs) - CenterY(rhs); + return std::sqrt((dx * dx) + (dy * dy)); +} + +static float AspectRatio(const Rect& rect) { + return rect.h > 0.0f ? (rect.w / rect.h) : 0.0f; +} + +static BehaviorEventType ToBehaviorEventType(ActionEventKind kind) { + return kind == ActionEventKind::Fall ? BehaviorEventType::Fall : BehaviorEventType::Fight; +} + +} // namespace + +struct ActionRecogNode::Impl { + std::string init_err; + std::vector rules; + std::map> history; +}; + +ActionRecogNode::ActionRecogNode() : impl_(std::make_unique()) {} + +ActionRecogNode::~ActionRecogNode() = default; + +std::string ActionRecogNode::Id() const { + return id_; +} + +std::string ActionRecogNode::Type() const { + return "action_recog"; +} + +bool ActionRecogNode::Init(const SimpleJson& config, const NodeContext& ctx) { + id_ = config.ValueOr("id", "action_recog"); + if (!ParseRules(config, impl_->rules, impl_->init_err)) { + LogError("[action_recog] invalid config: " + impl_->init_err); + return false; + } + output_queues_ = ctx.output_queues; + return true; +} + +bool ActionRecogNode::Start() { + return true; +} + +void ActionRecogNode::Stop() {} + +NodeStatus ActionRecogNode::Process(FramePtr frame) { + if (!frame) return NodeStatus::DROP; + + EnsureBehaviorEvents(*frame); + if (!frame->det) { + PushToDownstream(frame); + return NodeStatus::OK; + } + + std::map current_tracks; + for (const auto& det : frame->det->items) { + if (det.track_id < 0) continue; + current_tracks[det.track_id] = det.bbox; + auto& history = impl_->history[det.track_id]; + history.push_back(TrackSample{frame->pts, det.bbox}); + while (!history.empty() && frame->pts > history.front().pts && + (frame->pts - history.front().pts) > impl_->rules.front().window_ms) { + history.pop_front(); + } + } + + for (const auto& rule : impl_->rules) { + if (rule.kind == ActionEventKind::Fall) { + for (const auto& [track_id, bbox] : current_tracks) { + auto it = impl_->history.find(track_id); + if (it == impl_->history.end() || it->second.empty()) continue; + const auto& first = it->second.front(); + const float drop = CenterY(bbox) - CenterY(first.bbox); + const float aspect_delta = AspectRatio(bbox) - AspectRatio(first.bbox); + const uint64_t duration = frame->pts >= first.pts ? (frame->pts - first.pts) : 0; + if (drop < rule.min_drop_pixels) continue; + if (aspect_delta < rule.min_aspect_ratio_delta) continue; + if (duration < rule.activate_duration_ms) continue; + + BehaviorEventItem event; + event.type = ToBehaviorEventType(rule.kind); + event.status = BehaviorEventStatus::Active; + event.score = 1.0f; + event.bbox = bbox; + event.track_ids.push_back(track_id); + event.start_pts = first.pts; + event.last_pts = frame->pts; + event.duration_ms = duration; + event.source = "action_recog"; + frame->behavior_events->items.push_back(std::move(event)); + } + } else { + std::set> emitted_pairs; + for (auto left = current_tracks.begin(); left != current_tracks.end(); ++left) { + for (auto right = std::next(left); right != current_tracks.end(); ++right) { + const float proximity = CenterDistance(left->second, right->second); + if (proximity > rule.proximity_pixels) continue; + + const auto it_left = impl_->history.find(left->first); + const auto it_right = impl_->history.find(right->first); + if (it_left == impl_->history.end() || it_right == impl_->history.end()) continue; + if (it_left->second.empty() || it_right->second.empty()) continue; + + const float left_motion = MotionDistance(left->second, it_left->second.front().bbox); + const float right_motion = MotionDistance(right->second, it_right->second.front().bbox); + const uint64_t duration_left = frame->pts >= it_left->second.front().pts ? (frame->pts - it_left->second.front().pts) : 0; + const uint64_t duration_right = frame->pts >= it_right->second.front().pts ? (frame->pts - it_right->second.front().pts) : 0; + const uint64_t duration = std::min(duration_left, duration_right); + + const float combined_motion = left_motion + right_motion; + if (combined_motion < rule.min_motion_pixels) continue; + if (duration < rule.activate_duration_ms) continue; + + const auto pair_key = std::make_pair(left->first, right->first); + if (!emitted_pairs.insert(pair_key).second) continue; + + BehaviorEventItem event; + event.type = ToBehaviorEventType(rule.kind); + event.status = BehaviorEventStatus::Active; + event.score = 1.0f; + event.bbox = Rect{ + std::min(left->second.x, right->second.x), + std::min(left->second.y, right->second.y), + std::max(left->second.x + left->second.w, right->second.x + right->second.w) - std::min(left->second.x, right->second.x), + std::max(left->second.y + left->second.h, right->second.y + right->second.h) - std::min(left->second.y, right->second.y) + }; + event.track_ids.push_back(left->first); + event.track_ids.push_back(right->first); + event.start_pts = frame->pts - duration; + event.last_pts = frame->pts; + event.duration_ms = duration; + event.source = "action_recog"; + frame->behavior_events->items.push_back(std::move(event)); + } + } + } + } + + PushToDownstream(frame); + return NodeStatus::OK; +} + +void ActionRecogNode::EnsureBehaviorEvents(Frame& frame) { + if (!frame.behavior_events) { + frame.behavior_events = std::make_shared(); + } +} + +void ActionRecogNode::PushToDownstream(const FramePtr& frame) { + for (auto& q : output_queues_) { + if (q) q->Push(frame); + } +} + +#ifndef RK3588_TEST_BUILD +REGISTER_NODE(ActionRecogNode, "action_recog"); +#endif + +} // namespace rk3588 diff --git a/plugins/action_recog/action_recog_node.h b/plugins/action_recog/action_recog_node.h new file mode 100644 index 0000000..95f32fe --- /dev/null +++ b/plugins/action_recog/action_recog_node.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include + +#include "node.h" + +namespace rk3588 { + +class ActionRecogNode final : public INode { +public: + ActionRecogNode(); + ~ActionRecogNode() override; + + std::string Id() const override; + std::string Type() const override; + bool Init(const SimpleJson& config, const NodeContext& ctx) override; + bool Start() override; + void Stop() override; + NodeStatus Process(FramePtr frame) override; + +private: + void EnsureBehaviorEvents(Frame& frame); + void PushToDownstream(const FramePtr& frame); + + struct Impl; + std::unique_ptr impl_; + std::string id_; + std::vector>> output_queues_; +}; + +} // namespace rk3588 diff --git a/plugins/region_event/region_event_node.cpp b/plugins/region_event/region_event_node.cpp index dc3fb39..7e23770 100644 --- a/plugins/region_event/region_event_node.cpp +++ b/plugins/region_event/region_event_node.cpp @@ -227,6 +227,8 @@ void RegionEventNode::PushToDownstream(const FramePtr& frame) { } } +#ifndef RK3588_TEST_BUILD REGISTER_NODE(RegionEventNode, "region_event"); +#endif } // namespace rk3588 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 772f1e6..b20b6da 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -39,6 +39,7 @@ add_executable(rk3588_gtests test_frame_buffer.cpp test_behavior_event_model.cpp test_region_event.cpp + test_action_recog.cpp test_infer_backend.cpp test_image_processor.cpp test_codec_backend.cpp @@ -49,6 +50,7 @@ add_executable(rk3588_gtests ${CMAKE_SOURCE_DIR}/src/utils/config_expand.cpp ${CMAKE_SOURCE_DIR}/src/utils/dma_alloc.cpp ${CMAKE_SOURCE_DIR}/plugins/region_event/region_event_node.cpp + ${CMAKE_SOURCE_DIR}/plugins/action_recog/action_recog_node.cpp ) target_include_directories(rk3588_gtests PRIVATE @@ -56,6 +58,8 @@ target_include_directories(rk3588_gtests PRIVATE ${CMAKE_SOURCE_DIR}/third_party ) +target_compile_definitions(rk3588_gtests PRIVATE RK3588_TEST_BUILD) + target_link_libraries(rk3588_gtests PRIVATE project_options Threads::Threads diff --git a/tests/test_action_recog.cpp b/tests/test_action_recog.cpp new file mode 100644 index 0000000..01b4706 --- /dev/null +++ b/tests/test_action_recog.cpp @@ -0,0 +1,128 @@ +#include + +#include +#include + +#include "frame/frame.h" +#include "node.h" +#include "utils/simple_json.h" +#include "../plugins/action_recog/action_recog_node.h" + +namespace rk3588 { +namespace { + +SimpleJson ParseActionConfig(const std::string& text) { + SimpleJson config; + std::string err; + const bool ok = ParseSimpleJson(text, config, err); + EXPECT_TRUE(ok); + return config; +} + +TEST(ActionRecogTest, EmitsFallAfterRapidDropAndLowPosePersistence) { + ActionRecogNode node; + + const std::string config_text = R"({ + "id": "action_evt", + "events": [ + { + "type": "fall", + "window_ms": 1500, + "min_drop_pixels": 120, + "min_aspect_ratio_delta": 0.35, + "activate_duration_ms": 0 + } + ] + })"; + + SimpleJson config = ParseActionConfig(config_text); + NodeContext ctx; + auto out = std::make_shared>(4, QueueDropStrategy::DropOldest); + ctx.output_queues.push_back(out); + + ASSERT_TRUE(node.Init(config, ctx)); + ASSERT_TRUE(node.Start()); + + auto frame1 = std::make_shared(); + frame1->width = 1920; + frame1->height = 1080; + frame1->pts = 1000; + frame1->det = std::make_shared(); + frame1->det->img_w = 1920; + frame1->det->img_h = 1080; + frame1->det->items.push_back(Detection{0, 0.92f, Rect{800.0f, 240.0f, 120.0f, 320.0f}, 17}); + + auto frame2 = std::make_shared(); + frame2->width = 1920; + frame2->height = 1080; + frame2->pts = 1600; + frame2->det = std::make_shared(); + frame2->det->img_w = 1920; + frame2->det->img_h = 1080; + frame2->det->items.push_back(Detection{0, 0.94f, Rect{760.0f, 460.0f, 260.0f, 140.0f}, 17}); + + EXPECT_EQ(static_cast(node.Process(frame1)), static_cast(NodeStatus::OK)); + EXPECT_EQ(static_cast(node.Process(frame2)), static_cast(NodeStatus::OK)); + ASSERT_NE(frame2->behavior_events, nullptr); + ASSERT_EQ(frame2->behavior_events->items.size(), 1u); + EXPECT_EQ(frame2->behavior_events->items[0].type, BehaviorEventType::Fall); + ASSERT_EQ(frame2->behavior_events->items[0].track_ids.size(), 1u); + EXPECT_EQ(frame2->behavior_events->items[0].track_ids[0], 17); +} + +TEST(ActionRecogTest, EmitsFightWhenTwoTracksStayCloseWithRepeatedMotion) { + ActionRecogNode node; + + const std::string config_text = R"({ + "id": "action_evt", + "events": [ + { + "type": "fight", + "window_ms": 1200, + "proximity_pixels": 220, + "min_motion_pixels": 70, + "activate_duration_ms": 0 + } + ] + })"; + + SimpleJson config = ParseActionConfig(config_text); + NodeContext ctx; + auto out = std::make_shared>(4, QueueDropStrategy::DropOldest); + ctx.output_queues.push_back(out); + + ASSERT_TRUE(node.Init(config, ctx)); + ASSERT_TRUE(node.Start()); + + auto frame1 = std::make_shared(); + frame1->width = 1920; + frame1->height = 1080; + frame1->pts = 1000; + frame1->det = std::make_shared(); + frame1->det->img_w = 1920; + frame1->det->img_h = 1080; + frame1->det->items.push_back(Detection{0, 0.90f, Rect{700.0f, 320.0f, 120.0f, 300.0f}, 31}); + frame1->det->items.push_back(Detection{0, 0.89f, Rect{860.0f, 320.0f, 120.0f, 300.0f}, 32}); + + auto frame2 = std::make_shared(); + frame2->width = 1920; + frame2->height = 1080; + frame2->pts = 1300; + frame2->det = std::make_shared(); + frame2->det->img_w = 1920; + frame2->det->img_h = 1080; + frame2->det->items.push_back(Detection{0, 0.91f, Rect{780.0f, 340.0f, 120.0f, 300.0f}, 31}); + frame2->det->items.push_back(Detection{0, 0.92f, Rect{820.0f, 300.0f, 120.0f, 300.0f}, 32}); + + EXPECT_EQ(static_cast(node.Process(frame1)), static_cast(NodeStatus::OK)); + EXPECT_EQ(static_cast(node.Process(frame2)), static_cast(NodeStatus::OK)); + ASSERT_NE(frame2->behavior_events, nullptr); + ASSERT_EQ(frame2->behavior_events->items.size(), 1u); + EXPECT_EQ(frame2->behavior_events->items[0].type, BehaviorEventType::Fight); + ASSERT_EQ(frame2->behavior_events->items[0].track_ids.size(), 2u); + EXPECT_EQ(frame2->behavior_events->items[0].track_ids[0], 31); + EXPECT_EQ(frame2->behavior_events->items[0].track_ids[1], 32); +} + +} // namespace +} // namespace rk3588