feat: add temporal action recognition plugin

plugins/CMakeLists.txt

@@ -470,6 +470,18 @@ set_target_properties(region_event PROPERTIES
     RUNTIME_OUTPUT_DIRECTORY ${RK_PLUGIN_OUTPUT_DIR}
 )
 
+# action_recog plugin (temporal rule-based behavior events)
+add_library(action_recog SHARED
+    action_recog/action_recog_node.cpp
+)
+target_include_directories(action_recog PRIVATE ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/third_party)
+target_link_libraries(action_recog PRIVATE project_options Threads::Threads)
+set_target_properties(action_recog PROPERTIES
+    OUTPUT_NAME "action_recog"
+    LIBRARY_OUTPUT_DIRECTORY ${RK_PLUGIN_OUTPUT_DIR}
+    RUNTIME_OUTPUT_DIRECTORY ${RK_PLUGIN_OUTPUT_DIR}
+)
+
 # storage plugin (continuous recording with segment management)
 add_library(storage SHARED
     storage/storage_node.cpp
@@ -541,7 +553,7 @@ set_target_properties(ai_shoe_det PROPERTIES
     RUNTIME_OUTPUT_DIRECTORY ${RK_PLUGIN_OUTPUT_DIR}
 )
 
-install(TARGETS input_rtsp input_file publish preprocess ai_yolo ai_face_det ai_scrfd ai_scrfd_sliding ai_face_recog tracker gate osd alarm logic_gate region_event storage ai_scheduler ai_shoe_det
+install(TARGETS input_rtsp input_file publish preprocess ai_yolo ai_face_det ai_scrfd ai_scrfd_sliding ai_face_recog tracker gate osd alarm logic_gate region_event action_recog storage ai_scheduler ai_shoe_det
     LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}/rk3588-media-server/plugins
     RUNTIME DESTINATION ${CMAKE_INSTALL_LIBDIR}/rk3588-media-server/plugins
 )

plugins/action_recog/action_recog_node.cpp

@@ -0,0 +1,254 @@
+#include "action_recog_node.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <deque>
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "behavior/behavior_event.h"
+#include "utils/logger.h"
+
+namespace rk3588 {
+namespace {
+
+enum class ActionEventKind {
+    Fall,
+    Fight
+};
+
+struct ActionEventRule {
+    ActionEventKind kind = ActionEventKind::Fall;
+    uint64_t window_ms = 1000;
+    uint64_t activate_duration_ms = 0;
+    float min_drop_pixels = 0.0f;
+    float min_aspect_ratio_delta = 0.0f;
+    float proximity_pixels = 0.0f;
+    float min_motion_pixels = 0.0f;
+};
+
+struct TrackSample {
+    uint64_t pts = 0;
+    Rect bbox{};
+};
+
+static bool ParseRules(const SimpleJson& config, std::vector<ActionEventRule>& out, std::string& err) {
+    const SimpleJson* events = config.Find("events");
+    if (!events || !events->IsArray()) {
+        err = "events must be array";
+        return false;
+    }
+
+    out.clear();
+    for (const auto& ev : events->AsArray()) {
+        if (!ev.IsObject()) {
+            err = "event entry must be object";
+            return false;
+        }
+
+        ActionEventRule rule;
+        const std::string type = ev.ValueOr<std::string>("type", "");
+        rule.window_ms = static_cast<uint64_t>(std::max(0, ev.ValueOr<int>("window_ms", 1000)));
+        rule.activate_duration_ms = static_cast<uint64_t>(std::max(0, ev.ValueOr<int>("activate_duration_ms", 0)));
+
+        if (type == "fall") {
+            rule.kind = ActionEventKind::Fall;
+            rule.min_drop_pixels = ev.ValueOr<float>("min_drop_pixels", 0.0f);
+            rule.min_aspect_ratio_delta = ev.ValueOr<float>("min_aspect_ratio_delta", 0.0f);
+        } else if (type == "fight") {
+            rule.kind = ActionEventKind::Fight;
+            rule.proximity_pixels = ev.ValueOr<float>("proximity_pixels", 0.0f);
+            rule.min_motion_pixels = ev.ValueOr<float>("min_motion_pixels", 0.0f);
+        } else {
+            err = "unsupported event type: " + type;
+            return false;
+        }
+
+        out.push_back(std::move(rule));
+    }
+    return true;
+}
+
+static float CenterX(const Rect& rect) {
+    return rect.x + (rect.w * 0.5f);
+}
+
+static float CenterY(const Rect& rect) {
+    return rect.y + (rect.h * 0.5f);
+}
+
+static float CenterDistance(const Rect& lhs, const Rect& rhs) {
+    const float dx = CenterX(lhs) - CenterX(rhs);
+    const float dy = CenterY(lhs) - CenterY(rhs);
+    return std::sqrt((dx * dx) + (dy * dy));
+}
+
+static float MotionDistance(const Rect& lhs, const Rect& rhs) {
+    const float dx = CenterX(lhs) - CenterX(rhs);
+    const float dy = CenterY(lhs) - CenterY(rhs);
+    return std::sqrt((dx * dx) + (dy * dy));
+}
+
+static float AspectRatio(const Rect& rect) {
+    return rect.h > 0.0f ? (rect.w / rect.h) : 0.0f;
+}
+
+static BehaviorEventType ToBehaviorEventType(ActionEventKind kind) {
+    return kind == ActionEventKind::Fall ? BehaviorEventType::Fall : BehaviorEventType::Fight;
+}
+
+}  // namespace
+
+struct ActionRecogNode::Impl {
+    std::string init_err;
+    std::vector<ActionEventRule> rules;
+    std::map<int, std::deque<TrackSample>> history;
+};
+
+ActionRecogNode::ActionRecogNode() : impl_(std::make_unique<Impl>()) {}
+
+ActionRecogNode::~ActionRecogNode() = default;
+
+std::string ActionRecogNode::Id() const {
+    return id_;
+}
+
+std::string ActionRecogNode::Type() const {
+    return "action_recog";
+}
+
+bool ActionRecogNode::Init(const SimpleJson& config, const NodeContext& ctx) {
+    id_ = config.ValueOr<std::string>("id", "action_recog");
+    if (!ParseRules(config, impl_->rules, impl_->init_err)) {
+        LogError("[action_recog] invalid config: " + impl_->init_err);
+        return false;
+    }
+    output_queues_ = ctx.output_queues;
+    return true;
+}
+
+bool ActionRecogNode::Start() {
+    return true;
+}
+
+void ActionRecogNode::Stop() {}
+
+NodeStatus ActionRecogNode::Process(FramePtr frame) {
+    if (!frame) return NodeStatus::DROP;
+
+    EnsureBehaviorEvents(*frame);
+    if (!frame->det) {
+        PushToDownstream(frame);
+        return NodeStatus::OK;
+    }
+
+    std::map<int, Rect> current_tracks;
+    for (const auto& det : frame->det->items) {
+        if (det.track_id < 0) continue;
+        current_tracks[det.track_id] = det.bbox;
+        auto& history = impl_->history[det.track_id];
+        history.push_back(TrackSample{frame->pts, det.bbox});
+        while (!history.empty() && frame->pts > history.front().pts &&
+               (frame->pts - history.front().pts) > impl_->rules.front().window_ms) {
+            history.pop_front();
+        }
+    }
+
+    for (const auto& rule : impl_->rules) {
+        if (rule.kind == ActionEventKind::Fall) {
+            for (const auto& [track_id, bbox] : current_tracks) {
+                auto it = impl_->history.find(track_id);
+                if (it == impl_->history.end() || it->second.empty()) continue;
+                const auto& first = it->second.front();
+                const float drop = CenterY(bbox) - CenterY(first.bbox);
+                const float aspect_delta = AspectRatio(bbox) - AspectRatio(first.bbox);
+                const uint64_t duration = frame->pts >= first.pts ? (frame->pts - first.pts) : 0;
+                if (drop < rule.min_drop_pixels) continue;
+                if (aspect_delta < rule.min_aspect_ratio_delta) continue;
+                if (duration < rule.activate_duration_ms) continue;
+
+                BehaviorEventItem event;
+                event.type = ToBehaviorEventType(rule.kind);
+                event.status = BehaviorEventStatus::Active;
+                event.score = 1.0f;
+                event.bbox = bbox;
+                event.track_ids.push_back(track_id);
+                event.start_pts = first.pts;
+                event.last_pts = frame->pts;
+                event.duration_ms = duration;
+                event.source = "action_recog";
+                frame->behavior_events->items.push_back(std::move(event));
+            }
+        } else {
+            std::set<std::pair<int, int>> emitted_pairs;
+            for (auto left = current_tracks.begin(); left != current_tracks.end(); ++left) {
+                for (auto right = std::next(left); right != current_tracks.end(); ++right) {
+                    const float proximity = CenterDistance(left->second, right->second);
+                    if (proximity > rule.proximity_pixels) continue;
+
+                    const auto it_left = impl_->history.find(left->first);
+                    const auto it_right = impl_->history.find(right->first);
+                    if (it_left == impl_->history.end() || it_right == impl_->history.end()) continue;
+                    if (it_left->second.empty() || it_right->second.empty()) continue;
+
+                    const float left_motion = MotionDistance(left->second, it_left->second.front().bbox);
+                    const float right_motion = MotionDistance(right->second, it_right->second.front().bbox);
+                    const uint64_t duration_left = frame->pts >= it_left->second.front().pts ? (frame->pts - it_left->second.front().pts) : 0;
+                    const uint64_t duration_right = frame->pts >= it_right->second.front().pts ? (frame->pts - it_right->second.front().pts) : 0;
+                    const uint64_t duration = std::min(duration_left, duration_right);
+
+                    const float combined_motion = left_motion + right_motion;
+                    if (combined_motion < rule.min_motion_pixels) continue;
+                    if (duration < rule.activate_duration_ms) continue;
+
+                    const auto pair_key = std::make_pair(left->first, right->first);
+                    if (!emitted_pairs.insert(pair_key).second) continue;
+
+                    BehaviorEventItem event;
+                    event.type = ToBehaviorEventType(rule.kind);
+                    event.status = BehaviorEventStatus::Active;
+                    event.score = 1.0f;
+                    event.bbox = Rect{
+                        std::min(left->second.x, right->second.x),
+                        std::min(left->second.y, right->second.y),
+                        std::max(left->second.x + left->second.w, right->second.x + right->second.w) - std::min(left->second.x, right->second.x),
+                        std::max(left->second.y + left->second.h, right->second.y + right->second.h) - std::min(left->second.y, right->second.y)
+                    };
+                    event.track_ids.push_back(left->first);
+                    event.track_ids.push_back(right->first);
+                    event.start_pts = frame->pts - duration;
+                    event.last_pts = frame->pts;
+                    event.duration_ms = duration;
+                    event.source = "action_recog";
+                    frame->behavior_events->items.push_back(std::move(event));
+                }
+            }
+        }
+    }
+
+    PushToDownstream(frame);
+    return NodeStatus::OK;
+}
+
+void ActionRecogNode::EnsureBehaviorEvents(Frame& frame) {
+    if (!frame.behavior_events) {
+        frame.behavior_events = std::make_shared<BehaviorEventResult>();
+    }
+}
+
+void ActionRecogNode::PushToDownstream(const FramePtr& frame) {
+    for (auto& q : output_queues_) {
+        if (q) q->Push(frame);
+    }
+}
+
+#ifndef RK3588_TEST_BUILD
+REGISTER_NODE(ActionRecogNode, "action_recog");
+#endif
+
+}  // namespace rk3588

plugins/action_recog/action_recog_node.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "node.h"
+
+namespace rk3588 {
+
+class ActionRecogNode final : public INode {
+public:
+    ActionRecogNode();
+    ~ActionRecogNode() override;
+
+    std::string Id() const override;
+    std::string Type() const override;
+    bool Init(const SimpleJson& config, const NodeContext& ctx) override;
+    bool Start() override;
+    void Stop() override;
+    NodeStatus Process(FramePtr frame) override;
+
+private:
+    void EnsureBehaviorEvents(Frame& frame);
+    void PushToDownstream(const FramePtr& frame);
+
+    struct Impl;
+    std::unique_ptr<Impl> impl_;
+    std::string id_;
+    std::vector<std::shared_ptr<SpscQueue<FramePtr>>> output_queues_;
+};
+
+}  // namespace rk3588

plugins/region_event/region_event_node.cpp

@@ -227,6 +227,8 @@ void RegionEventNode::PushToDownstream(const FramePtr& frame) {
     }
 }
 
+#ifndef RK3588_TEST_BUILD
 REGISTER_NODE(RegionEventNode, "region_event");
+#endif
 
 }  // namespace rk3588

tests/CMakeLists.txt

@@ -39,6 +39,7 @@ add_executable(rk3588_gtests
     test_frame_buffer.cpp
     test_behavior_event_model.cpp
     test_region_event.cpp
+    test_action_recog.cpp
     test_infer_backend.cpp
     test_image_processor.cpp
     test_codec_backend.cpp
@@ -49,6 +50,7 @@ add_executable(rk3588_gtests
     ${CMAKE_SOURCE_DIR}/src/utils/config_expand.cpp
     ${CMAKE_SOURCE_DIR}/src/utils/dma_alloc.cpp
     ${CMAKE_SOURCE_DIR}/plugins/region_event/region_event_node.cpp
+    ${CMAKE_SOURCE_DIR}/plugins/action_recog/action_recog_node.cpp
 )
 
 target_include_directories(rk3588_gtests PRIVATE
@@ -56,6 +58,8 @@ target_include_directories(rk3588_gtests PRIVATE
     ${CMAKE_SOURCE_DIR}/third_party
 )
 
+target_compile_definitions(rk3588_gtests PRIVATE RK3588_TEST_BUILD)
+
 target_link_libraries(rk3588_gtests PRIVATE
     project_options
     Threads::Threads

tests/test_action_recog.cpp

@@ -0,0 +1,128 @@
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <string>
+
+#include "frame/frame.h"
+#include "node.h"
+#include "utils/simple_json.h"
+#include "../plugins/action_recog/action_recog_node.h"
+
+namespace rk3588 {
+namespace {
+
+SimpleJson ParseActionConfig(const std::string& text) {
+    SimpleJson config;
+    std::string err;
+    const bool ok = ParseSimpleJson(text, config, err);
+    EXPECT_TRUE(ok);
+    return config;
+}
+
+TEST(ActionRecogTest, EmitsFallAfterRapidDropAndLowPosePersistence) {
+    ActionRecogNode node;
+
+    const std::string config_text = R"({
+        "id": "action_evt",
+        "events": [
+            {
+                "type": "fall",
+                "window_ms": 1500,
+                "min_drop_pixels": 120,
+                "min_aspect_ratio_delta": 0.35,
+                "activate_duration_ms": 0
+            }
+        ]
+    })";
+
+    SimpleJson config = ParseActionConfig(config_text);
+    NodeContext ctx;
+    auto out = std::make_shared<SpscQueue<FramePtr>>(4, QueueDropStrategy::DropOldest);
+    ctx.output_queues.push_back(out);
+
+    ASSERT_TRUE(node.Init(config, ctx));
+    ASSERT_TRUE(node.Start());
+
+    auto frame1 = std::make_shared<Frame>();
+    frame1->width = 1920;
+    frame1->height = 1080;
+    frame1->pts = 1000;
+    frame1->det = std::make_shared<DetectionResult>();
+    frame1->det->img_w = 1920;
+    frame1->det->img_h = 1080;
+    frame1->det->items.push_back(Detection{0, 0.92f, Rect{800.0f, 240.0f, 120.0f, 320.0f}, 17});
+
+    auto frame2 = std::make_shared<Frame>();
+    frame2->width = 1920;
+    frame2->height = 1080;
+    frame2->pts = 1600;
+    frame2->det = std::make_shared<DetectionResult>();
+    frame2->det->img_w = 1920;
+    frame2->det->img_h = 1080;
+    frame2->det->items.push_back(Detection{0, 0.94f, Rect{760.0f, 460.0f, 260.0f, 140.0f}, 17});
+
+    EXPECT_EQ(static_cast<int>(node.Process(frame1)), static_cast<int>(NodeStatus::OK));
+    EXPECT_EQ(static_cast<int>(node.Process(frame2)), static_cast<int>(NodeStatus::OK));
+    ASSERT_NE(frame2->behavior_events, nullptr);
+    ASSERT_EQ(frame2->behavior_events->items.size(), 1u);
+    EXPECT_EQ(frame2->behavior_events->items[0].type, BehaviorEventType::Fall);
+    ASSERT_EQ(frame2->behavior_events->items[0].track_ids.size(), 1u);
+    EXPECT_EQ(frame2->behavior_events->items[0].track_ids[0], 17);
+}
+
+TEST(ActionRecogTest, EmitsFightWhenTwoTracksStayCloseWithRepeatedMotion) {
+    ActionRecogNode node;
+
+    const std::string config_text = R"({
+        "id": "action_evt",
+        "events": [
+            {
+                "type": "fight",
+                "window_ms": 1200,
+                "proximity_pixels": 220,
+                "min_motion_pixels": 70,
+                "activate_duration_ms": 0
+            }
+        ]
+    })";
+
+    SimpleJson config = ParseActionConfig(config_text);
+    NodeContext ctx;
+    auto out = std::make_shared<SpscQueue<FramePtr>>(4, QueueDropStrategy::DropOldest);
+    ctx.output_queues.push_back(out);
+
+    ASSERT_TRUE(node.Init(config, ctx));
+    ASSERT_TRUE(node.Start());
+
+    auto frame1 = std::make_shared<Frame>();
+    frame1->width = 1920;
+    frame1->height = 1080;
+    frame1->pts = 1000;
+    frame1->det = std::make_shared<DetectionResult>();
+    frame1->det->img_w = 1920;
+    frame1->det->img_h = 1080;
+    frame1->det->items.push_back(Detection{0, 0.90f, Rect{700.0f, 320.0f, 120.0f, 300.0f}, 31});
+    frame1->det->items.push_back(Detection{0, 0.89f, Rect{860.0f, 320.0f, 120.0f, 300.0f}, 32});
+
+    auto frame2 = std::make_shared<Frame>();
+    frame2->width = 1920;
+    frame2->height = 1080;
+    frame2->pts = 1300;
+    frame2->det = std::make_shared<DetectionResult>();
+    frame2->det->img_w = 1920;
+    frame2->det->img_h = 1080;
+    frame2->det->items.push_back(Detection{0, 0.91f, Rect{780.0f, 340.0f, 120.0f, 300.0f}, 31});
+    frame2->det->items.push_back(Detection{0, 0.92f, Rect{820.0f, 300.0f, 120.0f, 300.0f}, 32});
+
+    EXPECT_EQ(static_cast<int>(node.Process(frame1)), static_cast<int>(NodeStatus::OK));
+    EXPECT_EQ(static_cast<int>(node.Process(frame2)), static_cast<int>(NodeStatus::OK));
+    ASSERT_NE(frame2->behavior_events, nullptr);
+    ASSERT_EQ(frame2->behavior_events->items.size(), 1u);
+    EXPECT_EQ(frame2->behavior_events->items[0].type, BehaviorEventType::Fight);
+    ASSERT_EQ(frame2->behavior_events->items[0].track_ids.size(), 2u);
+    EXPECT_EQ(frame2->behavior_events->items[0].track_ids[0], 31);
+    EXPECT_EQ(frame2->behavior_events->items[0].track_ids[1], 32);
+}
+
+}  // namespace
+}  // namespace rk3588