diff --git a/CMakeLists.txt b/CMakeLists.txt index bd07453..1e3299f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,6 +57,7 @@ set(SRC_FILES src/ai_scheduler.cpp src/http_server.cpp src/hw_factory.cpp + src/hw_image_processor.cpp src/utils/dma_alloc.cpp src/utils/config_expand.cpp ) diff --git a/include/graph_manager.h b/include/graph_manager.h index f18e070..38014cb 100644 --- a/include/graph_manager.h +++ b/include/graph_manager.h @@ -20,6 +20,8 @@ namespace rk3588 { +class IInferBackend; + struct QueueSnapshot { size_t size = 0; size_t capacity = 0; @@ -158,6 +160,7 @@ private: SimpleJson graph_cfg_; size_t built_default_queue_size_ = 8; QueueDropStrategy built_default_strategy_ = QueueDropStrategy::DropOldest; + std::shared_ptr infer_backend_; std::vector nodes_; std::vector edges_; std::unique_ptr executor_; diff --git a/include/hw/i_infer_backend.h b/include/hw/i_infer_backend.h index 1c12152..dab78d6 100644 --- a/include/hw/i_infer_backend.h +++ b/include/hw/i_infer_backend.h @@ -11,6 +11,8 @@ public: virtual ~IInferBackend() = default; virtual ModelHandle LoadModel(const std::string& model_path, std::string& err) = 0; + virtual void UnloadModel(ModelHandle handle) = 0; + virtual bool GetModelInfo(ModelHandle handle, ModelInfo& info) const = 0; virtual InferResult Infer(ModelHandle handle, const InferInput& input) = 0; virtual AiScheduler::BorrowedInferResult InferBorrowed(ModelHandle handle, const InferInput& input) = 0; }; diff --git a/include/hw/rk3588_defaults.h b/include/hw/rk3588_defaults.h index 813be2b..2222911 100644 --- a/include/hw/rk3588_defaults.h +++ b/include/hw/rk3588_defaults.h @@ -1,52 +1,74 @@ #pragma once +#include +#include #include +#include #include "hw/i_decoder.h" #include "hw/i_encoder.h" #include "hw/i_image_processor.h" #include "hw/i_infer_backend.h" +#include "utils/simple_json.h" #include "utils/result.h" namespace rk3588 { -class Rk3588InferBackend : public IInferBackend { +class RknnInferBackend : public IInferBackend { public: - ModelHandle LoadModel(const std::string& /*model_path*/, std::string& err) override { - err = "not implemented"; - return kInvalidModelHandle; + ModelHandle LoadModel(const std::string& model_path, std::string& err) override { + return AiScheduler::Instance().LoadModel(model_path, err); } - InferResult Infer(ModelHandle /*handle*/, const InferInput& /*input*/) override { - InferResult result; - result.success = false; - result.error = "not implemented"; - return result; + void UnloadModel(ModelHandle handle) override { + AiScheduler::Instance().UnloadModel(handle); } - AiScheduler::BorrowedInferResult InferBorrowed(ModelHandle /*handle*/, const InferInput& /*input*/) override { - AiScheduler::BorrowedInferResult result; - result.success = false; - result.error = "not implemented"; - return result; + bool GetModelInfo(ModelHandle handle, ModelInfo& info) const override { + return AiScheduler::Instance().GetModelInfo(handle, info); } + + InferResult Infer(ModelHandle handle, const InferInput& input) override { + return AiScheduler::Instance().Infer(handle, input); + } + + AiScheduler::BorrowedInferResult InferBorrowed(ModelHandle handle, const InferInput& input) override { + { + std::lock_guard lock(mu_); + last_borrowed_input_ = input; + } + return AiScheduler::Instance().InferBorrowed(handle, input); + } + + InferInput GetLastBorrowedInput() const { + std::lock_guard lock(mu_); + return last_borrowed_input_; + } + +private: + mutable std::mutex mu_; + InferInput last_borrowed_input_; }; +using Rk3588InferBackend = RknnInferBackend; + class Rk3588ImageProcessor : public IImageProcessor { public: - Status Resize(const Frame& /*src*/, Frame& /*dst*/) override { - return FailStatus("not implemented"); - } + explicit Rk3588ImageProcessor(const SimpleJson& config); - Status CvtColor(const Frame& /*src*/, Frame& /*dst*/, PixelFormat /*dst_format*/) override { - return FailStatus("not implemented"); - } + Status Resize(const Frame& src, Frame& dst) override; + Status CvtColor(const Frame& src, Frame& dst, PixelFormat dst_format) override; + Status Normalize(const Frame& src, std::vector& out, + const std::vector& mean, + const std::vector& std) override; - Status Normalize(const Frame& /*src*/, std::vector& /*out*/, - const std::vector& /*mean*/, - const std::vector& /*std*/) override { - return FailStatus("not implemented"); - } + bool IsUsingRga() const; + int RgaMaxInflight() const; + std::string RgaGateKey() const; + +private: + std::shared_ptr impl_; + bool use_rga_ = true; }; class Rk3588Decoder : public IDecoder { diff --git a/include/node.h b/include/node.h index b4ec589..4766d82 100644 --- a/include/node.h +++ b/include/node.h @@ -12,6 +12,9 @@ namespace rk3588 { using FramePtr = std::shared_ptr; +class IInferBackend; +class IImageProcessor; + enum class NodeStatus { OK, // Normal processing DROP, // Drop this frame (e.g., condition not met) @@ -23,6 +26,8 @@ struct NodeContext { std::shared_ptr> input_queue; // For nodes that produce downstream outputs (one queue per outgoing edge). std::vector>> output_queues; + std::shared_ptr infer_backend; + std::shared_ptr image_processor; }; class INode { diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 27694c9..94f3f78 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -186,6 +186,7 @@ set_target_properties(publish PROPERTIES add_library(preprocess SHARED preprocess/preprocess_node.cpp + ${CMAKE_SOURCE_DIR}/src/hw_image_processor.cpp ${CMAKE_SOURCE_DIR}/src/utils/dma_alloc.cpp ) target_include_directories(preprocess PRIVATE ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/third_party) diff --git a/plugins/ai_face_det/ai_face_det_node.cpp b/plugins/ai_face_det/ai_face_det_node.cpp index 0f3a288..1d3e660 100644 --- a/plugins/ai_face_det/ai_face_det_node.cpp +++ b/plugins/ai_face_det/ai_face_det_node.cpp @@ -11,7 +11,7 @@ #include #include -#include "ai_scheduler.h" +#include "hw/i_infer_backend.h" #include "face/face_result.h" #include "node.h" #include "utils/dma_alloc.h" @@ -440,19 +440,25 @@ public: return false; } + infer_backend_ = ctx.infer_backend; + if (!infer_backend_) { + LogError("[ai_face_det] no infer backend for node " + id_); + return false; + } + #if defined(RK3588_ENABLE_RKNN) if (model_path_.empty()) { LogError("[ai_face_det] model_path is required"); return false; } std::string err; - model_handle_ = AiScheduler::Instance().LoadModel(model_path_, err); + model_handle_ = infer_backend_->LoadModel(model_path_, err); if (model_handle_ == kInvalidModelHandle) { LogError("[ai_face_det] failed to load model: " + err); return false; } ModelInfo info; - if (AiScheduler::Instance().GetModelInfo(model_handle_, info)) { + if (infer_backend_->GetModelInfo(model_handle_, info)) { model_w_ = info.input_width; model_h_ = info.input_height; n_output_ = info.n_output; @@ -510,7 +516,7 @@ public: void Stop() override { #if defined(RK3588_ENABLE_RKNN) if (model_handle_ != kInvalidModelHandle) { - AiScheduler::Instance().UnloadModel(model_handle_); + infer_backend_->UnloadModel(model_handle_); model_handle_ = kInvalidModelHandle; } #endif @@ -627,7 +633,7 @@ private: if (sync_src) DmaSyncEndFd(frame->dma_fd); - auto r = AiScheduler::Instance().InferBorrowed(model_handle_, input); + auto r = infer_backend_->InferBorrowed(model_handle_, input); if (!r.success) { LogWarn("[ai_face_det] inference failed: " + r.error); return; @@ -860,6 +866,7 @@ private: std::shared_ptr> input_queue_; std::vector>> output_queues_; + std::shared_ptr infer_backend_; std::vector input_buf_; std::vector float_input_buf_; diff --git a/plugins/ai_face_recog/ai_face_recog_node.cpp b/plugins/ai_face_recog/ai_face_recog_node.cpp index 10285f7..dbcb2c3 100644 --- a/plugins/ai_face_recog/ai_face_recog_node.cpp +++ b/plugins/ai_face_recog/ai_face_recog_node.cpp @@ -12,7 +12,7 @@ #include #include -#include "ai_scheduler.h" +#include "hw/i_infer_backend.h" #include "face/face_result.h" #include "node.h" #include "utils/dma_alloc.h" @@ -663,19 +663,25 @@ public: return false; } + infer_backend_ = ctx.infer_backend; + if (!infer_backend_) { + LogError("[ai_face_recog] no infer backend for node " + id_); + return false; + } + #if defined(RK3588_ENABLE_RKNN) if (model_path_.empty()) { LogError("[ai_face_recog] model_path is required"); return false; } std::string err; - model_handle_ = AiScheduler::Instance().LoadModel(model_path_, err); + model_handle_ = infer_backend_->LoadModel(model_path_, err); if (model_handle_ == kInvalidModelHandle) { LogError("[ai_face_recog] failed to load model: " + err); return false; } ModelInfo info; - if (AiScheduler::Instance().GetModelInfo(model_handle_, info)) { + if (infer_backend_->GetModelInfo(model_handle_, info)) { model_w_ = info.input_width; model_h_ = info.input_height; } @@ -746,7 +752,7 @@ public: void Stop() override { #if defined(RK3588_ENABLE_RKNN) if (model_handle_ != kInvalidModelHandle) { - AiScheduler::Instance().UnloadModel(model_handle_); + infer_backend_->UnloadModel(model_handle_); model_handle_ = kInvalidModelHandle; } #endif @@ -893,7 +899,7 @@ private: in.type = RKNN_TENSOR_UINT8; } - auto r = AiScheduler::Instance().InferBorrowed(model_handle_, in); + auto r = infer_backend_->InferBorrowed(model_handle_, in); if (!r.success || r.outputs.empty()) { LogWarn(std::string("[ai_face_recog] inference failed: ") + (r.error.empty() ? "unknown" : r.error)); continue; @@ -944,6 +950,7 @@ private: std::shared_ptr> input_queue_; std::vector>> output_queues_; + std::shared_ptr infer_backend_; std::vector face_buf_; std::vector float_input_buf_; diff --git a/plugins/ai_yolo/ai_yolo_node.cpp b/plugins/ai_yolo/ai_yolo_node.cpp index 66eaad9..698c4a6 100644 --- a/plugins/ai_yolo/ai_yolo_node.cpp +++ b/plugins/ai_yolo/ai_yolo_node.cpp @@ -9,7 +9,7 @@ #include #include -#include "ai_scheduler.h" +#include "hw/i_infer_backend.h" #include "node.h" #include "utils/dma_alloc.h" #include "utils/logger.h" @@ -319,6 +319,12 @@ public: } output_queues_ = ctx.output_queues; + infer_backend_ = ctx.infer_backend; + if (!infer_backend_) { + LogError("[ai_yolo] no infer backend for node " + id_); + return false; + } + #if defined(RK3588_ENABLE_RKNN) if (model_path_.empty()) { LogError("[ai_yolo] model_path is required"); @@ -326,14 +332,14 @@ public: } std::string err; - model_handle_ = AiScheduler::Instance().LoadModel(model_path_, err); + model_handle_ = infer_backend_->LoadModel(model_path_, err); if (model_handle_ == kInvalidModelHandle) { LogError("[ai_yolo] failed to load model: " + err); return false; } ModelInfo info; - if (AiScheduler::Instance().GetModelInfo(model_handle_, info)) { + if (infer_backend_->GetModelInfo(model_handle_, info)) { model_input_w_ = info.input_width; model_input_h_ = info.input_height; n_output_ = info.n_output; @@ -347,7 +353,7 @@ public: } } - LogInfo("[ai_yolo] model loaded via AiScheduler: " + model_path_ + + LogInfo("[ai_yolo] model loaded via InferBackend: " + model_path_ + " (handle=" + std::to_string(model_handle_) + ", version=" + (yolo_version_ == YoloVersion::V5 ? "v5" : "v8") + ")"); #else @@ -365,7 +371,7 @@ public: void Stop() override { #if defined(RK3588_ENABLE_RKNN) if (model_handle_ != kInvalidModelHandle) { - AiScheduler::Instance().UnloadModel(model_handle_); + infer_backend_->UnloadModel(model_handle_); model_handle_ = kInvalidModelHandle; } #endif @@ -460,7 +466,7 @@ private: input.height = h; input.is_nhwc = true; - auto result = AiScheduler::Instance().InferBorrowed(model_handle_, input); + auto result = infer_backend_->InferBorrowed(model_handle_, input); if (!result.success) { LogWarn("[ai_yolo] inference failed: " + result.error); return; @@ -720,6 +726,7 @@ private: std::shared_ptr> input_queue_; std::vector>> output_queues_; + std::shared_ptr infer_backend_; uint64_t processed_ = 0; bool stats_log_ = false; diff --git a/plugins/preprocess/preprocess_node.cpp b/plugins/preprocess/preprocess_node.cpp index 679a978..9984b5a 100644 --- a/plugins/preprocess/preprocess_node.cpp +++ b/plugins/preprocess/preprocess_node.cpp @@ -1,159 +1,17 @@ +#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include #include +#include "hw/i_image_processor.h" #include "node.h" -#include "utils/dma_alloc.h" #include "utils/logger.h" -#if defined(RK3588_ENABLE_RGA) -#include "im2d.hpp" -#include "im2d_buffer.h" -#include "im2d_type.h" -#endif - -#if defined(RK3588_ENABLE_FFMPEG) -extern "C" { -#include -#include -} -#endif - namespace rk3588 { namespace { -inline int Align16(int v) { return (v + 15) & ~15; } - -#if defined(RK3588_ENABLE_RGA) -int ToRgaFormat(PixelFormat fmt) { - switch (fmt) { - case PixelFormat::NV12: return RK_FORMAT_YCbCr_420_SP; - case PixelFormat::YUV420: return RK_FORMAT_YCbCr_420_P; - case PixelFormat::RGB: return RK_FORMAT_RGB_888; - case PixelFormat::BGR: return RK_FORMAT_BGR_888; - default: return RK_FORMAT_UNKNOWN; - } -} - -std::atomic& GlobalRgaMaxInflightRef() { - static std::atomic v{2}; - return v; -} - -int GlobalRgaMaxInflight() { - static std::once_flag once; - std::call_once(once, []() { - const char* s = std::getenv("RK3588_RGA_MAX_INFLIGHT"); - if (!s || !*s) return; - try { - int v = std::stoi(s); - if (v > 0 && v <= 32) GlobalRgaMaxInflightRef().store(v); - } catch (...) { - LogWarn(std::string("[preprocess] invalid RK3588_RGA_MAX_INFLIGHT='") + s + "'"); - } - }); - int v = GlobalRgaMaxInflightRef().load(); - return v > 0 ? v : 1; -} - -class RgaGate { -public: - explicit RgaGate(int max_inflight) : max_inflight_(max_inflight > 0 ? max_inflight : 1) {} - - void Acquire() { - std::unique_lock lock(mu_); - cv_.wait(lock, [&]() { return in_flight_ < max_inflight_; }); - ++in_flight_; - } - - void Release() { - std::lock_guard lock(mu_); - if (in_flight_ > 0) { - --in_flight_; - } - cv_.notify_one(); - } - - void SetMaxInflight(int v) { - if (v <= 0) return; - if (v > 32) v = 32; - { - std::lock_guard lock(mu_); - max_inflight_ = v; - } - cv_.notify_all(); - } - - int MaxInflight() const { - std::lock_guard lock(mu_); - return max_inflight_; - } - -private: - mutable std::mutex mu_; - std::condition_variable cv_; - int in_flight_ = 0; - int max_inflight_ = 1; -}; - -class RgaGateRegistry { -public: - static RgaGateRegistry& Instance() { - static RgaGateRegistry* inst = new RgaGateRegistry(); - return *inst; - } - - RgaGate& Get(const std::string& key) { - std::lock_guard lock(mu_); - auto it = gates_.find(key); - if (it != gates_.end()) return *it->second; - auto gate = std::make_unique(GlobalRgaMaxInflight()); - RgaGate& ref = *gate; - gates_.emplace(key, std::move(gate)); - return ref; - } - -private: - std::mutex mu_; - std::unordered_map> gates_; -}; - -RgaGate& GetRgaGate(const std::string& key) { - const std::string k = key.empty() ? "global" : key; - return RgaGateRegistry::Instance().Get(k); -} - -class ScopedRgaGate { -public: - explicit ScopedRgaGate(const std::string& key) : gate_(&GetRgaGate(key)) { gate_->Acquire(); } - ~ScopedRgaGate() { gate_->Release(); } - ScopedRgaGate(const ScopedRgaGate&) = delete; - ScopedRgaGate& operator=(const ScopedRgaGate&) = delete; - -private: - RgaGate* gate_ = nullptr; -}; - -void EnsureRgaInitializedOnce() { - static std::once_flag once; - std::call_once(once, []() { - const IM_STATUS st = imcheckHeader(); - if (st != IM_STATUS_NOERROR && st != IM_STATUS_SUCCESS) { - LogWarn(std::string("[preprocess] imcheckHeader failed: ") + imStrError(st)); - } - }); -} -#endif - PixelFormat ParseFormat(const std::string& s) { if (s == "nv12" || s == "NV12") return PixelFormat::NV12; if (s == "yuv420" || s == "YUV420") return PixelFormat::YUV420; @@ -162,145 +20,6 @@ PixelFormat ParseFormat(const std::string& s) { return PixelFormat::UNKNOWN; } -size_t CalcImageSize(int w, int h, PixelFormat fmt) { - switch (fmt) { - case PixelFormat::NV12: - case PixelFormat::YUV420: - return static_cast(w) * h * 3 / 2; - case PixelFormat::RGB: - case PixelFormat::BGR: - return static_cast(w) * h * 3; - default: - return 0; - } -} - -size_t CalcImageSizeStrided(int wstride, int hstride, PixelFormat fmt) { - if (wstride <= 0 || hstride <= 0) return 0; - const size_t ws = static_cast(wstride); - const size_t hs = static_cast(hstride); - switch (fmt) { - case PixelFormat::NV12: { - // Y: ws*hs, UV: ws*(hs/2) - return ws * hs + ws * (hs / 2); - } - case PixelFormat::YUV420: { - // Y: ws*hs, U/V: (ws/2)*(hs/2) - const size_t y = ws * hs; - const size_t uv = (ws / 2) * (hs / 2); - return y + uv + uv; - } - case PixelFormat::RGB: - case PixelFormat::BGR: - return ws * hs * 3; - default: - return 0; - } -} - -bool CopyToStridedBuffer(const Frame& src, uint8_t* dst, size_t dst_size, - int dst_wstride, int dst_hstride) { - if (!dst || dst_size == 0) return false; - if (dst_wstride <= 0 || dst_hstride <= 0) return false; - - std::memset(dst, 0, dst_size); - - const int w = src.width; - const int h = src.height; - if (w <= 0 || h <= 0) return false; - - if (src.format == PixelFormat::NV12) { - const size_t y_bytes = static_cast(dst_wstride) * dst_hstride; - const size_t uv_bytes = static_cast(dst_wstride) * (dst_hstride / 2); - if (y_bytes + uv_bytes > dst_size) return false; - - const uint8_t* src_y = src.planes[0].data ? src.planes[0].data : src.data; - const uint8_t* src_uv = src.planes[1].data ? src.planes[1].data : nullptr; - const int src_y_stride = src.planes[0].stride > 0 ? src.planes[0].stride : w; - const int src_uv_stride = src.planes[1].stride > 0 ? src.planes[1].stride : w; - if (!src_y) return false; - if (!src_uv) { - // Fallback: packed NV12 layout. - if (!src.data) return false; - src_uv = src.data + static_cast(src_y_stride) * static_cast(h); - } - - for (int row = 0; row < h; ++row) { - std::memcpy(dst + static_cast(row) * dst_wstride, - src_y + static_cast(row) * src_y_stride, - static_cast(w)); - } - - uint8_t* dst_uv = dst + y_bytes; - const int uv_rows = h / 2; - for (int row = 0; row < uv_rows; ++row) { - std::memcpy(dst_uv + static_cast(row) * dst_wstride, - src_uv + static_cast(row) * src_uv_stride, - static_cast(w)); - } - return true; - } - - if (src.format == PixelFormat::YUV420) { - const size_t y_bytes = static_cast(dst_wstride) * dst_hstride; - const size_t uv_stride = static_cast(dst_wstride) / 2; - const size_t uv_h = static_cast(dst_hstride) / 2; - const size_t u_bytes = uv_stride * uv_h; - const size_t v_bytes = u_bytes; - if (y_bytes + u_bytes + v_bytes > dst_size) return false; - - const uint8_t* src_y = src.planes[0].data ? src.planes[0].data : src.data; - const uint8_t* src_u = src.planes[1].data ? src.planes[1].data : nullptr; - const uint8_t* src_v = src.planes[2].data ? src.planes[2].data : nullptr; - const int src_y_stride = src.planes[0].stride > 0 ? src.planes[0].stride : w; - const int src_u_stride = src.planes[1].stride > 0 ? src.planes[1].stride : (w / 2); - const int src_v_stride = src.planes[2].stride > 0 ? src.planes[2].stride : (w / 2); - if (!src_y || !src_u || !src_v) return false; - - for (int row = 0; row < h; ++row) { - std::memcpy(dst + static_cast(row) * dst_wstride, - src_y + static_cast(row) * src_y_stride, - static_cast(w)); - } - - uint8_t* dst_u = dst + y_bytes; - uint8_t* dst_v = dst + y_bytes + u_bytes; - const int uv_rows = h / 2; - const int uv_cols = w / 2; - for (int row = 0; row < uv_rows; ++row) { - std::memcpy(dst_u + static_cast(row) * uv_stride, - src_u + static_cast(row) * src_u_stride, - static_cast(uv_cols)); - std::memcpy(dst_v + static_cast(row) * uv_stride, - src_v + static_cast(row) * src_v_stride, - static_cast(uv_cols)); - } - return true; - } - - if (src.format == PixelFormat::RGB || src.format == PixelFormat::BGR) { - const size_t need = static_cast(dst_wstride) * dst_hstride * 3; - if (need > dst_size) return false; - - const uint8_t* src_rgb = src.planes[0].data ? src.planes[0].data : src.data; - const int src_stride = src.planes[0].stride > 0 - ? src.planes[0].stride - : (src.stride > 0 ? src.stride : w * 3); - if (!src_rgb) return false; - - const size_t dst_stride = static_cast(dst_wstride) * 3; - const size_t row_bytes = static_cast(w) * 3; - for (int row = 0; row < h; ++row) { - std::memcpy(dst + static_cast(row) * dst_stride, - src_rgb + static_cast(row) * src_stride, - row_bytes); - } - return true; - } - - return false; -} - } // namespace class PreprocessNode : public INode { @@ -314,32 +33,18 @@ public: dst_h_ = config.ValueOr("dst_h", 640); keep_ratio_ = config.ValueOr("keep_ratio", false); - if (config.Find("dst_packed")) { - dst_packed_ = config.ValueOr("dst_packed", false); - dst_packed_explicit_ = true; - } else { - dst_packed_ = false; - dst_packed_explicit_ = false; - } - std::string fmt_str = config.ValueOr("dst_format", ""); if (!fmt_str.empty()) { dst_fmt_ = ParseFormat(fmt_str); } -#if defined(RK3588_ENABLE_RGA) - rga_gate_ = config.ValueOr("rga_gate", "global"); - const int rga_max_inflight = config.ValueOr("rga_max_inflight", 0); - if (rga_max_inflight > 0) { - GetRgaGate(rga_gate_).SetMaxInflight(rga_max_inflight); - } -#endif const bool requested_use_rga = config.ValueOr("use_rga", true); use_rga_ = requested_use_rga; if (const SimpleJson* dbg = config.Find("debug"); dbg && dbg->IsObject()) { stats_log_ = dbg->ValueOr("stats", stats_log_); - stats_interval_ = std::max(1, static_cast(dbg->ValueOr("stats_interval", static_cast(stats_interval_)))); + stats_interval_ = std::max( + 1, static_cast(dbg->ValueOr("stats_interval", static_cast(stats_interval_)))); } input_queue_ = ctx.input_queue; @@ -366,46 +71,90 @@ public: return false; } #endif + + image_processor_ = ctx.image_processor; + if (!image_processor_) { + LogError("[preprocess] no image processor for node " + id_); + return false; + } + return true; } bool Start() override { - std::string extra; -#if defined(RK3588_ENABLE_RGA) - if (use_rga_) { - extra = " gate=" + rga_gate_ + " max_inflight=" + std::to_string(GetRgaGate(rga_gate_).MaxInflight()); - } -#endif - LogInfo("[preprocess] start id=" + id_ + " dst=" + std::to_string(dst_w_) + "x" + std::to_string(dst_h_) + - (use_rga_ ? " (rga)" : " (swscale)") + extra); + LogInfo("[preprocess] start id=" + id_ + " dst=" + std::to_string(dst_w_) + "x" + + std::to_string(dst_h_) + (use_rga_ ? " (rga)" : " (swscale)")); return true; } - void Stop() override { -#if defined(RK3588_ENABLE_FFMPEG) - if (sws_ctx_) { - sws_freeContext(sws_ctx_); - sws_ctx_ = nullptr; - } -#endif - } + void Stop() override {} NodeStatus Process(FramePtr frame) override { if (!frame) return NodeStatus::DROP; + if (!image_processor_) return NodeStatus::ERROR; -#if defined(RK3588_ENABLE_RGA) - if (use_rga_) { - if (!ProcessRga(frame)) { - return NodeStatus::ERROR; - } - } else { - ProcessSwscale(frame); + PixelFormat out_fmt = (dst_fmt_ != PixelFormat::UNKNOWN) ? dst_fmt_ : frame->format; + int out_w = dst_w_; + int out_h = dst_h_; + + if (out_w <= 0) out_w = frame->width; + if (out_h <= 0) out_h = frame->height; + + if (keep_ratio_ && dst_w_ > 0 && dst_h_ > 0 && frame->width > 0 && frame->height > 0) { + float scale = std::min(static_cast(dst_w_) / frame->width, + static_cast(dst_h_) / frame->height); + out_w = static_cast(frame->width * scale); + out_h = static_cast(frame->height * scale); + out_w = (out_w + 1) & ~1; + out_h = (out_h + 1) & ~1; } -#elif defined(RK3588_ENABLE_FFMPEG) - ProcessSwscale(frame); -#else - ProcessPassthrough(frame); -#endif + + const bool need_resize = (frame->width != out_w || frame->height != out_h); + const bool need_cvt = (frame->format != out_fmt); + + if (need_resize) { + WarnMetaResizeOnce(frame, out_w, out_h); + } + + if (!need_resize && !need_cvt) { + ProcessPassthrough(frame); + return NodeStatus::OK; + } + + Frame out; + out.width = out_w; + out.height = out_h; + out.format = out_fmt; + + Status st = image_processor_->Resize(*frame, out); + if (st.Failed()) { + if (!use_rga_ && st.ErrMessage().find("unsupported format") != std::string::npos) { + ProcessPassthrough(frame); + return NodeStatus::OK; + } + LogError("[preprocess] " + st.ErrMessage()); + return NodeStatus::ERROR; + } + + auto out_frame = std::make_shared(out); + out_frame->pts = frame->pts; + out_frame->frame_id = frame->frame_id; + out_frame->det = frame->det; + out_frame->face_det = frame->face_det; + out_frame->face_recog = frame->face_recog; + out_frame->user_meta = frame->user_meta; + + PushToDownstream(out_frame); + ++processed_; + + if (stats_log_ && stats_interval_ > 0 && (processed_ % stats_interval_) == 0) { + LogInfo("[preprocess] " + std::string(use_rga_ ? "rga" : "swscale") + + " frame=" + std::to_string(out_frame->frame_id) + + " " + std::to_string(frame->width) + "x" + std::to_string(frame->height) + + " -> " + std::to_string(out_w) + "x" + std::to_string(out_h) + + " id=" + id_); + } + return NodeStatus::OK; } @@ -433,495 +182,13 @@ private: } } -#if defined(RK3588_ENABLE_RGA) - bool ProcessRga(FramePtr frame) { - EnsureRgaInitializedOnce(); - - PixelFormat out_fmt = (dst_fmt_ != PixelFormat::UNKNOWN) ? dst_fmt_ : frame->format; - int out_w = dst_w_; - int out_h = dst_h_; - - // Allow config to follow source resolution: - // - dst_w <= 0 => out_w = frame->width - // - dst_h <= 0 => out_h = frame->height - if (out_w <= 0) out_w = frame->width; - if (out_h <= 0) out_h = frame->height; - - if (keep_ratio_ && dst_w_ > 0 && dst_h_ > 0 && frame->width > 0 && frame->height > 0) { - float scale = std::min(static_cast(dst_w_) / frame->width, - static_cast(dst_h_) / frame->height); - out_w = static_cast(frame->width * scale); - out_h = static_cast(frame->height * scale); - out_w = (out_w + 1) & ~1; - out_h = (out_h + 1) & ~1; - } - - int src_fmt_rga = ToRgaFormat(frame->format); - int dst_fmt_rga = ToRgaFormat(out_fmt); - bool need_cvt = (src_fmt_rga != dst_fmt_rga); - bool need_resize = (frame->width != out_w || frame->height != out_h); - - if (need_resize) { - WarnMetaResizeOnce(frame, out_w, out_h); - } - - // If no processing needed, passthrough directly - if (!need_cvt && !need_resize) { - PushToDownstream(frame); - ++processed_; - if (stats_log_ && stats_interval_ > 0 && (processed_ % stats_interval_) == 0) { - LogInfo("[preprocess] passthrough frame=" + std::to_string(frame->frame_id) + - " " + std::to_string(frame->width) + "x" + std::to_string(frame->height) + " (no change)" + - " id=" + id_); - } - return true; - } - - // Calculate proper strides. - // IMPORTANT: For DMA-BUF frames (e.g. MPP decode output), the actual vertical stride - // (ver_stride) may be larger than `height`. We must honor that, otherwise RGA will - // read UV from the wrong offset (典型花屏/错色). - int src_wstride = Align16(frame->width); - int src_hstride = Align16(frame->height); - if (frame->format == PixelFormat::NV12 || frame->format == PixelFormat::YUV420) { - const int y_stride = frame->planes[0].stride > 0 ? frame->planes[0].stride - : (frame->stride > 0 ? frame->stride : frame->width); - if (y_stride > 0) src_wstride = y_stride; - if (frame->planes[0].size > 0 && y_stride > 0) { - const int hs = frame->planes[0].size / y_stride; - if (hs >= frame->height) src_hstride = hs; - } - } else if (frame->format == PixelFormat::RGB || frame->format == PixelFormat::BGR) { - const int stride_bytes = frame->planes[0].stride > 0 ? frame->planes[0].stride - : (frame->stride > 0 ? frame->stride : frame->width * 3); - if (stride_bytes > 0 && (stride_bytes % 3) == 0) { - src_wstride = stride_bytes / 3; - } - if (frame->planes[0].size > 0 && stride_bytes > 0) { - const int hs = frame->planes[0].size / stride_bytes; - if (hs >= frame->height) src_hstride = hs; - } - } - int dst_wstride = Align16(out_w); - // For AI input (RGB/BGR), allow a tightly packed output to avoid an extra per-frame memcpy - // in downstream nodes (e.g. ai_yolo). - const bool want_packed_rgb = (out_fmt == PixelFormat::RGB || out_fmt == PixelFormat::BGR) && - (!dst_packed_explicit_ || dst_packed_); - if (want_packed_rgb) { - dst_wstride = out_w; - } - int dst_hstride = Align16(out_h); - - if (src_fmt_rga == RK_FORMAT_UNKNOWN || dst_fmt_rga == RK_FORMAT_UNKNOWN) { - LogError("[preprocess] unsupported format for RGA"); - return false; - } - - size_t out_size = CalcImageSizeStrided(dst_wstride, dst_hstride, out_fmt); - if (out_size == 0) { - LogError("[preprocess] invalid output size for RGA"); - return false; - } - - // Use DMA-BUF allocation to avoid >4GB address issue with RGA - auto dma_buf = DmaAlloc(out_size); - if (!dma_buf || !dma_buf->valid()) { - LogError("[preprocess] DMA alloc failed"); - return false; - } - - if (stats_log_ && processed_ < 3) { - LogInfo("[preprocess] src: " + std::to_string(frame->width) + "x" + std::to_string(frame->height) + - " fmt=" + std::to_string(static_cast(frame->format)) + - " rga_fmt=" + std::to_string(src_fmt_rga) + - " wstride=" + std::to_string(src_wstride) + - " hstride=" + std::to_string(src_hstride) + - " data_size=" + std::to_string(frame->data_size)); - LogInfo("[preprocess] dst: " + std::to_string(out_w) + "x" + std::to_string(out_h) + - " fmt=" + std::to_string(static_cast(out_fmt)) + - " rga_fmt=" + std::to_string(dst_fmt_rga) + - " wstride=" + std::to_string(dst_wstride) + - " hstride=" + std::to_string(dst_hstride) + - " out_size=" + std::to_string(out_size)); - } - - rga_buffer_t src_buf{}; - rga_buffer_t dst_buf{}; - DmaBufferPtr src_dma_buf; // keep alive if we allocate/copy - - // Lazily allocate tmp buffer only when we must fall back to a 2-step pipeline. - // (resize + cvtcolor). Prefer a single improcess() call to reduce scheduling overhead. - DmaBufferPtr tmp_dma; - - const bool can_cpu_read_src = (frame->data != nullptr && frame->data_size > 0); - auto CopySrcToDmaIfPossible = [&]() -> bool { - if (!can_cpu_read_src) return false; - // Use tight/aligned strides for the copied source buffer. - const int copy_wstride = Align16(frame->width); - const int copy_hstride = Align16(frame->height); - const size_t src_size = CalcImageSizeStrided(copy_wstride, copy_hstride, frame->format); - src_dma_buf = DmaAlloc(src_size); - if (!src_dma_buf || !src_dma_buf->valid()) { - return false; - } - - // If source is a DMA-BUF, sync it before CPU reads. - if (frame->dma_fd >= 0) { - DmaSyncStartFd(frame->dma_fd); - } - - // CPU writes to a DMA-BUF must be flushed before RGA reads it. - DmaSyncStartFd(src_dma_buf->fd); - const bool ok = CopyToStridedBuffer(*frame, src_dma_buf->data(), src_dma_buf->size, - copy_wstride, copy_hstride); - DmaSyncEndFd(src_dma_buf->fd); - - if (frame->dma_fd >= 0) { - DmaSyncEndFd(frame->dma_fd); - } - - if (!ok) { - src_dma_buf.reset(); - return false; - } - - // Update strides used for RGA to match the copied buffer. - src_wstride = copy_wstride; - src_hstride = copy_hstride; - return true; - }; - - // If there's no DMA fd, we must allocate/copy into a DMA-BUF for RGA. - if (frame->dma_fd < 0) { - if (!CopySrcToDmaIfPossible()) { - LogError("[preprocess] no dma_fd and src copy failed"); - return false; - } - } - - auto RunRgaOnce = [&](int src_fd, std::string& err) -> bool { - // Serialize/limit librga/im2d usage; multiple pipelines call RGA concurrently. - ScopedRgaGate guard(rga_gate_); - - src_buf = wrapbuffer_fd_t(src_fd, frame->width, frame->height, - src_wstride, src_hstride, src_fmt_rga); - dst_buf = wrapbuffer_fd_t(dma_buf->fd, out_w, out_h, - dst_wstride, dst_hstride, dst_fmt_rga); - - auto Check = [&](const rga_buffer_t& s, const rga_buffer_t& d) -> bool { - const im_rect sr{0, 0, s.width, s.height}; - const im_rect dr{0, 0, d.width, d.height}; - // Do NOT call the imcheck(...) variadic macro with 0 extra args: - // under -Wpedantic/-std=c++11 it expands to a zero-sized array. - rga_buffer_t pat{}; - const im_rect pr{0, 0, 0, 0}; - const IM_STATUS chk = imcheck_t(s, d, pat, sr, dr, pr, 0); - if (chk != IM_STATUS_NOERROR && chk != IM_STATUS_SUCCESS) { - err = std::string("RGA imcheck failed: ") + imStrError(chk); - return false; - } - return true; - }; - - IM_STATUS status = IM_STATUS_SUCCESS; - - if (need_resize && need_cvt) { - // Try to fuse resize + CSC in a single call. - if (Check(src_buf, dst_buf)) { - rga_buffer_t pat{}; - const im_rect sr{0, 0, src_buf.width, src_buf.height}; - const im_rect dr{0, 0, dst_buf.width, dst_buf.height}; - const im_rect pr{0, 0, 0, 0}; - status = improcess(src_buf, dst_buf, pat, sr, dr, pr, - 0, nullptr, nullptr, IM_SYNC); - if (status == IM_STATUS_SUCCESS) { - return true; - } - } - - // Fallback: 2-step (resize + cvtcolor). - if (!tmp_dma || !tmp_dma->valid()) { - tmp_dma = DmaAlloc(CalcImageSizeStrided(dst_wstride, dst_hstride, frame->format)); - if (!tmp_dma || !tmp_dma->valid()) { - err = "DMA alloc for tmp failed"; - return false; - } - } - - rga_buffer_t tmp = wrapbuffer_fd_t(tmp_dma->fd, out_w, out_h, - dst_wstride, dst_hstride, src_fmt_rga); - if (!Check(src_buf, tmp) || !Check(tmp, dst_buf)) { - return false; - } - status = imresize(src_buf, tmp, 0, 0, 0, 1, nullptr); - if (status == IM_STATUS_SUCCESS) { - status = imcvtcolor(tmp, dst_buf, src_fmt_rga, dst_fmt_rga, IM_COLOR_SPACE_DEFAULT, 1, nullptr); - } - } else if (need_resize) { - if (!Check(src_buf, dst_buf)) { - return false; - } - status = imresize(src_buf, dst_buf, 0, 0, 0, 1, nullptr); - } else if (need_cvt) { - if (!Check(src_buf, dst_buf)) { - return false; - } - status = imcvtcolor(src_buf, dst_buf, src_fmt_rga, dst_fmt_rga, IM_COLOR_SPACE_DEFAULT, 1, nullptr); - } - - if (status != IM_STATUS_SUCCESS) { - err = std::string("RGA failed: ") + imStrError(status); - return false; - } - return true; - }; - - std::string rga_err; - - const int src_fd = (src_dma_buf && src_dma_buf->valid()) ? src_dma_buf->fd : frame->dma_fd; - if (src_fd < 0 || !RunRgaOnce(src_fd, rga_err)) { - LogError(std::string("[preprocess] ") + (rga_err.empty() ? "RGA failed" : rga_err)); - return false; - } - - auto out_frame = std::make_shared(); - out_frame->width = out_w; - out_frame->height = out_h; - out_frame->format = out_fmt; - out_frame->stride = (out_fmt == PixelFormat::RGB || out_fmt == PixelFormat::BGR) - ? (dst_wstride * 3) - : dst_wstride; - out_frame->dma_fd = dma_buf->fd; - out_frame->data = dma_buf->data(); - out_frame->data_size = dma_buf->size; - out_frame->data_owner = dma_buf; // DmaBuffer shared_ptr keeps fd alive - out_frame->pts = frame->pts; - out_frame->frame_id = frame->frame_id; - out_frame->det = frame->det; - out_frame->face_det = frame->face_det; - out_frame->face_recog = frame->face_recog; - out_frame->user_meta = frame->user_meta; - - SetupPlanes(*out_frame, out_fmt); - PushToDownstream(out_frame); - ++processed_; - - if (stats_log_ && stats_interval_ > 0 && (processed_ % stats_interval_) == 0) { - LogInfo("[preprocess] rga frame=" + std::to_string(out_frame->frame_id) + - " " + std::to_string(frame->width) + "x" + std::to_string(frame->height) + - " -> " + std::to_string(out_w) + "x" + std::to_string(out_h)); - } - - return true; - } -#endif - -#if defined(RK3588_ENABLE_FFMPEG) - void ProcessSwscale(FramePtr frame) { - PixelFormat out_fmt = (dst_fmt_ != PixelFormat::UNKNOWN) ? dst_fmt_ : frame->format; - int out_w = dst_w_; - int out_h = dst_h_; - - if (out_w <= 0) out_w = frame->width; - if (out_h <= 0) out_h = frame->height; - - if (keep_ratio_ && dst_w_ > 0 && dst_h_ > 0 && frame->width > 0 && frame->height > 0) { - float scale = std::min(static_cast(dst_w_) / frame->width, - static_cast(dst_h_) / frame->height); - out_w = static_cast(frame->width * scale); - out_h = static_cast(frame->height * scale); - out_w = (out_w + 1) & ~1; - out_h = (out_h + 1) & ~1; - } - - if (frame->width != out_w || frame->height != out_h) { - WarnMetaResizeOnce(frame, out_w, out_h); - } - - AVPixelFormat src_av_fmt = ToAvFormat(frame->format); - AVPixelFormat dst_av_fmt = ToAvFormat(out_fmt); - - if (src_av_fmt == AV_PIX_FMT_NONE || dst_av_fmt == AV_PIX_FMT_NONE) { - PushToDownstream(frame); - return; - } - - if (!sws_ctx_ || frame->width != last_src_w_ || frame->height != last_src_h_ || - src_av_fmt != last_src_fmt_ || dst_av_fmt != last_dst_fmt_) { - if (sws_ctx_) sws_freeContext(sws_ctx_); - sws_ctx_ = sws_getContext(frame->width, frame->height, src_av_fmt, - out_w, out_h, dst_av_fmt, - SWS_BILINEAR, nullptr, nullptr, nullptr); - last_src_w_ = frame->width; - last_src_h_ = frame->height; - last_src_fmt_ = src_av_fmt; - last_dst_fmt_ = dst_av_fmt; - } - - if (!sws_ctx_) { - PushToDownstream(frame); - return; - } - - size_t out_size = CalcImageSize(out_w, out_h, out_fmt); - auto buffer = std::make_shared>(out_size); - - uint8_t* src_data[4] = {nullptr}; - int src_linesize[4] = {0}; - uint8_t* dst_data[4] = {nullptr}; - int dst_linesize[4] = {0}; - - SetupAvPlanes(frame.get(), src_data, src_linesize); - av_image_fill_arrays(dst_data, dst_linesize, buffer->data(), - dst_av_fmt, out_w, out_h, 1); - - sws_scale(sws_ctx_, src_data, src_linesize, 0, frame->height, - dst_data, dst_linesize); - - auto out_frame = std::make_shared(); - out_frame->width = out_w; - out_frame->height = out_h; - out_frame->format = out_fmt; - out_frame->stride = (out_fmt == PixelFormat::RGB || out_fmt == PixelFormat::BGR) - ? (out_w * 3) - : out_w; - out_frame->data = buffer->data(); - out_frame->data_size = buffer->size(); - out_frame->data_owner = buffer; - out_frame->pts = frame->pts; - out_frame->frame_id = frame->frame_id; - out_frame->det = frame->det; - out_frame->face_det = frame->face_det; - out_frame->face_recog = frame->face_recog; - out_frame->user_meta = frame->user_meta; - - SetupPlanes(*out_frame, out_fmt); - PushToDownstream(out_frame); - ++processed_; - - if (stats_log_ && stats_interval_ > 0 && (processed_ % stats_interval_) == 0) { - LogInfo("[preprocess] swscale frame=" + std::to_string(out_frame->frame_id) + - " " + std::to_string(frame->width) + "x" + std::to_string(frame->height) + - " -> " + std::to_string(out_w) + "x" + std::to_string(out_h) + - " id=" + id_); - } - } - - static AVPixelFormat ToAvFormat(PixelFormat fmt) { - switch (fmt) { - case PixelFormat::NV12: return AV_PIX_FMT_NV12; - case PixelFormat::YUV420: return AV_PIX_FMT_YUV420P; - case PixelFormat::RGB: return AV_PIX_FMT_RGB24; - case PixelFormat::BGR: return AV_PIX_FMT_BGR24; - default: return AV_PIX_FMT_NONE; - } - } - - static void SetupAvPlanes(const Frame* f, uint8_t* data[4], int linesize[4]) { - if (!f->data) return; - if (f->format == PixelFormat::NV12) { - data[0] = f->planes[0].data ? f->planes[0].data : f->data; - data[1] = f->planes[1].data ? f->planes[1].data : (f->data + f->width * f->height); - linesize[0] = f->planes[0].stride > 0 ? f->planes[0].stride : f->width; - linesize[1] = f->planes[1].stride > 0 ? f->planes[1].stride : f->width; - } else if (f->format == PixelFormat::YUV420) { - data[0] = f->planes[0].data ? f->planes[0].data : f->data; - int y_size = f->width * f->height; - int uv_size = y_size / 4; - data[1] = f->planes[1].data ? f->planes[1].data : (f->data + y_size); - data[2] = f->planes[2].data ? f->planes[2].data : (f->data + y_size + uv_size); - linesize[0] = f->planes[0].stride > 0 ? f->planes[0].stride : f->width; - linesize[1] = f->planes[1].stride > 0 ? f->planes[1].stride : f->width / 2; - linesize[2] = f->planes[2].stride > 0 ? f->planes[2].stride : f->width / 2; - } else { - data[0] = f->data; - linesize[0] = f->stride > 0 ? f->stride : f->width * 3; - } - } -#endif - - void SetupPlanes(Frame& f, PixelFormat fmt) { - if (!f.data) return; - - if (fmt == PixelFormat::NV12) { - f.plane_count = 2; - int y_stride = f.stride > 0 ? f.stride : f.width; - if (y_stride <= 0) y_stride = f.width; - - size_t y_bytes = static_cast(y_stride) * static_cast(f.height); - if (f.data_size > 0) { - const size_t candidate = (f.data_size * 2) / 3; // total = Y + UV = Y*3/2 - if (candidate >= y_bytes && candidate <= f.data_size && - (candidate % static_cast(y_stride)) == 0) { - y_bytes = candidate; - } - } - - size_t uv_bytes = y_bytes / 2; - if (f.data_size > 0 && y_bytes + uv_bytes > f.data_size) { - uv_bytes = f.data_size > y_bytes ? (f.data_size - y_bytes) : 0; - } - - f.planes[0] = {f.data, y_stride, static_cast(y_bytes), 0}; - f.planes[1] = {f.data + y_bytes, y_stride, static_cast(uv_bytes), static_cast(y_bytes)}; - return; - } - - if (fmt == PixelFormat::YUV420) { - f.plane_count = 3; - int y_stride = f.stride > 0 ? f.stride : f.width; - if (y_stride <= 0) y_stride = f.width; - - size_t y_bytes = static_cast(y_stride) * static_cast(f.height); - size_t hstride = static_cast(f.height); - if (f.data_size > 0) { - const size_t candidate = (f.data_size * 2) / 3; - if (candidate >= y_bytes && candidate <= f.data_size && - (candidate % static_cast(y_stride)) == 0) { - y_bytes = candidate; - hstride = y_bytes / static_cast(y_stride); - } - } - - size_t uv_stride = static_cast(y_stride) / 2; - size_t uv_h = hstride / 2; - size_t u_bytes = uv_stride * uv_h; - size_t v_bytes = u_bytes; - size_t need = y_bytes + u_bytes + v_bytes; - - if (f.data_size > 0 && need > f.data_size) { - // Fallback to tightly packed layout. - y_stride = f.width; - y_bytes = static_cast(f.width) * static_cast(f.height); - uv_stride = static_cast(f.width) / 2; - uv_h = static_cast(f.height) / 2; - u_bytes = uv_stride * uv_h; - v_bytes = u_bytes; - } - - f.planes[0] = {f.data, y_stride, static_cast(y_bytes), 0}; - f.planes[1] = {f.data + y_bytes, static_cast(uv_stride), static_cast(u_bytes), static_cast(y_bytes)}; - f.planes[2] = {f.data + y_bytes + u_bytes, static_cast(uv_stride), static_cast(v_bytes), static_cast(y_bytes + u_bytes)}; - return; - } - - // RGB/BGR - f.plane_count = 1; - int stride_bytes = f.stride > 0 ? f.stride : (f.width * 3); - f.planes[0] = {f.data, stride_bytes, static_cast(f.data_size), 0}; - } - std::string id_; int dst_w_ = 640; int dst_h_ = 640; bool keep_ratio_ = false; PixelFormat dst_fmt_ = PixelFormat::UNKNOWN; - bool dst_packed_ = false; - bool dst_packed_explicit_ = false; bool use_rga_ = true; - std::string rga_gate_ = "global"; - bool stats_log_ = false; uint64_t stats_interval_ = 100; @@ -929,15 +196,8 @@ private: std::shared_ptr> input_queue_; std::vector>> output_queues_; + std::shared_ptr image_processor_; uint64_t processed_ = 0; - -#if defined(RK3588_ENABLE_FFMPEG) - SwsContext* sws_ctx_ = nullptr; - int last_src_w_ = 0; - int last_src_h_ = 0; - AVPixelFormat last_src_fmt_ = AV_PIX_FMT_NONE; - AVPixelFormat last_dst_fmt_ = AV_PIX_FMT_NONE; -#endif }; REGISTER_NODE(PreprocessNode, "preprocess"); diff --git a/src/graph_manager.cpp b/src/graph_manager.cpp index 62eb54c..f28268a 100644 --- a/src/graph_manager.cpp +++ b/src/graph_manager.cpp @@ -22,6 +22,7 @@ namespace fs = std::filesystem; #include "utils/logger.h" #include "utils/simple_json_writer.h" #include "utils/thread_affinity.h" +#include "hw/hw_factory.h" namespace rk3588 { @@ -472,6 +473,8 @@ bool Graph::Build(const SimpleJson& graph_cfg, PluginLoader& loader, size_t defa name_ = name_it->second.AsString(name_); } + infer_backend_ = HwFactory::CreateInferBackend(graph_cfg_); + // Parse nodes auto nodes_it = obj.find("nodes"); if (nodes_it == obj.end() || !nodes_it->second.IsArray()) { @@ -491,6 +494,10 @@ bool Graph::Build(const SimpleJson& graph_cfg, PluginLoader& loader, size_t defa entry.role = node_val.ValueOr("role", ""); entry.enabled = node_val.ValueOr("enable", true); entry.metrics = std::make_shared(); + entry.context.infer_backend = infer_backend_; + if (entry.type == "preprocess") { + entry.context.image_processor = HwFactory::CreateImageProcessor(entry.config); + } if (entry.id.empty() || entry.type.empty()) { err = "Node missing id or type"; diff --git a/src/hw_factory.cpp b/src/hw_factory.cpp index 6a95bee..9d4199d 100644 --- a/src/hw_factory.cpp +++ b/src/hw_factory.cpp @@ -8,8 +8,8 @@ std::shared_ptr HwFactory::CreateInferBackend(const SimpleJson& / return std::make_shared(); } -std::shared_ptr HwFactory::CreateImageProcessor(const SimpleJson& /*config*/) { - return std::make_shared(); +std::shared_ptr HwFactory::CreateImageProcessor(const SimpleJson& config) { + return std::make_shared(config); } std::shared_ptr HwFactory::CreateDecoder(const SimpleJson& /*config*/) { diff --git a/src/hw_image_processor.cpp b/src/hw_image_processor.cpp new file mode 100644 index 0000000..136e349 --- /dev/null +++ b/src/hw_image_processor.cpp @@ -0,0 +1,786 @@ +#include "hw/rk3588_defaults.h" +#include "hw/rk3588_defaults.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "utils/dma_alloc.h" +#include "utils/logger.h" + +#if defined(RK3588_ENABLE_RGA) +#include "im2d.hpp" +#include "im2d_buffer.h" +#include "im2d_type.h" +#endif + +#if defined(RK3588_ENABLE_FFMPEG) +extern "C" { +#include +#include +} +#endif + +namespace rk3588 { +namespace { + +inline int Align16(int v) { return (v + 15) & ~15; } + +size_t CalcImageSize(int w, int h, PixelFormat fmt) { + switch (fmt) { + case PixelFormat::NV12: + case PixelFormat::YUV420: + return static_cast(w) * h * 3 / 2; + case PixelFormat::RGB: + case PixelFormat::BGR: + return static_cast(w) * h * 3; + default: + return 0; + } +} + +size_t CalcImageSizeStrided(int wstride, int hstride, PixelFormat fmt) { + if (wstride <= 0 || hstride <= 0) return 0; + const size_t ws = static_cast(wstride); + const size_t hs = static_cast(hstride); + switch (fmt) { + case PixelFormat::NV12: { + return ws * hs + ws * (hs / 2); + } + case PixelFormat::YUV420: { + const size_t y = ws * hs; + const size_t uv = (ws / 2) * (hs / 2); + return y + uv + uv; + } + case PixelFormat::RGB: + case PixelFormat::BGR: + return ws * hs * 3; + default: + return 0; + } +} + +void SetupPlanes(Frame& f, PixelFormat fmt) { + if (!f.data) return; + + if (fmt == PixelFormat::NV12) { + f.plane_count = 2; + int y_stride = f.stride > 0 ? f.stride : f.width; + if (y_stride <= 0) y_stride = f.width; + + size_t y_bytes = static_cast(y_stride) * static_cast(f.height); + if (f.data_size > 0) { + const size_t candidate = (f.data_size * 2) / 3; + if (candidate >= y_bytes && candidate <= f.data_size && + (candidate % static_cast(y_stride)) == 0) { + y_bytes = candidate; + } + } + + size_t uv_bytes = y_bytes / 2; + if (f.data_size > 0 && y_bytes + uv_bytes > f.data_size) { + uv_bytes = f.data_size > y_bytes ? (f.data_size - y_bytes) : 0; + } + + f.planes[0] = {f.data, y_stride, static_cast(y_bytes), 0}; + f.planes[1] = {f.data + y_bytes, y_stride, static_cast(uv_bytes), static_cast(y_bytes)}; + return; + } + + if (fmt == PixelFormat::YUV420) { + f.plane_count = 3; + int y_stride = f.stride > 0 ? f.stride : f.width; + if (y_stride <= 0) y_stride = f.width; + + size_t y_bytes = static_cast(y_stride) * static_cast(f.height); + size_t hstride = static_cast(f.height); + if (f.data_size > 0) { + const size_t candidate = (f.data_size * 2) / 3; + if (candidate >= y_bytes && candidate <= f.data_size && + (candidate % static_cast(y_stride)) == 0) { + y_bytes = candidate; + hstride = y_bytes / static_cast(y_stride); + } + } + + size_t uv_stride = static_cast(y_stride) / 2; + size_t uv_h = hstride / 2; + size_t u_bytes = uv_stride * uv_h; + size_t v_bytes = u_bytes; + size_t need = y_bytes + u_bytes + v_bytes; + + if (f.data_size > 0 && need > f.data_size) { + y_stride = f.width; + y_bytes = static_cast(f.width) * static_cast(f.height); + uv_stride = static_cast(f.width) / 2; + uv_h = static_cast(f.height) / 2; + u_bytes = uv_stride * uv_h; + v_bytes = u_bytes; + } + + f.planes[0] = {f.data, y_stride, static_cast(y_bytes), 0}; + f.planes[1] = {f.data + y_bytes, static_cast(uv_stride), static_cast(u_bytes), static_cast(y_bytes)}; + f.planes[2] = {f.data + y_bytes + u_bytes, static_cast(uv_stride), static_cast(v_bytes), static_cast(y_bytes + u_bytes)}; + return; + } + + f.plane_count = 1; + int stride_bytes = f.stride > 0 ? f.stride : (f.width * 3); + f.planes[0] = {f.data, stride_bytes, static_cast(f.data_size), 0}; +} + +#if defined(RK3588_ENABLE_RGA) +int ToRgaFormat(PixelFormat fmt) { + switch (fmt) { + case PixelFormat::NV12: return RK_FORMAT_YCbCr_420_SP; + case PixelFormat::YUV420: return RK_FORMAT_YCbCr_420_P; + case PixelFormat::RGB: return RK_FORMAT_RGB_888; + case PixelFormat::BGR: return RK_FORMAT_BGR_888; + default: return RK_FORMAT_UNKNOWN; + } +} + +std::atomic& GlobalRgaMaxInflightRef() { + static std::atomic v{2}; + return v; +} + +int GlobalRgaMaxInflight() { + static std::once_flag once; + std::call_once(once, []() { + const char* s = std::getenv("RK3588_RGA_MAX_INFLIGHT"); + if (!s || !*s) return; + try { + int v = std::stoi(s); + if (v > 0 && v <= 32) GlobalRgaMaxInflightRef().store(v); + } catch (...) { + LogWarn(std::string("[image_processor] invalid RK3588_RGA_MAX_INFLIGHT='") + s + "'"); + } + }); + int v = GlobalRgaMaxInflightRef().load(); + return v > 0 ? v : 1; +} + +class RgaGate { +public: + explicit RgaGate(int max_inflight) : max_inflight_(max_inflight > 0 ? max_inflight : 1) {} + + void Acquire() { + std::unique_lock lock(mu_); + cv_.wait(lock, [&]() { return in_flight_ < max_inflight_; }); + ++in_flight_; + } + + void Release() { + std::lock_guard lock(mu_); + if (in_flight_ > 0) { + --in_flight_; + } + cv_.notify_one(); + } + + void SetMaxInflight(int v) { + if (v <= 0) return; + if (v > 32) v = 32; + { + std::lock_guard lock(mu_); + max_inflight_ = v; + } + cv_.notify_all(); + } + + int MaxInflight() const { + std::lock_guard lock(mu_); + return max_inflight_; + } + +private: + mutable std::mutex mu_; + std::condition_variable cv_; + int in_flight_ = 0; + int max_inflight_ = 1; +}; + +class RgaGateRegistry { +public: + static RgaGateRegistry& Instance() { + static RgaGateRegistry* inst = new RgaGateRegistry(); + return *inst; + } + + RgaGate& Get(const std::string& key) { + std::lock_guard lock(mu_); + auto it = gates_.find(key); + if (it != gates_.end()) return *it->second; + auto gate = std::make_unique(GlobalRgaMaxInflight()); + RgaGate& ref = *gate; + gates_.emplace(key, std::move(gate)); + return ref; + } + +private: + std::mutex mu_; + std::unordered_map> gates_; +}; + +RgaGate& GetRgaGate(const std::string& key) { + const std::string k = key.empty() ? "global" : key; + return RgaGateRegistry::Instance().Get(k); +} + +class ScopedRgaGate { +public: + explicit ScopedRgaGate(const std::string& key) : gate_(&GetRgaGate(key)) { gate_->Acquire(); } + ~ScopedRgaGate() { gate_->Release(); } + ScopedRgaGate(const ScopedRgaGate&) = delete; + ScopedRgaGate& operator=(const ScopedRgaGate&) = delete; + +private: + RgaGate* gate_ = nullptr; +}; + +void EnsureRgaInitializedOnce() { + static std::once_flag once; + std::call_once(once, []() { + const IM_STATUS st = imcheckHeader(); + if (st != IM_STATUS_NOERROR && st != IM_STATUS_SUCCESS) { + LogWarn(std::string("[image_processor] imcheckHeader failed: ") + imStrError(st)); + } + }); +} + +bool CopyToStridedBuffer(const Frame& src, uint8_t* dst, size_t dst_size, + int dst_wstride, int dst_hstride) { + if (!dst || dst_size == 0) return false; + if (dst_wstride <= 0 || dst_hstride <= 0) return false; + + std::memset(dst, 0, dst_size); + + const int w = src.width; + const int h = src.height; + if (w <= 0 || h <= 0) return false; + + if (src.format == PixelFormat::NV12) { + const size_t y_bytes = static_cast(dst_wstride) * dst_hstride; + const size_t uv_bytes = static_cast(dst_wstride) * (dst_hstride / 2); + if (y_bytes + uv_bytes > dst_size) return false; + + const uint8_t* src_y = src.planes[0].data ? src.planes[0].data : src.data; + const uint8_t* src_uv = src.planes[1].data ? src.planes[1].data : nullptr; + const int src_y_stride = src.planes[0].stride > 0 ? src.planes[0].stride : w; + const int src_uv_stride = src.planes[1].stride > 0 ? src.planes[1].stride : w; + if (!src_y) return false; + if (!src_uv) { + if (!src.data) return false; + src_uv = src.data + static_cast(src_y_stride) * static_cast(h); + } + + for (int row = 0; row < h; ++row) { + std::memcpy(dst + static_cast(row) * dst_wstride, + src_y + static_cast(row) * src_y_stride, + static_cast(w)); + } + + uint8_t* dst_uv = dst + y_bytes; + const int uv_rows = h / 2; + for (int row = 0; row < uv_rows; ++row) { + std::memcpy(dst_uv + static_cast(row) * dst_wstride, + src_uv + static_cast(row) * src_uv_stride, + static_cast(w)); + } + return true; + } + + if (src.format == PixelFormat::YUV420) { + const size_t y_bytes = static_cast(dst_wstride) * dst_hstride; + const size_t uv_stride = static_cast(dst_wstride) / 2; + const size_t uv_h = static_cast(dst_hstride) / 2; + const size_t u_bytes = uv_stride * uv_h; + const size_t v_bytes = u_bytes; + if (y_bytes + u_bytes + v_bytes > dst_size) return false; + + const uint8_t* src_y = src.planes[0].data ? src.planes[0].data : src.data; + const uint8_t* src_u = src.planes[1].data ? src.planes[1].data : nullptr; + const uint8_t* src_v = src.planes[2].data ? src.planes[2].data : nullptr; + const int src_y_stride = src.planes[0].stride > 0 ? src.planes[0].stride : w; + const int src_u_stride = src.planes[1].stride > 0 ? src.planes[1].stride : (w / 2); + const int src_v_stride = src.planes[2].stride > 0 ? src.planes[2].stride : (w / 2); + if (!src_y || !src_u || !src_v) return false; + + for (int row = 0; row < h; ++row) { + std::memcpy(dst + static_cast(row) * dst_wstride, + src_y + static_cast(row) * src_y_stride, + static_cast(w)); + } + + uint8_t* dst_u = dst + y_bytes; + uint8_t* dst_v = dst + y_bytes + u_bytes; + const int uv_rows = h / 2; + const int uv_cols = w / 2; + for (int row = 0; row < uv_rows; ++row) { + std::memcpy(dst_u + static_cast(row) * uv_stride, + src_u + static_cast(row) * src_u_stride, + static_cast(uv_cols)); + std::memcpy(dst_v + static_cast(row) * uv_stride, + src_v + static_cast(row) * src_v_stride, + static_cast(uv_cols)); + } + return true; + } + + if (src.format == PixelFormat::RGB || src.format == PixelFormat::BGR) { + const size_t need = static_cast(dst_wstride) * dst_hstride * 3; + if (need > dst_size) return false; + + const uint8_t* src_rgb = src.planes[0].data ? src.planes[0].data : src.data; + const int src_stride = src.planes[0].stride > 0 + ? src.planes[0].stride + : (src.stride > 0 ? src.stride : w * 3); + if (!src_rgb) return false; + + const size_t dst_stride = static_cast(dst_wstride) * 3; + const size_t row_bytes = static_cast(w) * 3; + for (int row = 0; row < h; ++row) { + std::memcpy(dst + static_cast(row) * dst_stride, + src_rgb + static_cast(row) * src_stride, + row_bytes); + } + return true; + } + + return false; +} + +class RgaImageProcessor : public IImageProcessor { +public: + explicit RgaImageProcessor(const SimpleJson& config) { + rga_gate_ = config.ValueOr("rga_gate", "global"); + const int rga_max_inflight = config.ValueOr("rga_max_inflight", 0); + if (rga_max_inflight > 0) { + GetRgaGate(rga_gate_).SetMaxInflight(rga_max_inflight); + } + + if (config.Find("dst_packed")) { + dst_packed_ = config.ValueOr("dst_packed", false); + dst_packed_explicit_ = true; + } else { + dst_packed_ = false; + dst_packed_explicit_ = false; + } + } + + Status Resize(const Frame& src, Frame& dst) override { + EnsureRgaInitializedOnce(); + + if (dst.width <= 0 || dst.height <= 0) return FailStatus("invalid output size"); + if (dst.format == PixelFormat::UNKNOWN) return FailStatus("invalid output format"); + + const PixelFormat out_fmt = dst.format; + const int out_w = dst.width; + const int out_h = dst.height; + + int src_fmt_rga = ToRgaFormat(src.format); + int dst_fmt_rga = ToRgaFormat(out_fmt); + bool need_cvt = (src_fmt_rga != dst_fmt_rga); + bool need_resize = (src.width != out_w || src.height != out_h); + + if (src_fmt_rga == RK_FORMAT_UNKNOWN || dst_fmt_rga == RK_FORMAT_UNKNOWN) { + return FailStatus("unsupported format for RGA"); + } + + int src_wstride = Align16(src.width); + int src_hstride = Align16(src.height); + if (src.format == PixelFormat::NV12 || src.format == PixelFormat::YUV420) { + const int y_stride = src.planes[0].stride > 0 ? src.planes[0].stride + : (src.stride > 0 ? src.stride : src.width); + if (y_stride > 0) src_wstride = y_stride; + if (src.planes[0].size > 0 && y_stride > 0) { + const int hs = src.planes[0].size / y_stride; + if (hs >= src.height) src_hstride = hs; + } + } else if (src.format == PixelFormat::RGB || src.format == PixelFormat::BGR) { + const int stride_bytes = src.planes[0].stride > 0 ? src.planes[0].stride + : (src.stride > 0 ? src.stride : src.width * 3); + if (stride_bytes > 0 && (stride_bytes % 3) == 0) { + src_wstride = stride_bytes / 3; + } + if (src.planes[0].size > 0 && stride_bytes > 0) { + const int hs = src.planes[0].size / stride_bytes; + if (hs >= src.height) src_hstride = hs; + } + } + + int dst_wstride = Align16(out_w); + const bool want_packed_rgb = (out_fmt == PixelFormat::RGB || out_fmt == PixelFormat::BGR) && + (!dst_packed_explicit_ || dst_packed_); + if (want_packed_rgb) { + dst_wstride = out_w; + } + int dst_hstride = Align16(out_h); + + size_t out_size = CalcImageSizeStrided(dst_wstride, dst_hstride, out_fmt); + if (out_size == 0) { + return FailStatus("invalid output size for RGA"); + } + + auto dma_buf = DmaAlloc(out_size); + if (!dma_buf || !dma_buf->valid()) { + return FailStatus("DMA alloc failed"); + } + + rga_buffer_t src_buf{}; + rga_buffer_t dst_buf{}; + DmaBufferPtr src_dma_buf; + DmaBufferPtr tmp_dma; + + const bool can_cpu_read_src = (src.data != nullptr && src.data_size > 0); + auto CopySrcToDmaIfPossible = [&]() -> bool { + if (!can_cpu_read_src) return false; + const int copy_wstride = Align16(src.width); + const int copy_hstride = Align16(src.height); + const size_t src_size = CalcImageSizeStrided(copy_wstride, copy_hstride, src.format); + src_dma_buf = DmaAlloc(src_size); + if (!src_dma_buf || !src_dma_buf->valid()) { + return false; + } + + if (src.dma_fd >= 0) { + DmaSyncStartFd(src.dma_fd); + } + + DmaSyncStartFd(src_dma_buf->fd); + const bool ok = CopyToStridedBuffer(src, src_dma_buf->data(), src_dma_buf->size, + copy_wstride, copy_hstride); + DmaSyncEndFd(src_dma_buf->fd); + + if (src.dma_fd >= 0) { + DmaSyncEndFd(src.dma_fd); + } + + if (!ok) { + src_dma_buf.reset(); + return false; + } + + src_wstride = copy_wstride; + src_hstride = copy_hstride; + return true; + }; + + if (src.dma_fd < 0) { + if (!CopySrcToDmaIfPossible()) { + return FailStatus("no dma_fd and src copy failed"); + } + } + + auto RunRgaOnce = [&](int src_fd, std::string& err) -> bool { + ScopedRgaGate guard(rga_gate_); + + src_buf = wrapbuffer_fd_t(src_fd, src.width, src.height, + src_wstride, src_hstride, src_fmt_rga); + dst_buf = wrapbuffer_fd_t(dma_buf->fd, out_w, out_h, + dst_wstride, dst_hstride, dst_fmt_rga); + + auto Check = [&](const rga_buffer_t& s, const rga_buffer_t& d) -> bool { + const im_rect sr{0, 0, s.width, s.height}; + const im_rect dr{0, 0, d.width, d.height}; + rga_buffer_t pat{}; + const im_rect pr{0, 0, 0, 0}; + const IM_STATUS chk = imcheck_t(s, d, pat, sr, dr, pr, 0); + if (chk != IM_STATUS_NOERROR && chk != IM_STATUS_SUCCESS) { + err = std::string("RGA imcheck failed: ") + imStrError(chk); + return false; + } + return true; + }; + + IM_STATUS status = IM_STATUS_SUCCESS; + + if (need_resize && need_cvt) { + if (Check(src_buf, dst_buf)) { + rga_buffer_t pat{}; + const im_rect sr{0, 0, src_buf.width, src_buf.height}; + const im_rect dr{0, 0, dst_buf.width, dst_buf.height}; + const im_rect pr{0, 0, 0, 0}; + status = improcess(src_buf, dst_buf, pat, sr, dr, pr, + 0, nullptr, nullptr, IM_SYNC); + if (status == IM_STATUS_SUCCESS) { + return true; + } + } + + if (!tmp_dma || !tmp_dma->valid()) { + tmp_dma = DmaAlloc(CalcImageSizeStrided(dst_wstride, dst_hstride, src.format)); + if (!tmp_dma || !tmp_dma->valid()) { + err = "DMA alloc for tmp failed"; + return false; + } + } + + rga_buffer_t tmp = wrapbuffer_fd_t(tmp_dma->fd, out_w, out_h, + dst_wstride, dst_hstride, src_fmt_rga); + if (!Check(src_buf, tmp) || !Check(tmp, dst_buf)) { + return false; + } + status = imresize(src_buf, tmp, 0, 0, 0, 1, nullptr); + if (status == IM_STATUS_SUCCESS) { + status = imcvtcolor(tmp, dst_buf, src_fmt_rga, dst_fmt_rga, IM_COLOR_SPACE_DEFAULT, 1, nullptr); + } + } else if (need_resize) { + if (!Check(src_buf, dst_buf)) { + return false; + } + status = imresize(src_buf, dst_buf, 0, 0, 0, 1, nullptr); + } else if (need_cvt) { + if (!Check(src_buf, dst_buf)) { + return false; + } + status = imcvtcolor(src_buf, dst_buf, src_fmt_rga, dst_fmt_rga, IM_COLOR_SPACE_DEFAULT, 1, nullptr); + } + + if (status != IM_STATUS_SUCCESS) { + err = std::string("RGA failed: ") + imStrError(status); + return false; + } + return true; + }; + + std::string rga_err; + const int src_fd = (src_dma_buf && src_dma_buf->valid()) ? src_dma_buf->fd : src.dma_fd; + if (src_fd < 0 || !RunRgaOnce(src_fd, rga_err)) { + return FailStatus(rga_err.empty() ? "RGA failed" : rga_err); + } + + dst.stride = (out_fmt == PixelFormat::RGB || out_fmt == PixelFormat::BGR) + ? (dst_wstride * 3) + : dst_wstride; + dst.dma_fd = dma_buf->fd; + dst.data = dma_buf->data(); + dst.data_size = dma_buf->size; + dst.data_owner = dma_buf; + SetupPlanes(dst, out_fmt); + return OkStatus(); + } + + Status CvtColor(const Frame& src, Frame& dst, PixelFormat dst_format) override { + dst.width = src.width; + dst.height = src.height; + dst.format = dst_format; + return Resize(src, dst); + } + + Status Normalize(const Frame& /*src*/, std::vector& /*out*/, + const std::vector& /*mean*/, + const std::vector& /*std*/) override { + return FailStatus("not implemented"); + } + + int MaxInflight() const { return GetRgaGate(rga_gate_).MaxInflight(); } + const std::string& GateKey() const { return rga_gate_; } + +private: + std::string rga_gate_ = "global"; + bool dst_packed_ = false; + bool dst_packed_explicit_ = false; +}; +#endif + +#if defined(RK3588_ENABLE_FFMPEG) +AVPixelFormat ToAvFormat(PixelFormat fmt) { + switch (fmt) { + case PixelFormat::NV12: return AV_PIX_FMT_NV12; + case PixelFormat::YUV420: return AV_PIX_FMT_YUV420P; + case PixelFormat::RGB: return AV_PIX_FMT_RGB24; + case PixelFormat::BGR: return AV_PIX_FMT_BGR24; + default: return AV_PIX_FMT_NONE; + } +} + +void SetupAvPlanes(const Frame* f, uint8_t* data[4], int linesize[4]) { + if (!f || !f->data) return; + if (f->format == PixelFormat::NV12) { + data[0] = f->planes[0].data ? f->planes[0].data : f->data; + data[1] = f->planes[1].data ? f->planes[1].data : (f->data + f->width * f->height); + linesize[0] = f->planes[0].stride > 0 ? f->planes[0].stride : f->width; + linesize[1] = f->planes[1].stride > 0 ? f->planes[1].stride : f->width; + } else if (f->format == PixelFormat::YUV420) { + data[0] = f->planes[0].data ? f->planes[0].data : f->data; + int y_size = f->width * f->height; + int uv_size = y_size / 4; + data[1] = f->planes[1].data ? f->planes[1].data : (f->data + y_size); + data[2] = f->planes[2].data ? f->planes[2].data : (f->data + y_size + uv_size); + linesize[0] = f->planes[0].stride > 0 ? f->planes[0].stride : f->width; + linesize[1] = f->planes[1].stride > 0 ? f->planes[1].stride : f->width / 2; + linesize[2] = f->planes[2].stride > 0 ? f->planes[2].stride : f->width / 2; + } else { + data[0] = f->data; + linesize[0] = f->stride > 0 ? f->stride : f->width * 3; + } +} + +class SwscaleImageProcessor : public IImageProcessor { +public: + ~SwscaleImageProcessor() override { + if (sws_ctx_) { + sws_freeContext(sws_ctx_); + sws_ctx_ = nullptr; + } + } + + Status Resize(const Frame& src, Frame& dst) override { + if (dst.width <= 0 || dst.height <= 0) return FailStatus("invalid output size"); + if (dst.format == PixelFormat::UNKNOWN) return FailStatus("invalid output format"); + + AVPixelFormat src_av_fmt = ToAvFormat(src.format); + AVPixelFormat dst_av_fmt = ToAvFormat(dst.format); + + if (src_av_fmt == AV_PIX_FMT_NONE || dst_av_fmt == AV_PIX_FMT_NONE) { + return FailStatus("unsupported format"); + } + + if (!sws_ctx_ || src.width != last_src_w_ || src.height != last_src_h_ || + src_av_fmt != last_src_fmt_ || dst_av_fmt != last_dst_fmt_ || + dst.width != last_dst_w_ || dst.height != last_dst_h_) { + if (sws_ctx_) sws_freeContext(sws_ctx_); + sws_ctx_ = sws_getContext(src.width, src.height, src_av_fmt, + dst.width, dst.height, dst_av_fmt, + SWS_BILINEAR, nullptr, nullptr, nullptr); + last_src_w_ = src.width; + last_src_h_ = src.height; + last_src_fmt_ = src_av_fmt; + last_dst_fmt_ = dst_av_fmt; + last_dst_w_ = dst.width; + last_dst_h_ = dst.height; + } + + if (!sws_ctx_) { + return FailStatus("sws_getContext failed"); + } + + size_t out_size = CalcImageSize(dst.width, dst.height, dst.format); + if (out_size == 0) return FailStatus("invalid output size"); + auto buffer = std::make_shared>(out_size); + + uint8_t* src_data[4] = {nullptr}; + int src_linesize[4] = {0}; + uint8_t* dst_data[4] = {nullptr}; + int dst_linesize[4] = {0}; + + SetupAvPlanes(&src, src_data, src_linesize); + av_image_fill_arrays(dst_data, dst_linesize, buffer->data(), + dst_av_fmt, dst.width, dst.height, 1); + + sws_scale(sws_ctx_, src_data, src_linesize, 0, src.height, + dst_data, dst_linesize); + + dst.stride = (dst.format == PixelFormat::RGB || dst.format == PixelFormat::BGR) + ? (dst.width * 3) + : dst.width; + dst.data = buffer->data(); + dst.data_size = buffer->size(); + dst.data_owner = buffer; + SetupPlanes(dst, dst.format); + return OkStatus(); + } + + Status CvtColor(const Frame& src, Frame& dst, PixelFormat dst_format) override { + dst.width = src.width; + dst.height = src.height; + dst.format = dst_format; + return Resize(src, dst); + } + + Status Normalize(const Frame& /*src*/, std::vector& /*out*/, + const std::vector& /*mean*/, + const std::vector& /*std*/) override { + return FailStatus("not implemented"); + } + +private: + SwsContext* sws_ctx_ = nullptr; + int last_src_w_ = 0; + int last_src_h_ = 0; + int last_dst_w_ = 0; + int last_dst_h_ = 0; + AVPixelFormat last_src_fmt_ = AV_PIX_FMT_NONE; + AVPixelFormat last_dst_fmt_ = AV_PIX_FMT_NONE; +}; +#else +class SwscaleImageProcessor : public IImageProcessor { +public: + Status Resize(const Frame& /*src*/, Frame& /*dst*/) override { + return FailStatus("swscale disabled"); + } + Status CvtColor(const Frame& /*src*/, Frame& /*dst*/, PixelFormat /*dst_format*/) override { + return FailStatus("swscale disabled"); + } + Status Normalize(const Frame& /*src*/, std::vector& /*out*/, + const std::vector& /*mean*/, + const std::vector& /*std*/) override { + return FailStatus("swscale disabled"); + } +}; +#endif + +} // namespace + +Rk3588ImageProcessor::Rk3588ImageProcessor(const SimpleJson& config) { + use_rga_ = config.ValueOr("use_rga", true); + +#if defined(RK3588_ENABLE_RGA) + if (use_rga_) { + impl_ = std::make_shared(config); + } +#endif + + if (!impl_) { + impl_ = std::make_shared(); + } +} + +Status Rk3588ImageProcessor::Resize(const Frame& src, Frame& dst) { + if (!impl_) return FailStatus("no image processor backend"); + return impl_->Resize(src, dst); +} + +Status Rk3588ImageProcessor::CvtColor(const Frame& src, Frame& dst, PixelFormat dst_format) { + if (!impl_) return FailStatus("no image processor backend"); + return impl_->CvtColor(src, dst, dst_format); +} + +Status Rk3588ImageProcessor::Normalize(const Frame& src, std::vector& out, + const std::vector& mean, + const std::vector& std) { + if (!impl_) return FailStatus("no image processor backend"); + return impl_->Normalize(src, out, mean, std); +} + +bool Rk3588ImageProcessor::IsUsingRga() const { return use_rga_; } + +int Rk3588ImageProcessor::RgaMaxInflight() const { +#if defined(RK3588_ENABLE_RGA) + if (!use_rga_) return 0; + auto rga = std::dynamic_pointer_cast(impl_); + if (!rga) return 0; + return rga->MaxInflight(); +#else + return 0; +#endif +} + +std::string Rk3588ImageProcessor::RgaGateKey() const { +#if defined(RK3588_ENABLE_RGA) + if (!use_rga_) return ""; + auto rga = std::dynamic_pointer_cast(impl_); + if (!rga) return ""; + return rga->GateKey(); +#else + return ""; +#endif +} + +} // namespace rk3588 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 795164f..5e58c6a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -36,7 +36,11 @@ add_executable(rk3588_gtests test_simple_json.cpp test_config_expand.cpp test_hw_factory.cpp + test_infer_backend.cpp + test_image_processor.cpp ${CMAKE_SOURCE_DIR}/src/hw_factory.cpp + ${CMAKE_SOURCE_DIR}/src/hw_image_processor.cpp + ${CMAKE_SOURCE_DIR}/src/ai_scheduler.cpp ${CMAKE_SOURCE_DIR}/src/utils/config_expand.cpp ) diff --git a/tests/test_image_processor.cpp b/tests/test_image_processor.cpp new file mode 100644 index 0000000..ddb1da9 --- /dev/null +++ b/tests/test_image_processor.cpp @@ -0,0 +1,94 @@ +#include + +#include + +#include "frame/frame.h" +#include "hw/hw_factory.h" + +namespace rk3588 { +namespace { + +Frame MakeNv12Frame(int w, int h) { + Frame f; + f.width = w; + f.height = h; + f.format = PixelFormat::NV12; + const size_t size = static_cast(w) * h * 3 / 2; + auto buf = std::make_shared>(size, 0); + + const size_t y_bytes = static_cast(w) * h; + for (size_t i = 0; i < y_bytes; ++i) { + (*buf)[i] = 80; + } + for (size_t i = y_bytes; i < size; ++i) { + (*buf)[i] = 128; + } + + f.data = buf->data(); + f.data_size = buf->size(); + f.data_owner = buf; + f.plane_count = 2; + f.planes[0] = {f.data, w, static_cast(y_bytes), 0}; + f.planes[1] = {f.data + y_bytes, w, static_cast(size - y_bytes), static_cast(y_bytes)}; + return f; +} + +TEST(ImageProcessorTest, RgaVsSwscale_OutputShape) { +#if defined(RK3588_ENABLE_RGA) && defined(RK3588_ENABLE_FFMPEG) + SimpleJson rga_cfg(SimpleJson::Object{{"use_rga", true}}); + SimpleJson sw_cfg(SimpleJson::Object{{"use_rga", false}}); + + auto rga = HwFactory::CreateImageProcessor(rga_cfg); + auto sws = HwFactory::CreateImageProcessor(sw_cfg); + + ASSERT_NE(rga, nullptr); + ASSERT_NE(sws, nullptr); + + Frame src = MakeNv12Frame(64, 48); + + Frame dst_rga; + dst_rga.width = 128; + dst_rga.height = 96; + dst_rga.format = PixelFormat::RGB; + EXPECT_TRUE(rga->Resize(src, dst_rga)); + EXPECT_EQ(dst_rga.width, 128); + EXPECT_EQ(dst_rga.height, 96); + + Frame dst_sws; + dst_sws.width = 128; + dst_sws.height = 96; + dst_sws.format = PixelFormat::RGB; + EXPECT_TRUE(sws->Resize(src, dst_sws)); + EXPECT_EQ(dst_sws.width, 128); + EXPECT_EQ(dst_sws.height, 96); +#else + GTEST_SKIP() << "RGA/FFmpeg not enabled"; +#endif +} + +TEST(ImageProcessorTest, ColorConversion_Nv12ToRgb) { +#if defined(RK3588_ENABLE_FFMPEG) + SimpleJson cfg(SimpleJson::Object{{"use_rga", false}}); + auto proc = HwFactory::CreateImageProcessor(cfg); + ASSERT_NE(proc, nullptr); + + Frame src = MakeNv12Frame(2, 2); + Frame dst; + dst.width = 2; + dst.height = 2; + dst.format = PixelFormat::RGB; + + Status st = proc->CvtColor(src, dst, PixelFormat::RGB); + EXPECT_TRUE(st.IsOk()); + EXPECT_EQ(dst.width, 2); + EXPECT_EQ(dst.height, 2); + EXPECT_EQ(dst.format, PixelFormat::RGB); + EXPECT_NE(dst.data, nullptr); + EXPECT_GT(dst.data_size, 0u); +#else + GTEST_SKIP() << "FFmpeg not enabled"; +#endif +} + +} // namespace +} // namespace rk3588 diff --git a/tests/test_infer_backend.cpp b/tests/test_infer_backend.cpp new file mode 100644 index 0000000..ec6a763 --- /dev/null +++ b/tests/test_infer_backend.cpp @@ -0,0 +1,53 @@ +#include + +#include + +#include "hw/hw_factory.h" +#include "hw/rk3588_defaults.h" + +namespace rk3588 { +namespace { + +TEST(InferBackendTest, LoadModelSmoke) { + SimpleJson config(SimpleJson::Object{}); + auto infer = HwFactory::CreateInferBackend(config); + ASSERT_NE(infer, nullptr); + + std::filesystem::path repo_root = std::filesystem::path(__FILE__).parent_path().parent_path(); + std::filesystem::path model_path = repo_root / "models" / "RetinaFace_mobile320.rknn"; + if (!std::filesystem::exists(model_path)) { + GTEST_SKIP() << "model file not found: " << model_path.string(); + } + + std::string err; + ModelHandle handle = infer->LoadModel(model_path.string(), err); + +#if defined(RK3588_ENABLE_RKNN) + EXPECT_NE(handle, kInvalidModelHandle) << err; + if (handle != kInvalidModelHandle) { + infer->UnloadModel(handle); + } +#else + EXPECT_EQ(handle, kInvalidModelHandle); + EXPECT_FALSE(err.empty()); +#endif +} + +TEST(InferBackendTest, BorrowedInputUsesDmaFd) { + SimpleJson config(SimpleJson::Object{}); + auto infer = HwFactory::CreateInferBackend(config); + auto rk = std::dynamic_pointer_cast(infer); + ASSERT_NE(rk, nullptr); + + InferInput input; + input.dma_fd = 7; + input.dma_offset = 16; + rk->InferBorrowed(kInvalidModelHandle, input); + + InferInput last = rk->GetLastBorrowedInput(); + EXPECT_EQ(last.dma_fd, 7); + EXPECT_EQ(last.dma_offset, 16); +} + +} // namespace +} // namespace rk3588