Doni/safesight-edge
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

解耦开发3完成，准备编译

Browse Source
This commit is contained in:
sladro 2026-01-16 19:10:30 +08:00
parent 54cbb1434e
commit e4a1732c69
16 changed files with 1121 additions and 862 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -57,6 +57,7 @@ set(SRC_FILES
src/ai_scheduler.cpp
src/http_server.cpp
src/hw_factory.cpp
src/hw_image_processor.cpp
src/utils/dma_alloc.cpp
src/utils/config_expand.cpp
)

3
include/graph_manager.h
View File

@ -20,6 +20,8 @@
namespace rk3588 {
class IInferBackend;
struct QueueSnapshot {
size_t size = 0;
size_t capacity = 0;
@ -158,6 +160,7 @@ private:
SimpleJson graph_cfg_;
size_t built_default_queue_size_ = 8;
QueueDropStrategy built_default_strategy_ = QueueDropStrategy::DropOldest;
std::shared_ptr<IInferBackend> infer_backend_;
std::vector<NodeEntry> nodes_;
std::vector<EdgeEntry> edges_;
std::unique_ptr<Executor> executor_;

2
include/hw/i_infer_backend.h
View File

@ -11,6 +11,8 @@ public:
virtual ~IInferBackend() = default;
virtual ModelHandle LoadModel(const std::string& model_path, std::string& err) = 0;
virtual void UnloadModel(ModelHandle handle) = 0;
virtual bool GetModelInfo(ModelHandle handle, ModelInfo& info) const = 0;
virtual InferResult Infer(ModelHandle handle, const InferInput& input) = 0;
virtual AiScheduler::BorrowedInferResult InferBorrowed(ModelHandle handle, const InferInput& input) = 0;
};

72
include/hw/rk3588_defaults.h
View File

@ -1,52 +1,74 @@
#pragma once
#include <memory>
#include <mutex>
#include <string>
#include <vector>
#include "hw/i_decoder.h"
#include "hw/i_encoder.h"
#include "hw/i_image_processor.h"
#include "hw/i_infer_backend.h"
#include "utils/simple_json.h"
#include "utils/result.h"
namespace rk3588 {
class Rk3588InferBackend : public IInferBackend {
class RknnInferBackend : public IInferBackend {
public:
ModelHandle LoadModel(const std::string& /*model_path*/, std::string& err) override {
err = "not implemented";
return kInvalidModelHandle;
ModelHandle LoadModel(const std::string& model_path, std::string& err) override {
return AiScheduler::Instance().LoadModel(model_path, err);
}
InferResult Infer(ModelHandle /*handle*/, const InferInput& /*input*/) override {
InferResult result;
result.success = false;
result.error = "not implemented";
return result;
void UnloadModel(ModelHandle handle) override {
AiScheduler::Instance().UnloadModel(handle);
}
AiScheduler::BorrowedInferResult InferBorrowed(ModelHandle /*handle*/, const InferInput& /*input*/) override {
AiScheduler::BorrowedInferResult result;
result.success = false;
result.error = "not implemented";
return result;
bool GetModelInfo(ModelHandle handle, ModelInfo& info) const override {
return AiScheduler::Instance().GetModelInfo(handle, info);
}
InferResult Infer(ModelHandle handle, const InferInput& input) override {
return AiScheduler::Instance().Infer(handle, input);
}
AiScheduler::BorrowedInferResult InferBorrowed(ModelHandle handle, const InferInput& input) override {
{
std::lock_guard<std::mutex> lock(mu_);
last_borrowed_input_ = input;
}
return AiScheduler::Instance().InferBorrowed(handle, input);
}
InferInput GetLastBorrowedInput() const {
std::lock_guard<std::mutex> lock(mu_);
return last_borrowed_input_;
}
private:
mutable std::mutex mu_;
InferInput last_borrowed_input_;
};
using Rk3588InferBackend = RknnInferBackend;
class Rk3588ImageProcessor : public IImageProcessor {
public:
Status Resize(const Frame& /*src*/, Frame& /*dst*/) override {
return FailStatus("not implemented");
}
explicit Rk3588ImageProcessor(const SimpleJson& config);
Status CvtColor(const Frame& /*src*/, Frame& /*dst*/, PixelFormat /*dst_format*/) override {
return FailStatus("not implemented");
}
Status Resize(const Frame& src, Frame& dst) override;
Status CvtColor(const Frame& src, Frame& dst, PixelFormat dst_format) override;
Status Normalize(const Frame& src, std::vector<float>& out,
const std::vector<float>& mean,
const std::vector<float>& std) override;
Status Normalize(const Frame& /*src*/, std::vector<float>& /*out*/,
const std::vector<float>& /*mean*/,
const std::vector<float>& /*std*/) override {
return FailStatus("not implemented");
}
bool IsUsingRga() const;
int RgaMaxInflight() const;
std::string RgaGateKey() const;
private:
std::shared_ptr<IImageProcessor> impl_;
bool use_rga_ = true;
};
class Rk3588Decoder : public IDecoder {

5
include/node.h
View File

@ -12,6 +12,9 @@ namespace rk3588 {
using FramePtr = std::shared_ptr<Frame>;
class IInferBackend;
class IImageProcessor;
enum class NodeStatus {
OK, // Normal processing
DROP, // Drop this frame (e.g., condition not met)
@ -23,6 +26,8 @@ struct NodeContext {
std::shared_ptr<SpscQueue<FramePtr>> input_queue;
// For nodes that produce downstream outputs (one queue per outgoing edge).
std::vector<std::shared_ptr<SpscQueue<FramePtr>>> output_queues;
std::shared_ptr<IInferBackend> infer_backend;
std::shared_ptr<IImageProcessor> image_processor;
};
class INode {

1
plugins/CMakeLists.txt
View File

@ -186,6 +186,7 @@ set_target_properties(publish PROPERTIES
add_library(preprocess SHARED
preprocess/preprocess_node.cpp
${CMAKE_SOURCE_DIR}/src/hw_image_processor.cpp
${CMAKE_SOURCE_DIR}/src/utils/dma_alloc.cpp
)
target_include_directories(preprocess PRIVATE ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/third_party)

17
plugins/ai_face_det/ai_face_det_node.cpp
View File

@ -11,7 +11,7 @@
#include <string>
#include <vector>
#include "ai_scheduler.h"
#include "hw/i_infer_backend.h"
#include "face/face_result.h"
#include "node.h"
#include "utils/dma_alloc.h"
@ -440,19 +440,25 @@ public:
return false;
}
infer_backend_ = ctx.infer_backend;
if (!infer_backend_) {
LogError("[ai_face_det] no infer backend for node " + id_);
return false;
}
#if defined(RK3588_ENABLE_RKNN)
if (model_path_.empty()) {
LogError("[ai_face_det] model_path is required");
return false;
}
std::string err;
model_handle_ = AiScheduler::Instance().LoadModel(model_path_, err);
model_handle_ = infer_backend_->LoadModel(model_path_, err);
if (model_handle_ == kInvalidModelHandle) {
LogError("[ai_face_det] failed to load model: " + err);
return false;
}
ModelInfo info;
if (AiScheduler::Instance().GetModelInfo(model_handle_, info)) {
if (infer_backend_->GetModelInfo(model_handle_, info)) {
model_w_ = info.input_width;
model_h_ = info.input_height;
n_output_ = info.n_output;
@ -510,7 +516,7 @@ public:
void Stop() override {
#if defined(RK3588_ENABLE_RKNN)
if (model_handle_ != kInvalidModelHandle) {
AiScheduler::Instance().UnloadModel(model_handle_);
infer_backend_->UnloadModel(model_handle_);
model_handle_ = kInvalidModelHandle;
}
#endif
@ -627,7 +633,7 @@ private:
if (sync_src) DmaSyncEndFd(frame->dma_fd);
auto r = AiScheduler::Instance().InferBorrowed(model_handle_, input);
auto r = infer_backend_->InferBorrowed(model_handle_, input);
if (!r.success) {
LogWarn("[ai_face_det] inference failed: " + r.error);
return;
@ -860,6 +866,7 @@ private:
std::shared_ptr<SpscQueue<FramePtr>> input_queue_;
std::vector<std::shared_ptr<SpscQueue<FramePtr>>> output_queues_;
std::shared_ptr<IInferBackend> infer_backend_;
std::vector<uint8_t> input_buf_;
std::vector<float> float_input_buf_;

17
plugins/ai_face_recog/ai_face_recog_node.cpp
View File

@ -12,7 +12,7 @@
#include <utility>
#include <vector>
#include "ai_scheduler.h"
#include "hw/i_infer_backend.h"
#include "face/face_result.h"
#include "node.h"
#include "utils/dma_alloc.h"
@ -663,19 +663,25 @@ public:
return false;
}
infer_backend_ = ctx.infer_backend;
if (!infer_backend_) {
LogError("[ai_face_recog] no infer backend for node " + id_);
return false;
}
#if defined(RK3588_ENABLE_RKNN)
if (model_path_.empty()) {
LogError("[ai_face_recog] model_path is required");
return false;
}
std::string err;
model_handle_ = AiScheduler::Instance().LoadModel(model_path_, err);
model_handle_ = infer_backend_->LoadModel(model_path_, err);
if (model_handle_ == kInvalidModelHandle) {
LogError("[ai_face_recog] failed to load model: " + err);
return false;
}
ModelInfo info;
if (AiScheduler::Instance().GetModelInfo(model_handle_, info)) {
if (infer_backend_->GetModelInfo(model_handle_, info)) {
model_w_ = info.input_width;
model_h_ = info.input_height;
}
@ -746,7 +752,7 @@ public:
void Stop() override {
#if defined(RK3588_ENABLE_RKNN)
if (model_handle_ != kInvalidModelHandle) {
AiScheduler::Instance().UnloadModel(model_handle_);
infer_backend_->UnloadModel(model_handle_);
model_handle_ = kInvalidModelHandle;
}
#endif
@ -893,7 +899,7 @@ private:
in.type = RKNN_TENSOR_UINT8;
}
auto r = AiScheduler::Instance().InferBorrowed(model_handle_, in);
auto r = infer_backend_->InferBorrowed(model_handle_, in);
if (!r.success || r.outputs.empty()) {
LogWarn(std::string("[ai_face_recog] inference failed: ") + (r.error.empty() ? "unknown" : r.error));
continue;
@ -944,6 +950,7 @@ private:
std::shared_ptr<SpscQueue<FramePtr>> input_queue_;
std::vector<std::shared_ptr<SpscQueue<FramePtr>>> output_queues_;
std::shared_ptr<IInferBackend> infer_backend_;
std::vector<uint8_t> face_buf_;
std::vector<float> float_input_buf_;

19
plugins/ai_yolo/ai_yolo_node.cpp
View File

@ -9,7 +9,7 @@
#include <thread>
#include <vector>
#include "ai_scheduler.h"
#include "hw/i_infer_backend.h"
#include "node.h"
#include "utils/dma_alloc.h"
#include "utils/logger.h"
@ -319,6 +319,12 @@ public:
}
output_queues_ = ctx.output_queues;
infer_backend_ = ctx.infer_backend;
if (!infer_backend_) {
LogError("[ai_yolo] no infer backend for node " + id_);
return false;
}
#if defined(RK3588_ENABLE_RKNN)
if (model_path_.empty()) {
LogError("[ai_yolo] model_path is required");
@ -326,14 +332,14 @@ public:
}
std::string err;
model_handle_ = AiScheduler::Instance().LoadModel(model_path_, err);
model_handle_ = infer_backend_->LoadModel(model_path_, err);
if (model_handle_ == kInvalidModelHandle) {
LogError("[ai_yolo] failed to load model: " + err);
return false;
}
ModelInfo info;
if (AiScheduler::Instance().GetModelInfo(model_handle_, info)) {
if (infer_backend_->GetModelInfo(model_handle_, info)) {
model_input_w_ = info.input_width;
model_input_h_ = info.input_height;
n_output_ = info.n_output;
@ -347,7 +353,7 @@ public:
}
}
LogInfo("[ai_yolo] model loaded via AiScheduler: " + model_path_ +
LogInfo("[ai_yolo] model loaded via InferBackend: " + model_path_ +
" (handle=" + std::to_string(model_handle_) + ", version=" +
(yolo_version_ == YoloVersion::V5 ? "v5" : "v8") + ")");
#else
@ -365,7 +371,7 @@ public:
void Stop() override {
#if defined(RK3588_ENABLE_RKNN)
if (model_handle_ != kInvalidModelHandle) {
AiScheduler::Instance().UnloadModel(model_handle_);
infer_backend_->UnloadModel(model_handle_);
model_handle_ = kInvalidModelHandle;
}
#endif
@ -460,7 +466,7 @@ private:
input.height = h;
input.is_nhwc = true;
auto result = AiScheduler::Instance().InferBorrowed(model_handle_, input);
auto result = infer_backend_->InferBorrowed(model_handle_, input);
if (!result.success) {
LogWarn("[ai_yolo] inference failed: " + result.error);
return;
@ -720,6 +726,7 @@ private:
std::shared_ptr<SpscQueue<FramePtr>> input_queue_;
std::vector<std::shared_ptr<SpscQueue<FramePtr>>> output_queues_;
std::shared_ptr<IInferBackend> infer_backend_;
uint64_t processed_ = 0;
bool stats_log_ = false;

898
plugins/preprocess/preprocess_node.cpp
View File

@ -1,159 +1,17 @@
#include <algorithm>
#include <algorithm>
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <cstdlib>
#include <cstring>
#include <memory>
#include <mutex>
#include <thread>
#include <unordered_map>
#include <cstdint>
#include <string>
#include <vector>
#include "hw/i_image_processor.h"
#include "node.h"
#include "utils/dma_alloc.h"
#include "utils/logger.h"
#if defined(RK3588_ENABLE_RGA)
#include "im2d.hpp"
#include "im2d_buffer.h"
#include "im2d_type.h"
#endif
#if defined(RK3588_ENABLE_FFMPEG)
extern "C" {
#include <libavutil/imgutils.h>
#include <libswscale/swscale.h>
}
#endif
namespace rk3588 {
namespace {
inline int Align16(int v) { return (v + 15) & ~15; }
#if defined(RK3588_ENABLE_RGA)
int ToRgaFormat(PixelFormat fmt) {
switch (fmt) {
case PixelFormat::NV12: return RK_FORMAT_YCbCr_420_SP;
case PixelFormat::YUV420: return RK_FORMAT_YCbCr_420_P;
case PixelFormat::RGB: return RK_FORMAT_RGB_888;
case PixelFormat::BGR: return RK_FORMAT_BGR_888;
default: return RK_FORMAT_UNKNOWN;
}
}
std::atomic<int>& GlobalRgaMaxInflightRef() {
static std::atomic<int> v{2};
return v;
}
int GlobalRgaMaxInflight() {
static std::once_flag once;
std::call_once(once, []() {
const char* s = std::getenv("RK3588_RGA_MAX_INFLIGHT");
if (!s || !*s) return;
try {
int v = std::stoi(s);
if (v > 0 && v <= 32) GlobalRgaMaxInflightRef().store(v);
} catch (...) {
LogWarn(std::string("[preprocess] invalid RK3588_RGA_MAX_INFLIGHT='") + s + "'");
}
});
int v = GlobalRgaMaxInflightRef().load();
return v > 0 ? v : 1;
}
class RgaGate {
public:
explicit RgaGate(int max_inflight) : max_inflight_(max_inflight > 0 ? max_inflight : 1) {}
void Acquire() {
std::unique_lock<std::mutex> lock(mu_);
cv_.wait(lock, [&]() { return in_flight_ < max_inflight_; });
++in_flight_;
}
void Release() {
std::lock_guard<std::mutex> lock(mu_);
if (in_flight_ > 0) {
--in_flight_;
}
cv_.notify_one();
}
void SetMaxInflight(int v) {
if (v <= 0) return;
if (v > 32) v = 32;
{
std::lock_guard<std::mutex> lock(mu_);
max_inflight_ = v;
}
cv_.notify_all();
}
int MaxInflight() const {
std::lock_guard<std::mutex> lock(mu_);
return max_inflight_;
}
private:
mutable std::mutex mu_;
std::condition_variable cv_;
int in_flight_ = 0;
int max_inflight_ = 1;
};
class RgaGateRegistry {
public:
static RgaGateRegistry& Instance() {
static RgaGateRegistry* inst = new RgaGateRegistry();
return *inst;
}
RgaGate& Get(const std::string& key) {
std::lock_guard<std::mutex> lock(mu_);
auto it = gates_.find(key);
if (it != gates_.end()) return *it->second;
auto gate = std::make_unique<RgaGate>(GlobalRgaMaxInflight());
RgaGate& ref = *gate;
gates_.emplace(key, std::move(gate));
return ref;
}
private:
std::mutex mu_;
std::unordered_map<std::string, std::unique_ptr<RgaGate>> gates_;
};
RgaGate& GetRgaGate(const std::string& key) {
const std::string k = key.empty() ? "global" : key;
return RgaGateRegistry::Instance().Get(k);
}
class ScopedRgaGate {
public:
explicit ScopedRgaGate(const std::string& key) : gate_(&GetRgaGate(key)) { gate_->Acquire(); }
~ScopedRgaGate() { gate_->Release(); }
ScopedRgaGate(const ScopedRgaGate&) = delete;
ScopedRgaGate& operator=(const ScopedRgaGate&) = delete;
private:
RgaGate* gate_ = nullptr;
};
void EnsureRgaInitializedOnce() {
static std::once_flag once;
std::call_once(once, []() {
const IM_STATUS st = imcheckHeader();
if (st != IM_STATUS_NOERROR && st != IM_STATUS_SUCCESS) {
LogWarn(std::string("[preprocess] imcheckHeader failed: ") + imStrError(st));
}
});
}
#endif
PixelFormat ParseFormat(const std::string& s) {
if (s == "nv12" || s == "NV12") return PixelFormat::NV12;
if (s == "yuv420" || s == "YUV420") return PixelFormat::YUV420;
@ -162,145 +20,6 @@ PixelFormat ParseFormat(const std::string& s) {
return PixelFormat::UNKNOWN;
}
size_t CalcImageSize(int w, int h, PixelFormat fmt) {
switch (fmt) {
case PixelFormat::NV12:
case PixelFormat::YUV420:
return static_cast<size_t>(w) * h * 3 / 2;
case PixelFormat::RGB:
case PixelFormat::BGR:
return static_cast<size_t>(w) * h * 3;
default:
return 0;
}
}
size_t CalcImageSizeStrided(int wstride, int hstride, PixelFormat fmt) {
if (wstride <= 0 || hstride <= 0) return 0;
const size_t ws = static_cast<size_t>(wstride);
const size_t hs = static_cast<size_t>(hstride);
switch (fmt) {
case PixelFormat::NV12: {
// Y: ws*hs, UV: ws*(hs/2)
return ws * hs + ws * (hs / 2);
}
case PixelFormat::YUV420: {
// Y: ws*hs, U/V: (ws/2)*(hs/2)
const size_t y = ws * hs;
const size_t uv = (ws / 2) * (hs / 2);
return y + uv + uv;
}
case PixelFormat::RGB:
case PixelFormat::BGR:
return ws * hs * 3;
default:
return 0;
}
}
bool CopyToStridedBuffer(const Frame& src, uint8_t* dst, size_t dst_size,
int dst_wstride, int dst_hstride) {
if (!dst || dst_size == 0) return false;
if (dst_wstride <= 0 || dst_hstride <= 0) return false;
std::memset(dst, 0, dst_size);
const int w = src.width;
const int h = src.height;
if (w <= 0 || h <= 0) return false;
if (src.format == PixelFormat::NV12) {
const size_t y_bytes = static_cast<size_t>(dst_wstride) * dst_hstride;
const size_t uv_bytes = static_cast<size_t>(dst_wstride) * (dst_hstride / 2);
if (y_bytes + uv_bytes > dst_size) return false;
const uint8_t* src_y = src.planes[0].data ? src.planes[0].data : src.data;
const uint8_t* src_uv = src.planes[1].data ? src.planes[1].data : nullptr;
const int src_y_stride = src.planes[0].stride > 0 ? src.planes[0].stride : w;
const int src_uv_stride = src.planes[1].stride > 0 ? src.planes[1].stride : w;
if (!src_y) return false;
if (!src_uv) {
// Fallback: packed NV12 layout.
if (!src.data) return false;
src_uv = src.data + static_cast<size_t>(src_y_stride) * static_cast<size_t>(h);
}
for (int row = 0; row < h; ++row) {
std::memcpy(dst + static_cast<size_t>(row) * dst_wstride,
src_y + static_cast<size_t>(row) * src_y_stride,
static_cast<size_t>(w));
}
uint8_t* dst_uv = dst + y_bytes;
const int uv_rows = h / 2;
for (int row = 0; row < uv_rows; ++row) {
std::memcpy(dst_uv + static_cast<size_t>(row) * dst_wstride,
src_uv + static_cast<size_t>(row) * src_uv_stride,
static_cast<size_t>(w));
}
return true;
}
if (src.format == PixelFormat::YUV420) {
const size_t y_bytes = static_cast<size_t>(dst_wstride) * dst_hstride;
const size_t uv_stride = static_cast<size_t>(dst_wstride) / 2;
const size_t uv_h = static_cast<size_t>(dst_hstride) / 2;
const size_t u_bytes = uv_stride * uv_h;
const size_t v_bytes = u_bytes;
if (y_bytes + u_bytes + v_bytes > dst_size) return false;
const uint8_t* src_y = src.planes[0].data ? src.planes[0].data : src.data;
const uint8_t* src_u = src.planes[1].data ? src.planes[1].data : nullptr;
const uint8_t* src_v = src.planes[2].data ? src.planes[2].data : nullptr;
const int src_y_stride = src.planes[0].stride > 0 ? src.planes[0].stride : w;
const int src_u_stride = src.planes[1].stride > 0 ? src.planes[1].stride : (w / 2);
const int src_v_stride = src.planes[2].stride > 0 ? src.planes[2].stride : (w / 2);
if (!src_y || !src_u || !src_v) return false;
for (int row = 0; row < h; ++row) {
std::memcpy(dst + static_cast<size_t>(row) * dst_wstride,
src_y + static_cast<size_t>(row) * src_y_stride,
static_cast<size_t>(w));
}
uint8_t* dst_u = dst + y_bytes;
uint8_t* dst_v = dst + y_bytes + u_bytes;
const int uv_rows = h / 2;
const int uv_cols = w / 2;
for (int row = 0; row < uv_rows; ++row) {
std::memcpy(dst_u + static_cast<size_t>(row) * uv_stride,
src_u + static_cast<size_t>(row) * src_u_stride,
static_cast<size_t>(uv_cols));
std::memcpy(dst_v + static_cast<size_t>(row) * uv_stride,
src_v + static_cast<size_t>(row) * src_v_stride,
static_cast<size_t>(uv_cols));
}
return true;
}
if (src.format == PixelFormat::RGB || src.format == PixelFormat::BGR) {
const size_t need = static_cast<size_t>(dst_wstride) * dst_hstride * 3;
if (need > dst_size) return false;
const uint8_t* src_rgb = src.planes[0].data ? src.planes[0].data : src.data;
const int src_stride = src.planes[0].stride > 0
? src.planes[0].stride
: (src.stride > 0 ? src.stride : w * 3);
if (!src_rgb) return false;
const size_t dst_stride = static_cast<size_t>(dst_wstride) * 3;
const size_t row_bytes = static_cast<size_t>(w) * 3;
for (int row = 0; row < h; ++row) {
std::memcpy(dst + static_cast<size_t>(row) * dst_stride,
src_rgb + static_cast<size_t>(row) * src_stride,
row_bytes);
}
return true;
}
return false;
}
} // namespace
class PreprocessNode : public INode {
@ -314,32 +33,18 @@ public:
dst_h_ = config.ValueOr<int>("dst_h", 640);
keep_ratio_ = config.ValueOr<bool>("keep_ratio", false);
if (config.Find("dst_packed")) {
dst_packed_ = config.ValueOr<bool>("dst_packed", false);
dst_packed_explicit_ = true;
} else {
dst_packed_ = false;
dst_packed_explicit_ = false;
}
std::string fmt_str = config.ValueOr<std::string>("dst_format", "");
if (!fmt_str.empty()) {
dst_fmt_ = ParseFormat(fmt_str);
}
#if defined(RK3588_ENABLE_RGA)
rga_gate_ = config.ValueOr<std::string>("rga_gate", "global");
const int rga_max_inflight = config.ValueOr<int>("rga_max_inflight", 0);
if (rga_max_inflight > 0) {
GetRgaGate(rga_gate_).SetMaxInflight(rga_max_inflight);
}
#endif
const bool requested_use_rga = config.ValueOr<bool>("use_rga", true);
use_rga_ = requested_use_rga;
if (const SimpleJson* dbg = config.Find("debug"); dbg && dbg->IsObject()) {
stats_log_ = dbg->ValueOr<bool>("stats", stats_log_);
stats_interval_ = std::max<uint64_t>(1, static_cast<uint64_t>(dbg->ValueOr<int>("stats_interval", static_cast<int>(stats_interval_))));
stats_interval_ = std::max<uint64_t>(
1, static_cast<uint64_t>(dbg->ValueOr<int>("stats_interval", static_cast<int>(stats_interval_))));
}
input_queue_ = ctx.input_queue;
@ -366,46 +71,90 @@ public:
return false;
}
#endif
image_processor_ = ctx.image_processor;
if (!image_processor_) {
LogError("[preprocess] no image processor for node " + id_);
return false;
}
return true;
}
bool Start() override {
std::string extra;
#if defined(RK3588_ENABLE_RGA)
if (use_rga_) {
extra = " gate=" + rga_gate_ + " max_inflight=" + std::to_string(GetRgaGate(rga_gate_).MaxInflight());
}
#endif
LogInfo("[preprocess] start id=" + id_ + " dst=" + std::to_string(dst_w_) + "x" + std::to_string(dst_h_) +
(use_rga_ ? " (rga)" : " (swscale)") + extra);
LogInfo("[preprocess] start id=" + id_ + " dst=" + std::to_string(dst_w_) + "x" +
std::to_string(dst_h_) + (use_rga_ ? " (rga)" : " (swscale)"));
return true;
}
void Stop() override {
#if defined(RK3588_ENABLE_FFMPEG)
if (sws_ctx_) {
sws_freeContext(sws_ctx_);
sws_ctx_ = nullptr;
}
#endif
}
void Stop() override {}
NodeStatus Process(FramePtr frame) override {
if (!frame) return NodeStatus::DROP;
if (!image_processor_) return NodeStatus::ERROR;
#if defined(RK3588_ENABLE_RGA)
if (use_rga_) {
if (!ProcessRga(frame)) {
return NodeStatus::ERROR;
}
} else {
ProcessSwscale(frame);
PixelFormat out_fmt = (dst_fmt_ != PixelFormat::UNKNOWN) ? dst_fmt_ : frame->format;
int out_w = dst_w_;
int out_h = dst_h_;
if (out_w <= 0) out_w = frame->width;
if (out_h <= 0) out_h = frame->height;
if (keep_ratio_ && dst_w_ > 0 && dst_h_ > 0 && frame->width > 0 && frame->height > 0) {
float scale = std::min(static_cast<float>(dst_w_) / frame->width,
static_cast<float>(dst_h_) / frame->height);
out_w = static_cast<int>(frame->width * scale);
out_h = static_cast<int>(frame->height * scale);
out_w = (out_w + 1) & ~1;
out_h = (out_h + 1) & ~1;
}
#elif defined(RK3588_ENABLE_FFMPEG)
ProcessSwscale(frame);
#else
ProcessPassthrough(frame);
#endif
const bool need_resize = (frame->width != out_w || frame->height != out_h);
const bool need_cvt = (frame->format != out_fmt);
if (need_resize) {
WarnMetaResizeOnce(frame, out_w, out_h);
}
if (!need_resize && !need_cvt) {
ProcessPassthrough(frame);
return NodeStatus::OK;
}
Frame out;
out.width = out_w;
out.height = out_h;
out.format = out_fmt;
Status st = image_processor_->Resize(*frame, out);
if (st.Failed()) {
if (!use_rga_ && st.ErrMessage().find("unsupported format") != std::string::npos) {
ProcessPassthrough(frame);
return NodeStatus::OK;
}
LogError("[preprocess] " + st.ErrMessage());
return NodeStatus::ERROR;
}
auto out_frame = std::make_shared<Frame>(out);
out_frame->pts = frame->pts;
out_frame->frame_id = frame->frame_id;
out_frame->det = frame->det;
out_frame->face_det = frame->face_det;
out_frame->face_recog = frame->face_recog;
out_frame->user_meta = frame->user_meta;
PushToDownstream(out_frame);
++processed_;
if (stats_log_ && stats_interval_ > 0 && (processed_ % stats_interval_) == 0) {
LogInfo("[preprocess] " + std::string(use_rga_ ? "rga" : "swscale") +
" frame=" + std::to_string(out_frame->frame_id) +
" " + std::to_string(frame->width) + "x" + std::to_string(frame->height) +
" -> " + std::to_string(out_w) + "x" + std::to_string(out_h) +
" id=" + id_);
}
return NodeStatus::OK;
}
@ -433,495 +182,13 @@ private:
}
}
#if defined(RK3588_ENABLE_RGA)
bool ProcessRga(FramePtr frame) {
EnsureRgaInitializedOnce();
PixelFormat out_fmt = (dst_fmt_ != PixelFormat::UNKNOWN) ? dst_fmt_ : frame->format;
int out_w = dst_w_;
int out_h = dst_h_;
// Allow config to follow source resolution:
// - dst_w <= 0 => out_w = frame->width
// - dst_h <= 0 => out_h = frame->height
if (out_w <= 0) out_w = frame->width;
if (out_h <= 0) out_h = frame->height;
if (keep_ratio_ && dst_w_ > 0 && dst_h_ > 0 && frame->width > 0 && frame->height > 0) {
float scale = std::min(static_cast<float>(dst_w_) / frame->width,
static_cast<float>(dst_h_) / frame->height);
out_w = static_cast<int>(frame->width * scale);
out_h = static_cast<int>(frame->height * scale);
out_w = (out_w + 1) & ~1;
out_h = (out_h + 1) & ~1;
}
int src_fmt_rga = ToRgaFormat(frame->format);
int dst_fmt_rga = ToRgaFormat(out_fmt);
bool need_cvt = (src_fmt_rga != dst_fmt_rga);
bool need_resize = (frame->width != out_w || frame->height != out_h);
if (need_resize) {
WarnMetaResizeOnce(frame, out_w, out_h);
}
// If no processing needed, passthrough directly
if (!need_cvt && !need_resize) {
PushToDownstream(frame);
++processed_;
if (stats_log_ && stats_interval_ > 0 && (processed_ % stats_interval_) == 0) {
LogInfo("[preprocess] passthrough frame=" + std::to_string(frame->frame_id) +
" " + std::to_string(frame->width) + "x" + std::to_string(frame->height) + " (no change)" +
" id=" + id_);
}
return true;
}
// Calculate proper strides.
// IMPORTANT: For DMA-BUF frames (e.g. MPP decode output), the actual vertical stride
// (ver_stride) may be larger than `height`. We must honor that, otherwise RGA will
// read UV from the wrong offset (典型花屏/错色).
int src_wstride = Align16(frame->width);
int src_hstride = Align16(frame->height);
if (frame->format == PixelFormat::NV12 || frame->format == PixelFormat::YUV420) {
const int y_stride = frame->planes[0].stride > 0 ? frame->planes[0].stride
: (frame->stride > 0 ? frame->stride : frame->width);
if (y_stride > 0) src_wstride = y_stride;
if (frame->planes[0].size > 0 && y_stride > 0) {
const int hs = frame->planes[0].size / y_stride;
if (hs >= frame->height) src_hstride = hs;
}
} else if (frame->format == PixelFormat::RGB || frame->format == PixelFormat::BGR) {
const int stride_bytes = frame->planes[0].stride > 0 ? frame->planes[0].stride
: (frame->stride > 0 ? frame->stride : frame->width * 3);
if (stride_bytes > 0 && (stride_bytes % 3) == 0) {
src_wstride = stride_bytes / 3;
}
if (frame->planes[0].size > 0 && stride_bytes > 0) {
const int hs = frame->planes[0].size / stride_bytes;
if (hs >= frame->height) src_hstride = hs;
}
}
int dst_wstride = Align16(out_w);
// For AI input (RGB/BGR), allow a tightly packed output to avoid an extra per-frame memcpy
// in downstream nodes (e.g. ai_yolo).
const bool want_packed_rgb = (out_fmt == PixelFormat::RGB || out_fmt == PixelFormat::BGR) &&
(!dst_packed_explicit_ || dst_packed_);
if (want_packed_rgb) {
dst_wstride = out_w;
}
int dst_hstride = Align16(out_h);
if (src_fmt_rga == RK_FORMAT_UNKNOWN || dst_fmt_rga == RK_FORMAT_UNKNOWN) {
LogError("[preprocess] unsupported format for RGA");
return false;
}
size_t out_size = CalcImageSizeStrided(dst_wstride, dst_hstride, out_fmt);
if (out_size == 0) {
LogError("[preprocess] invalid output size for RGA");
return false;
}
// Use DMA-BUF allocation to avoid >4GB address issue with RGA
auto dma_buf = DmaAlloc(out_size);
if (!dma_buf || !dma_buf->valid()) {
LogError("[preprocess] DMA alloc failed");
return false;
}
if (stats_log_ && processed_ < 3) {
LogInfo("[preprocess] src: " + std::to_string(frame->width) + "x" + std::to_string(frame->height) +
" fmt=" + std::to_string(static_cast<int>(frame->format)) +
" rga_fmt=" + std::to_string(src_fmt_rga) +
" wstride=" + std::to_string(src_wstride) +
" hstride=" + std::to_string(src_hstride) +
" data_size=" + std::to_string(frame->data_size));
LogInfo("[preprocess] dst: " + std::to_string(out_w) + "x" + std::to_string(out_h) +
" fmt=" + std::to_string(static_cast<int>(out_fmt)) +
" rga_fmt=" + std::to_string(dst_fmt_rga) +
" wstride=" + std::to_string(dst_wstride) +
" hstride=" + std::to_string(dst_hstride) +
" out_size=" + std::to_string(out_size));
}
rga_buffer_t src_buf{};
rga_buffer_t dst_buf{};
DmaBufferPtr src_dma_buf; // keep alive if we allocate/copy
// Lazily allocate tmp buffer only when we must fall back to a 2-step pipeline.
// (resize + cvtcolor). Prefer a single improcess() call to reduce scheduling overhead.
DmaBufferPtr tmp_dma;
const bool can_cpu_read_src = (frame->data != nullptr && frame->data_size > 0);
auto CopySrcToDmaIfPossible = [&]() -> bool {
if (!can_cpu_read_src) return false;
// Use tight/aligned strides for the copied source buffer.
const int copy_wstride = Align16(frame->width);
const int copy_hstride = Align16(frame->height);
const size_t src_size = CalcImageSizeStrided(copy_wstride, copy_hstride, frame->format);
src_dma_buf = DmaAlloc(src_size);
if (!src_dma_buf || !src_dma_buf->valid()) {
return false;
}
// If source is a DMA-BUF, sync it before CPU reads.
if (frame->dma_fd >= 0) {
DmaSyncStartFd(frame->dma_fd);
}
// CPU writes to a DMA-BUF must be flushed before RGA reads it.
DmaSyncStartFd(src_dma_buf->fd);
const bool ok = CopyToStridedBuffer(*frame, src_dma_buf->data(), src_dma_buf->size,
copy_wstride, copy_hstride);
DmaSyncEndFd(src_dma_buf->fd);
if (frame->dma_fd >= 0) {
DmaSyncEndFd(frame->dma_fd);
}
if (!ok) {
src_dma_buf.reset();
return false;
}
// Update strides used for RGA to match the copied buffer.
src_wstride = copy_wstride;
src_hstride = copy_hstride;
return true;
};
// If there's no DMA fd, we must allocate/copy into a DMA-BUF for RGA.
if (frame->dma_fd < 0) {
if (!CopySrcToDmaIfPossible()) {
LogError("[preprocess] no dma_fd and src copy failed");
return false;
}
}
auto RunRgaOnce = [&](int src_fd, std::string& err) -> bool {
// Serialize/limit librga/im2d usage; multiple pipelines call RGA concurrently.
ScopedRgaGate guard(rga_gate_);
src_buf = wrapbuffer_fd_t(src_fd, frame->width, frame->height,
src_wstride, src_hstride, src_fmt_rga);
dst_buf = wrapbuffer_fd_t(dma_buf->fd, out_w, out_h,
dst_wstride, dst_hstride, dst_fmt_rga);
auto Check = [&](const rga_buffer_t& s, const rga_buffer_t& d) -> bool {
const im_rect sr{0, 0, s.width, s.height};
const im_rect dr{0, 0, d.width, d.height};
// Do NOT call the imcheck(...) variadic macro with 0 extra args:
// under -Wpedantic/-std=c++11 it expands to a zero-sized array.
rga_buffer_t pat{};
const im_rect pr{0, 0, 0, 0};
const IM_STATUS chk = imcheck_t(s, d, pat, sr, dr, pr, 0);
if (chk != IM_STATUS_NOERROR && chk != IM_STATUS_SUCCESS) {
err = std::string("RGA imcheck failed: ") + imStrError(chk);
return false;
}
return true;
};
IM_STATUS status = IM_STATUS_SUCCESS;
if (need_resize && need_cvt) {
// Try to fuse resize + CSC in a single call.
if (Check(src_buf, dst_buf)) {
rga_buffer_t pat{};
const im_rect sr{0, 0, src_buf.width, src_buf.height};
const im_rect dr{0, 0, dst_buf.width, dst_buf.height};
const im_rect pr{0, 0, 0, 0};
status = improcess(src_buf, dst_buf, pat, sr, dr, pr,
0, nullptr, nullptr, IM_SYNC);
if (status == IM_STATUS_SUCCESS) {
return true;
}
}
// Fallback: 2-step (resize + cvtcolor).
if (!tmp_dma || !tmp_dma->valid()) {
tmp_dma = DmaAlloc(CalcImageSizeStrided(dst_wstride, dst_hstride, frame->format));
if (!tmp_dma || !tmp_dma->valid()) {
err = "DMA alloc for tmp failed";
return false;
}
}
rga_buffer_t tmp = wrapbuffer_fd_t(tmp_dma->fd, out_w, out_h,
dst_wstride, dst_hstride, src_fmt_rga);
if (!Check(src_buf, tmp) || !Check(tmp, dst_buf)) {
return false;
}
status = imresize(src_buf, tmp, 0, 0, 0, 1, nullptr);
if (status == IM_STATUS_SUCCESS) {
status = imcvtcolor(tmp, dst_buf, src_fmt_rga, dst_fmt_rga, IM_COLOR_SPACE_DEFAULT, 1, nullptr);
}
} else if (need_resize) {
if (!Check(src_buf, dst_buf)) {
return false;
}
status = imresize(src_buf, dst_buf, 0, 0, 0, 1, nullptr);
} else if (need_cvt) {
if (!Check(src_buf, dst_buf)) {
return false;
}
status = imcvtcolor(src_buf, dst_buf, src_fmt_rga, dst_fmt_rga, IM_COLOR_SPACE_DEFAULT, 1, nullptr);
}
if (status != IM_STATUS_SUCCESS) {
err = std::string("RGA failed: ") + imStrError(status);
return false;
}
return true;
};
std::string rga_err;
const int src_fd = (src_dma_buf && src_dma_buf->valid()) ? src_dma_buf->fd : frame->dma_fd;
if (src_fd < 0 || !RunRgaOnce(src_fd, rga_err)) {
LogError(std::string("[preprocess] ") + (rga_err.empty() ? "RGA failed" : rga_err));
return false;
}
auto out_frame = std::make_shared<Frame>();
out_frame->width = out_w;
out_frame->height = out_h;
out_frame->format = out_fmt;
out_frame->stride = (out_fmt == PixelFormat::RGB || out_fmt == PixelFormat::BGR)
? (dst_wstride * 3)
: dst_wstride;
out_frame->dma_fd = dma_buf->fd;
out_frame->data = dma_buf->data();
out_frame->data_size = dma_buf->size;
out_frame->data_owner = dma_buf; // DmaBuffer shared_ptr keeps fd alive
out_frame->pts = frame->pts;
out_frame->frame_id = frame->frame_id;
out_frame->det = frame->det;
out_frame->face_det = frame->face_det;
out_frame->face_recog = frame->face_recog;
out_frame->user_meta = frame->user_meta;
SetupPlanes(*out_frame, out_fmt);
PushToDownstream(out_frame);
++processed_;
if (stats_log_ && stats_interval_ > 0 && (processed_ % stats_interval_) == 0) {
LogInfo("[preprocess] rga frame=" + std::to_string(out_frame->frame_id) +
" " + std::to_string(frame->width) + "x" + std::to_string(frame->height) +
" -> " + std::to_string(out_w) + "x" + std::to_string(out_h));
}
return true;
}
#endif
#if defined(RK3588_ENABLE_FFMPEG)
void ProcessSwscale(FramePtr frame) {
PixelFormat out_fmt = (dst_fmt_ != PixelFormat::UNKNOWN) ? dst_fmt_ : frame->format;
int out_w = dst_w_;
int out_h = dst_h_;
if (out_w <= 0) out_w = frame->width;
if (out_h <= 0) out_h = frame->height;
if (keep_ratio_ && dst_w_ > 0 && dst_h_ > 0 && frame->width > 0 && frame->height > 0) {
float scale = std::min(static_cast<float>(dst_w_) / frame->width,
static_cast<float>(dst_h_) / frame->height);
out_w = static_cast<int>(frame->width * scale);
out_h = static_cast<int>(frame->height * scale);
out_w = (out_w + 1) & ~1;
out_h = (out_h + 1) & ~1;
}
if (frame->width != out_w || frame->height != out_h) {
WarnMetaResizeOnce(frame, out_w, out_h);
}
AVPixelFormat src_av_fmt = ToAvFormat(frame->format);
AVPixelFormat dst_av_fmt = ToAvFormat(out_fmt);
if (src_av_fmt == AV_PIX_FMT_NONE || dst_av_fmt == AV_PIX_FMT_NONE) {
PushToDownstream(frame);
return;
}
if (!sws_ctx_ || frame->width != last_src_w_ || frame->height != last_src_h_ ||
src_av_fmt != last_src_fmt_ || dst_av_fmt != last_dst_fmt_) {
if (sws_ctx_) sws_freeContext(sws_ctx_);
sws_ctx_ = sws_getContext(frame->width, frame->height, src_av_fmt,
out_w, out_h, dst_av_fmt,
SWS_BILINEAR, nullptr, nullptr, nullptr);
last_src_w_ = frame->width;
last_src_h_ = frame->height;
last_src_fmt_ = src_av_fmt;
last_dst_fmt_ = dst_av_fmt;
}
if (!sws_ctx_) {
PushToDownstream(frame);
return;
}
size_t out_size = CalcImageSize(out_w, out_h, out_fmt);
auto buffer = std::make_shared<std::vector<uint8_t>>(out_size);
uint8_t* src_data[4] = {nullptr};
int src_linesize[4] = {0};
uint8_t* dst_data[4] = {nullptr};
int dst_linesize[4] = {0};
SetupAvPlanes(frame.get(), src_data, src_linesize);
av_image_fill_arrays(dst_data, dst_linesize, buffer->data(),
dst_av_fmt, out_w, out_h, 1);
sws_scale(sws_ctx_, src_data, src_linesize, 0, frame->height,
dst_data, dst_linesize);
auto out_frame = std::make_shared<Frame>();
out_frame->width = out_w;
out_frame->height = out_h;
out_frame->format = out_fmt;
out_frame->stride = (out_fmt == PixelFormat::RGB || out_fmt == PixelFormat::BGR)
? (out_w * 3)
: out_w;
out_frame->data = buffer->data();
out_frame->data_size = buffer->size();
out_frame->data_owner = buffer;
out_frame->pts = frame->pts;
out_frame->frame_id = frame->frame_id;
out_frame->det = frame->det;
out_frame->face_det = frame->face_det;
out_frame->face_recog = frame->face_recog;
out_frame->user_meta = frame->user_meta;
SetupPlanes(*out_frame, out_fmt);
PushToDownstream(out_frame);
++processed_;
if (stats_log_ && stats_interval_ > 0 && (processed_ % stats_interval_) == 0) {
LogInfo("[preprocess] swscale frame=" + std::to_string(out_frame->frame_id) +
" " + std::to_string(frame->width) + "x" + std::to_string(frame->height) +
" -> " + std::to_string(out_w) + "x" + std::to_string(out_h) +
" id=" + id_);
}
}
static AVPixelFormat ToAvFormat(PixelFormat fmt) {
switch (fmt) {
case PixelFormat::NV12: return AV_PIX_FMT_NV12;
case PixelFormat::YUV420: return AV_PIX_FMT_YUV420P;
case PixelFormat::RGB: return AV_PIX_FMT_RGB24;
case PixelFormat::BGR: return AV_PIX_FMT_BGR24;
default: return AV_PIX_FMT_NONE;
}
}
static void SetupAvPlanes(const Frame* f, uint8_t* data[4], int linesize[4]) {
if (!f->data) return;
if (f->format == PixelFormat::NV12) {
data[0] = f->planes[0].data ? f->planes[0].data : f->data;
data[1] = f->planes[1].data ? f->planes[1].data : (f->data + f->width * f->height);
linesize[0] = f->planes[0].stride > 0 ? f->planes[0].stride : f->width;
linesize[1] = f->planes[1].stride > 0 ? f->planes[1].stride : f->width;
} else if (f->format == PixelFormat::YUV420) {
data[0] = f->planes[0].data ? f->planes[0].data : f->data;
int y_size = f->width * f->height;
int uv_size = y_size / 4;
data[1] = f->planes[1].data ? f->planes[1].data : (f->data + y_size);
data[2] = f->planes[2].data ? f->planes[2].data : (f->data + y_size + uv_size);
linesize[0] = f->planes[0].stride > 0 ? f->planes[0].stride : f->width;
linesize[1] = f->planes[1].stride > 0 ? f->planes[1].stride : f->width / 2;
linesize[2] = f->planes[2].stride > 0 ? f->planes[2].stride : f->width / 2;
} else {
data[0] = f->data;
linesize[0] = f->stride > 0 ? f->stride : f->width * 3;
}
}
#endif
void SetupPlanes(Frame& f, PixelFormat fmt) {
if (!f.data) return;
if (fmt == PixelFormat::NV12) {
f.plane_count = 2;
int y_stride = f.stride > 0 ? f.stride : f.width;
if (y_stride <= 0) y_stride = f.width;
size_t y_bytes = static_cast<size_t>(y_stride) * static_cast<size_t>(f.height);
if (f.data_size > 0) {
const size_t candidate = (f.data_size * 2) / 3; // total = Y + UV = Y*3/2
if (candidate >= y_bytes && candidate <= f.data_size &&
(candidate % static_cast<size_t>(y_stride)) == 0) {
y_bytes = candidate;
}
}
size_t uv_bytes = y_bytes / 2;
if (f.data_size > 0 && y_bytes + uv_bytes > f.data_size) {
uv_bytes = f.data_size > y_bytes ? (f.data_size - y_bytes) : 0;
}
f.planes[0] = {f.data, y_stride, static_cast<int>(y_bytes), 0};
f.planes[1] = {f.data + y_bytes, y_stride, static_cast<int>(uv_bytes), static_cast<int>(y_bytes)};
return;
}
if (fmt == PixelFormat::YUV420) {
f.plane_count = 3;
int y_stride = f.stride > 0 ? f.stride : f.width;
if (y_stride <= 0) y_stride = f.width;
size_t y_bytes = static_cast<size_t>(y_stride) * static_cast<size_t>(f.height);
size_t hstride = static_cast<size_t>(f.height);
if (f.data_size > 0) {
const size_t candidate = (f.data_size * 2) / 3;
if (candidate >= y_bytes && candidate <= f.data_size &&
(candidate % static_cast<size_t>(y_stride)) == 0) {
y_bytes = candidate;
hstride = y_bytes / static_cast<size_t>(y_stride);
}
}
size_t uv_stride = static_cast<size_t>(y_stride) / 2;
size_t uv_h = hstride / 2;
size_t u_bytes = uv_stride * uv_h;
size_t v_bytes = u_bytes;
size_t need = y_bytes + u_bytes + v_bytes;
if (f.data_size > 0 && need > f.data_size) {
// Fallback to tightly packed layout.
y_stride = f.width;
y_bytes = static_cast<size_t>(f.width) * static_cast<size_t>(f.height);
uv_stride = static_cast<size_t>(f.width) / 2;
uv_h = static_cast<size_t>(f.height) / 2;
u_bytes = uv_stride * uv_h;
v_bytes = u_bytes;
}
f.planes[0] = {f.data, y_stride, static_cast<int>(y_bytes), 0};
f.planes[1] = {f.data + y_bytes, static_cast<int>(uv_stride), static_cast<int>(u_bytes), static_cast<int>(y_bytes)};
f.planes[2] = {f.data + y_bytes + u_bytes, static_cast<int>(uv_stride), static_cast<int>(v_bytes), static_cast<int>(y_bytes + u_bytes)};
return;
}
// RGB/BGR
f.plane_count = 1;
int stride_bytes = f.stride > 0 ? f.stride : (f.width * 3);
f.planes[0] = {f.data, stride_bytes, static_cast<int>(f.data_size), 0};
}
std::string id_;
int dst_w_ = 640;
int dst_h_ = 640;
bool keep_ratio_ = false;
PixelFormat dst_fmt_ = PixelFormat::UNKNOWN;
bool dst_packed_ = false;
bool dst_packed_explicit_ = false;
bool use_rga_ = true;
std::string rga_gate_ = "global";
bool stats_log_ = false;
uint64_t stats_interval_ = 100;
@ -929,15 +196,8 @@ private:
std::shared_ptr<SpscQueue<FramePtr>> input_queue_;
std::vector<std::shared_ptr<SpscQueue<FramePtr>>> output_queues_;
std::shared_ptr<IImageProcessor> image_processor_;
uint64_t processed_ = 0;
#if defined(RK3588_ENABLE_FFMPEG)
SwsContext* sws_ctx_ = nullptr;
int last_src_w_ = 0;
int last_src_h_ = 0;
AVPixelFormat last_src_fmt_ = AV_PIX_FMT_NONE;
AVPixelFormat last_dst_fmt_ = AV_PIX_FMT_NONE;
#endif
};
REGISTER_NODE(PreprocessNode, "preprocess");

7
src/graph_manager.cpp
View File

@ -22,6 +22,7 @@ namespace fs = std::filesystem;
#include "utils/logger.h"
#include "utils/simple_json_writer.h"
#include "utils/thread_affinity.h"
#include "hw/hw_factory.h"
namespace rk3588 {
@ -472,6 +473,8 @@ bool Graph::Build(const SimpleJson& graph_cfg, PluginLoader& loader, size_t defa
name_ = name_it->second.AsString(name_);
}
infer_backend_ = HwFactory::CreateInferBackend(graph_cfg_);
// Parse nodes
auto nodes_it = obj.find("nodes");
if (nodes_it == obj.end() || !nodes_it->second.IsArray()) {
@ -491,6 +494,10 @@ bool Graph::Build(const SimpleJson& graph_cfg, PluginLoader& loader, size_t defa
entry.role = node_val.ValueOr<std::string>("role", "");
entry.enabled = node_val.ValueOr<bool>("enable", true);
entry.metrics = std::make_shared<NodeMetrics>();
entry.context.infer_backend = infer_backend_;
if (entry.type == "preprocess") {
entry.context.image_processor = HwFactory::CreateImageProcessor(entry.config);
}
if (entry.id.empty() || entry.type.empty()) {
err = "Node missing id or type";

4
src/hw_factory.cpp
View File

@ -8,8 +8,8 @@ std::shared_ptr<IInferBackend> HwFactory::CreateInferBackend(const SimpleJson& /
return std::make_shared<Rk3588InferBackend>();
}
std::shared_ptr<IImageProcessor> HwFactory::CreateImageProcessor(const SimpleJson& /*config*/) {
return std::make_shared<Rk3588ImageProcessor>();
std::shared_ptr<IImageProcessor> HwFactory::CreateImageProcessor(const SimpleJson& config) {
return std::make_shared<Rk3588ImageProcessor>(config);
}
std::shared_ptr<IDecoder> HwFactory::CreateDecoder(const SimpleJson& /*config*/) {

786
src/hw_image_processor.cpp Normal file
View File

@ -0,0 +1,786 @@
#include "hw/rk3588_defaults.h"
#include "hw/rk3588_defaults.h"
#include <algorithm>
#include <atomic>
#include <condition_variable>
#include <cstdlib>
#include <cstring>
#include <mutex>
#include <unordered_map>
#include "utils/dma_alloc.h"
#include "utils/logger.h"
#if defined(RK3588_ENABLE_RGA)
#include "im2d.hpp"
#include "im2d_buffer.h"
#include "im2d_type.h"
#endif
#if defined(RK3588_ENABLE_FFMPEG)
extern "C" {
#include <libavutil/imgutils.h>
#include <libswscale/swscale.h>
}
#endif
namespace rk3588 {
namespace {
inline int Align16(int v) { return (v + 15) & ~15; }
size_t CalcImageSize(int w, int h, PixelFormat fmt) {
switch (fmt) {
case PixelFormat::NV12:
case PixelFormat::YUV420:
return static_cast<size_t>(w) * h * 3 / 2;
case PixelFormat::RGB:
case PixelFormat::BGR:
return static_cast<size_t>(w) * h * 3;
default:
return 0;
}
}
size_t CalcImageSizeStrided(int wstride, int hstride, PixelFormat fmt) {
if (wstride <= 0 || hstride <= 0) return 0;
const size_t ws = static_cast<size_t>(wstride);
const size_t hs = static_cast<size_t>(hstride);
switch (fmt) {
case PixelFormat::NV12: {
return ws * hs + ws * (hs / 2);
}
case PixelFormat::YUV420: {
const size_t y = ws * hs;
const size_t uv = (ws / 2) * (hs / 2);
return y + uv + uv;
}
case PixelFormat::RGB:
case PixelFormat::BGR:
return ws * hs * 3;
default:
return 0;
}
}
void SetupPlanes(Frame& f, PixelFormat fmt) {
if (!f.data) return;
if (fmt == PixelFormat::NV12) {
f.plane_count = 2;
int y_stride = f.stride > 0 ? f.stride : f.width;
if (y_stride <= 0) y_stride = f.width;
size_t y_bytes = static_cast<size_t>(y_stride) * static_cast<size_t>(f.height);
if (f.data_size > 0) {
const size_t candidate = (f.data_size * 2) / 3;
if (candidate >= y_bytes && candidate <= f.data_size &&
(candidate % static_cast<size_t>(y_stride)) == 0) {
y_bytes = candidate;
}
}
size_t uv_bytes = y_bytes / 2;
if (f.data_size > 0 && y_bytes + uv_bytes > f.data_size) {
uv_bytes = f.data_size > y_bytes ? (f.data_size - y_bytes) : 0;
}
f.planes[0] = {f.data, y_stride, static_cast<int>(y_bytes), 0};
f.planes[1] = {f.data + y_bytes, y_stride, static_cast<int>(uv_bytes), static_cast<int>(y_bytes)};
return;
}
if (fmt == PixelFormat::YUV420) {
f.plane_count = 3;
int y_stride = f.stride > 0 ? f.stride : f.width;
if (y_stride <= 0) y_stride = f.width;
size_t y_bytes = static_cast<size_t>(y_stride) * static_cast<size_t>(f.height);
size_t hstride = static_cast<size_t>(f.height);
if (f.data_size > 0) {
const size_t candidate = (f.data_size * 2) / 3;
if (candidate >= y_bytes && candidate <= f.data_size &&
(candidate % static_cast<size_t>(y_stride)) == 0) {
y_bytes = candidate;
hstride = y_bytes / static_cast<size_t>(y_stride);
}
}
size_t uv_stride = static_cast<size_t>(y_stride) / 2;
size_t uv_h = hstride / 2;
size_t u_bytes = uv_stride * uv_h;
size_t v_bytes = u_bytes;
size_t need = y_bytes + u_bytes + v_bytes;
if (f.data_size > 0 && need > f.data_size) {
y_stride = f.width;
y_bytes = static_cast<size_t>(f.width) * static_cast<size_t>(f.height);
uv_stride = static_cast<size_t>(f.width) / 2;
uv_h = static_cast<size_t>(f.height) / 2;
u_bytes = uv_stride * uv_h;
v_bytes = u_bytes;
}
f.planes[0] = {f.data, y_stride, static_cast<int>(y_bytes), 0};
f.planes[1] = {f.data + y_bytes, static_cast<int>(uv_stride), static_cast<int>(u_bytes), static_cast<int>(y_bytes)};
f.planes[2] = {f.data + y_bytes + u_bytes, static_cast<int>(uv_stride), static_cast<int>(v_bytes), static_cast<int>(y_bytes + u_bytes)};
return;
}
f.plane_count = 1;
int stride_bytes = f.stride > 0 ? f.stride : (f.width * 3);
f.planes[0] = {f.data, stride_bytes, static_cast<int>(f.data_size), 0};
}
#if defined(RK3588_ENABLE_RGA)
int ToRgaFormat(PixelFormat fmt) {
switch (fmt) {
case PixelFormat::NV12: return RK_FORMAT_YCbCr_420_SP;
case PixelFormat::YUV420: return RK_FORMAT_YCbCr_420_P;
case PixelFormat::RGB: return RK_FORMAT_RGB_888;
case PixelFormat::BGR: return RK_FORMAT_BGR_888;
default: return RK_FORMAT_UNKNOWN;
}
}
std::atomic<int>& GlobalRgaMaxInflightRef() {
static std::atomic<int> v{2};
return v;
}
int GlobalRgaMaxInflight() {
static std::once_flag once;
std::call_once(once, []() {
const char* s = std::getenv("RK3588_RGA_MAX_INFLIGHT");
if (!s || !*s) return;
try {
int v = std::stoi(s);
if (v > 0 && v <= 32) GlobalRgaMaxInflightRef().store(v);
} catch (...) {
LogWarn(std::string("[image_processor] invalid RK3588_RGA_MAX_INFLIGHT='") + s + "'");
}
});
int v = GlobalRgaMaxInflightRef().load();
return v > 0 ? v : 1;
}
class RgaGate {
public:
explicit RgaGate(int max_inflight) : max_inflight_(max_inflight > 0 ? max_inflight : 1) {}
void Acquire() {
std::unique_lock<std::mutex> lock(mu_);
cv_.wait(lock, [&]() { return in_flight_ < max_inflight_; });
++in_flight_;
}
void Release() {
std::lock_guard<std::mutex> lock(mu_);
if (in_flight_ > 0) {
--in_flight_;
}
cv_.notify_one();
}
void SetMaxInflight(int v) {
if (v <= 0) return;
if (v > 32) v = 32;
{
std::lock_guard<std::mutex> lock(mu_);
max_inflight_ = v;
}
cv_.notify_all();
}
int MaxInflight() const {
std::lock_guard<std::mutex> lock(mu_);
return max_inflight_;
}
private:
mutable std::mutex mu_;
std::condition_variable cv_;
int in_flight_ = 0;
int max_inflight_ = 1;
};
class RgaGateRegistry {
public:
static RgaGateRegistry& Instance() {
static RgaGateRegistry* inst = new RgaGateRegistry();
return *inst;
}
RgaGate& Get(const std::string& key) {
std::lock_guard<std::mutex> lock(mu_);
auto it = gates_.find(key);
if (it != gates_.end()) return *it->second;
auto gate = std::make_unique<RgaGate>(GlobalRgaMaxInflight());
RgaGate& ref = *gate;
gates_.emplace(key, std::move(gate));
return ref;
}
private:
std::mutex mu_;
std::unordered_map<std::string, std::unique_ptr<RgaGate>> gates_;
};
RgaGate& GetRgaGate(const std::string& key) {
const std::string k = key.empty() ? "global" : key;
return RgaGateRegistry::Instance().Get(k);
}
class ScopedRgaGate {
public:
explicit ScopedRgaGate(const std::string& key) : gate_(&GetRgaGate(key)) { gate_->Acquire(); }
~ScopedRgaGate() { gate_->Release(); }
ScopedRgaGate(const ScopedRgaGate&) = delete;
ScopedRgaGate& operator=(const ScopedRgaGate&) = delete;
private:
RgaGate* gate_ = nullptr;
};
void EnsureRgaInitializedOnce() {
static std::once_flag once;
std::call_once(once, []() {
const IM_STATUS st = imcheckHeader();
if (st != IM_STATUS_NOERROR && st != IM_STATUS_SUCCESS) {
LogWarn(std::string("[image_processor] imcheckHeader failed: ") + imStrError(st));
}
});
}
bool CopyToStridedBuffer(const Frame& src, uint8_t* dst, size_t dst_size,
int dst_wstride, int dst_hstride) {
if (!dst || dst_size == 0) return false;
if (dst_wstride <= 0 || dst_hstride <= 0) return false;
std::memset(dst, 0, dst_size);
const int w = src.width;
const int h = src.height;
if (w <= 0 || h <= 0) return false;
if (src.format == PixelFormat::NV12) {
const size_t y_bytes = static_cast<size_t>(dst_wstride) * dst_hstride;
const size_t uv_bytes = static_cast<size_t>(dst_wstride) * (dst_hstride / 2);
if (y_bytes + uv_bytes > dst_size) return false;
const uint8_t* src_y = src.planes[0].data ? src.planes[0].data : src.data;
const uint8_t* src_uv = src.planes[1].data ? src.planes[1].data : nullptr;
const int src_y_stride = src.planes[0].stride > 0 ? src.planes[0].stride : w;
const int src_uv_stride = src.planes[1].stride > 0 ? src.planes[1].stride : w;
if (!src_y) return false;
if (!src_uv) {
if (!src.data) return false;
src_uv = src.data + static_cast<size_t>(src_y_stride) * static_cast<size_t>(h);
}
for (int row = 0; row < h; ++row) {
std::memcpy(dst + static_cast<size_t>(row) * dst_wstride,
src_y + static_cast<size_t>(row) * src_y_stride,
static_cast<size_t>(w));
}
uint8_t* dst_uv = dst + y_bytes;
const int uv_rows = h / 2;
for (int row = 0; row < uv_rows; ++row) {
std::memcpy(dst_uv + static_cast<size_t>(row) * dst_wstride,
src_uv + static_cast<size_t>(row) * src_uv_stride,
static_cast<size_t>(w));
}
return true;
}
if (src.format == PixelFormat::YUV420) {
const size_t y_bytes = static_cast<size_t>(dst_wstride) * dst_hstride;
const size_t uv_stride = static_cast<size_t>(dst_wstride) / 2;
const size_t uv_h = static_cast<size_t>(dst_hstride) / 2;
const size_t u_bytes = uv_stride * uv_h;
const size_t v_bytes = u_bytes;
if (y_bytes + u_bytes + v_bytes > dst_size) return false;
const uint8_t* src_y = src.planes[0].data ? src.planes[0].data : src.data;
const uint8_t* src_u = src.planes[1].data ? src.planes[1].data : nullptr;
const uint8_t* src_v = src.planes[2].data ? src.planes[2].data : nullptr;
const int src_y_stride = src.planes[0].stride > 0 ? src.planes[0].stride : w;
const int src_u_stride = src.planes[1].stride > 0 ? src.planes[1].stride : (w / 2);
const int src_v_stride = src.planes[2].stride > 0 ? src.planes[2].stride : (w / 2);
if (!src_y || !src_u || !src_v) return false;
for (int row = 0; row < h; ++row) {
std::memcpy(dst + static_cast<size_t>(row) * dst_wstride,
src_y + static_cast<size_t>(row) * src_y_stride,
static_cast<size_t>(w));
}
uint8_t* dst_u = dst + y_bytes;
uint8_t* dst_v = dst + y_bytes + u_bytes;
const int uv_rows = h / 2;
const int uv_cols = w / 2;
for (int row = 0; row < uv_rows; ++row) {
std::memcpy(dst_u + static_cast<size_t>(row) * uv_stride,
src_u + static_cast<size_t>(row) * src_u_stride,
static_cast<size_t>(uv_cols));
std::memcpy(dst_v + static_cast<size_t>(row) * uv_stride,
src_v + static_cast<size_t>(row) * src_v_stride,
static_cast<size_t>(uv_cols));
}
return true;
}
if (src.format == PixelFormat::RGB || src.format == PixelFormat::BGR) {
const size_t need = static_cast<size_t>(dst_wstride) * dst_hstride * 3;
if (need > dst_size) return false;
const uint8_t* src_rgb = src.planes[0].data ? src.planes[0].data : src.data;
const int src_stride = src.planes[0].stride > 0
? src.planes[0].stride
: (src.stride > 0 ? src.stride : w * 3);
if (!src_rgb) return false;
const size_t dst_stride = static_cast<size_t>(dst_wstride) * 3;
const size_t row_bytes = static_cast<size_t>(w) * 3;
for (int row = 0; row < h; ++row) {
std::memcpy(dst + static_cast<size_t>(row) * dst_stride,
src_rgb + static_cast<size_t>(row) * src_stride,
row_bytes);
}
return true;
}
return false;
}
class RgaImageProcessor : public IImageProcessor {
public:
explicit RgaImageProcessor(const SimpleJson& config) {
rga_gate_ = config.ValueOr<std::string>("rga_gate", "global");
const int rga_max_inflight = config.ValueOr<int>("rga_max_inflight", 0);
if (rga_max_inflight > 0) {
GetRgaGate(rga_gate_).SetMaxInflight(rga_max_inflight);
}
if (config.Find("dst_packed")) {
dst_packed_ = config.ValueOr<bool>("dst_packed", false);
dst_packed_explicit_ = true;
} else {
dst_packed_ = false;
dst_packed_explicit_ = false;
}
}
Status Resize(const Frame& src, Frame& dst) override {
EnsureRgaInitializedOnce();
if (dst.width <= 0 || dst.height <= 0) return FailStatus("invalid output size");
if (dst.format == PixelFormat::UNKNOWN) return FailStatus("invalid output format");
const PixelFormat out_fmt = dst.format;
const int out_w = dst.width;
const int out_h = dst.height;
int src_fmt_rga = ToRgaFormat(src.format);
int dst_fmt_rga = ToRgaFormat(out_fmt);
bool need_cvt = (src_fmt_rga != dst_fmt_rga);
bool need_resize = (src.width != out_w || src.height != out_h);
if (src_fmt_rga == RK_FORMAT_UNKNOWN || dst_fmt_rga == RK_FORMAT_UNKNOWN) {
return FailStatus("unsupported format for RGA");
}
int src_wstride = Align16(src.width);
int src_hstride = Align16(src.height);
if (src.format == PixelFormat::NV12 || src.format == PixelFormat::YUV420) {
const int y_stride = src.planes[0].stride > 0 ? src.planes[0].stride
: (src.stride > 0 ? src.stride : src.width);
if (y_stride > 0) src_wstride = y_stride;
if (src.planes[0].size > 0 && y_stride > 0) {
const int hs = src.planes[0].size / y_stride;
if (hs >= src.height) src_hstride = hs;
}
} else if (src.format == PixelFormat::RGB || src.format == PixelFormat::BGR) {
const int stride_bytes = src.planes[0].stride > 0 ? src.planes[0].stride
: (src.stride > 0 ? src.stride : src.width * 3);
if (stride_bytes > 0 && (stride_bytes % 3) == 0) {
src_wstride = stride_bytes / 3;
}
if (src.planes[0].size > 0 && stride_bytes > 0) {
const int hs = src.planes[0].size / stride_bytes;
if (hs >= src.height) src_hstride = hs;
}
}
int dst_wstride = Align16(out_w);
const bool want_packed_rgb = (out_fmt == PixelFormat::RGB || out_fmt == PixelFormat::BGR) &&
(!dst_packed_explicit_ || dst_packed_);
if (want_packed_rgb) {
dst_wstride = out_w;
}
int dst_hstride = Align16(out_h);
size_t out_size = CalcImageSizeStrided(dst_wstride, dst_hstride, out_fmt);
if (out_size == 0) {
return FailStatus("invalid output size for RGA");
}
auto dma_buf = DmaAlloc(out_size);
if (!dma_buf || !dma_buf->valid()) {
return FailStatus("DMA alloc failed");
}
rga_buffer_t src_buf{};
rga_buffer_t dst_buf{};
DmaBufferPtr src_dma_buf;
DmaBufferPtr tmp_dma;
const bool can_cpu_read_src = (src.data != nullptr && src.data_size > 0);
auto CopySrcToDmaIfPossible = [&]() -> bool {
if (!can_cpu_read_src) return false;
const int copy_wstride = Align16(src.width);
const int copy_hstride = Align16(src.height);
const size_t src_size = CalcImageSizeStrided(copy_wstride, copy_hstride, src.format);
src_dma_buf = DmaAlloc(src_size);
if (!src_dma_buf || !src_dma_buf->valid()) {
return false;
}
if (src.dma_fd >= 0) {
DmaSyncStartFd(src.dma_fd);
}
DmaSyncStartFd(src_dma_buf->fd);
const bool ok = CopyToStridedBuffer(src, src_dma_buf->data(), src_dma_buf->size,
copy_wstride, copy_hstride);
DmaSyncEndFd(src_dma_buf->fd);
if (src.dma_fd >= 0) {
DmaSyncEndFd(src.dma_fd);
}
if (!ok) {
src_dma_buf.reset();
return false;
}
src_wstride = copy_wstride;
src_hstride = copy_hstride;
return true;
};
if (src.dma_fd < 0) {
if (!CopySrcToDmaIfPossible()) {
return FailStatus("no dma_fd and src copy failed");
}
}
auto RunRgaOnce = [&](int src_fd, std::string& err) -> bool {
ScopedRgaGate guard(rga_gate_);
src_buf = wrapbuffer_fd_t(src_fd, src.width, src.height,
src_wstride, src_hstride, src_fmt_rga);
dst_buf = wrapbuffer_fd_t(dma_buf->fd, out_w, out_h,
dst_wstride, dst_hstride, dst_fmt_rga);
auto Check = [&](const rga_buffer_t& s, const rga_buffer_t& d) -> bool {
const im_rect sr{0, 0, s.width, s.height};
const im_rect dr{0, 0, d.width, d.height};
rga_buffer_t pat{};
const im_rect pr{0, 0, 0, 0};
const IM_STATUS chk = imcheck_t(s, d, pat, sr, dr, pr, 0);
if (chk != IM_STATUS_NOERROR && chk != IM_STATUS_SUCCESS) {
err = std::string("RGA imcheck failed: ") + imStrError(chk);
return false;
}
return true;
};
IM_STATUS status = IM_STATUS_SUCCESS;
if (need_resize && need_cvt) {
if (Check(src_buf, dst_buf)) {
rga_buffer_t pat{};
const im_rect sr{0, 0, src_buf.width, src_buf.height};
const im_rect dr{0, 0, dst_buf.width, dst_buf.height};
const im_rect pr{0, 0, 0, 0};
status = improcess(src_buf, dst_buf, pat, sr, dr, pr,
0, nullptr, nullptr, IM_SYNC);
if (status == IM_STATUS_SUCCESS) {
return true;
}
}
if (!tmp_dma || !tmp_dma->valid()) {
tmp_dma = DmaAlloc(CalcImageSizeStrided(dst_wstride, dst_hstride, src.format));
if (!tmp_dma || !tmp_dma->valid()) {
err = "DMA alloc for tmp failed";
return false;
}
}
rga_buffer_t tmp = wrapbuffer_fd_t(tmp_dma->fd, out_w, out_h,
dst_wstride, dst_hstride, src_fmt_rga);
if (!Check(src_buf, tmp) || !Check(tmp, dst_buf)) {
return false;
}
status = imresize(src_buf, tmp, 0, 0, 0, 1, nullptr);
if (status == IM_STATUS_SUCCESS) {
status = imcvtcolor(tmp, dst_buf, src_fmt_rga, dst_fmt_rga, IM_COLOR_SPACE_DEFAULT, 1, nullptr);
}
} else if (need_resize) {
if (!Check(src_buf, dst_buf)) {
return false;
}
status = imresize(src_buf, dst_buf, 0, 0, 0, 1, nullptr);
} else if (need_cvt) {
if (!Check(src_buf, dst_buf)) {
return false;
}
status = imcvtcolor(src_buf, dst_buf, src_fmt_rga, dst_fmt_rga, IM_COLOR_SPACE_DEFAULT, 1, nullptr);
}
if (status != IM_STATUS_SUCCESS) {
err = std::string("RGA failed: ") + imStrError(status);
return false;
}
return true;
};
std::string rga_err;
const int src_fd = (src_dma_buf && src_dma_buf->valid()) ? src_dma_buf->fd : src.dma_fd;
if (src_fd < 0 || !RunRgaOnce(src_fd, rga_err)) {
return FailStatus(rga_err.empty() ? "RGA failed" : rga_err);
}
dst.stride = (out_fmt == PixelFormat::RGB || out_fmt == PixelFormat::BGR)
? (dst_wstride * 3)
: dst_wstride;
dst.dma_fd = dma_buf->fd;
dst.data = dma_buf->data();
dst.data_size = dma_buf->size;
dst.data_owner = dma_buf;
SetupPlanes(dst, out_fmt);
return OkStatus();
}
Status CvtColor(const Frame& src, Frame& dst, PixelFormat dst_format) override {
dst.width = src.width;
dst.height = src.height;
dst.format = dst_format;
return Resize(src, dst);
}
Status Normalize(const Frame& /*src*/, std::vector<float>& /*out*/,
const std::vector<float>& /*mean*/,
const std::vector<float>& /*std*/) override {
return FailStatus("not implemented");
}
int MaxInflight() const { return GetRgaGate(rga_gate_).MaxInflight(); }
const std::string& GateKey() const { return rga_gate_; }
private:
std::string rga_gate_ = "global";
bool dst_packed_ = false;
bool dst_packed_explicit_ = false;
};
#endif
#if defined(RK3588_ENABLE_FFMPEG)
AVPixelFormat ToAvFormat(PixelFormat fmt) {
switch (fmt) {
case PixelFormat::NV12: return AV_PIX_FMT_NV12;
case PixelFormat::YUV420: return AV_PIX_FMT_YUV420P;
case PixelFormat::RGB: return AV_PIX_FMT_RGB24;
case PixelFormat::BGR: return AV_PIX_FMT_BGR24;
default: return AV_PIX_FMT_NONE;
}
}
void SetupAvPlanes(const Frame* f, uint8_t* data[4], int linesize[4]) {
if (!f || !f->data) return;
if (f->format == PixelFormat::NV12) {
data[0] = f->planes[0].data ? f->planes[0].data : f->data;
data[1] = f->planes[1].data ? f->planes[1].data : (f->data + f->width * f->height);
linesize[0] = f->planes[0].stride > 0 ? f->planes[0].stride : f->width;
linesize[1] = f->planes[1].stride > 0 ? f->planes[1].stride : f->width;
} else if (f->format == PixelFormat::YUV420) {
data[0] = f->planes[0].data ? f->planes[0].data : f->data;
int y_size = f->width * f->height;
int uv_size = y_size / 4;
data[1] = f->planes[1].data ? f->planes[1].data : (f->data + y_size);
data[2] = f->planes[2].data ? f->planes[2].data : (f->data + y_size + uv_size);
linesize[0] = f->planes[0].stride > 0 ? f->planes[0].stride : f->width;
linesize[1] = f->planes[1].stride > 0 ? f->planes[1].stride : f->width / 2;
linesize[2] = f->planes[2].stride > 0 ? f->planes[2].stride : f->width / 2;
} else {
data[0] = f->data;
linesize[0] = f->stride > 0 ? f->stride : f->width * 3;
}
}
class SwscaleImageProcessor : public IImageProcessor {
public:
~SwscaleImageProcessor() override {
if (sws_ctx_) {
sws_freeContext(sws_ctx_);
sws_ctx_ = nullptr;
}
}
Status Resize(const Frame& src, Frame& dst) override {
if (dst.width <= 0 || dst.height <= 0) return FailStatus("invalid output size");
if (dst.format == PixelFormat::UNKNOWN) return FailStatus("invalid output format");
AVPixelFormat src_av_fmt = ToAvFormat(src.format);
AVPixelFormat dst_av_fmt = ToAvFormat(dst.format);
if (src_av_fmt == AV_PIX_FMT_NONE || dst_av_fmt == AV_PIX_FMT_NONE) {
return FailStatus("unsupported format");
}
if (!sws_ctx_ || src.width != last_src_w_ || src.height != last_src_h_ ||
src_av_fmt != last_src_fmt_ || dst_av_fmt != last_dst_fmt_ ||
dst.width != last_dst_w_ || dst.height != last_dst_h_) {
if (sws_ctx_) sws_freeContext(sws_ctx_);
sws_ctx_ = sws_getContext(src.width, src.height, src_av_fmt,
dst.width, dst.height, dst_av_fmt,
SWS_BILINEAR, nullptr, nullptr, nullptr);
last_src_w_ = src.width;
last_src_h_ = src.height;
last_src_fmt_ = src_av_fmt;
last_dst_fmt_ = dst_av_fmt;
last_dst_w_ = dst.width;
last_dst_h_ = dst.height;
}
if (!sws_ctx_) {
return FailStatus("sws_getContext failed");
}
size_t out_size = CalcImageSize(dst.width, dst.height, dst.format);
if (out_size == 0) return FailStatus("invalid output size");
auto buffer = std::make_shared<std::vector<uint8_t>>(out_size);
uint8_t* src_data[4] = {nullptr};
int src_linesize[4] = {0};
uint8_t* dst_data[4] = {nullptr};
int dst_linesize[4] = {0};
SetupAvPlanes(&src, src_data, src_linesize);
av_image_fill_arrays(dst_data, dst_linesize, buffer->data(),
dst_av_fmt, dst.width, dst.height, 1);
sws_scale(sws_ctx_, src_data, src_linesize, 0, src.height,
dst_data, dst_linesize);
dst.stride = (dst.format == PixelFormat::RGB || dst.format == PixelFormat::BGR)
? (dst.width * 3)
: dst.width;
dst.data = buffer->data();
dst.data_size = buffer->size();
dst.data_owner = buffer;
SetupPlanes(dst, dst.format);
return OkStatus();
}
Status CvtColor(const Frame& src, Frame& dst, PixelFormat dst_format) override {
dst.width = src.width;
dst.height = src.height;
dst.format = dst_format;
return Resize(src, dst);
}
Status Normalize(const Frame& /*src*/, std::vector<float>& /*out*/,
const std::vector<float>& /*mean*/,
const std::vector<float>& /*std*/) override {
return FailStatus("not implemented");
}
private:
SwsContext* sws_ctx_ = nullptr;
int last_src_w_ = 0;
int last_src_h_ = 0;
int last_dst_w_ = 0;
int last_dst_h_ = 0;
AVPixelFormat last_src_fmt_ = AV_PIX_FMT_NONE;
AVPixelFormat last_dst_fmt_ = AV_PIX_FMT_NONE;
};
#else
class SwscaleImageProcessor : public IImageProcessor {
public:
Status Resize(const Frame& /*src*/, Frame& /*dst*/) override {
return FailStatus("swscale disabled");
}
Status CvtColor(const Frame& /*src*/, Frame& /*dst*/, PixelFormat /*dst_format*/) override {
return FailStatus("swscale disabled");
}
Status Normalize(const Frame& /*src*/, std::vector<float>& /*out*/,
const std::vector<float>& /*mean*/,
const std::vector<float>& /*std*/) override {
return FailStatus("swscale disabled");
}
};
#endif
} // namespace
Rk3588ImageProcessor::Rk3588ImageProcessor(const SimpleJson& config) {
use_rga_ = config.ValueOr<bool>("use_rga", true);
#if defined(RK3588_ENABLE_RGA)
if (use_rga_) {
impl_ = std::make_shared<RgaImageProcessor>(config);
}
#endif
if (!impl_) {
impl_ = std::make_shared<SwscaleImageProcessor>();
}
}
Status Rk3588ImageProcessor::Resize(const Frame& src, Frame& dst) {
if (!impl_) return FailStatus("no image processor backend");
return impl_->Resize(src, dst);
}
Status Rk3588ImageProcessor::CvtColor(const Frame& src, Frame& dst, PixelFormat dst_format) {
if (!impl_) return FailStatus("no image processor backend");
return impl_->CvtColor(src, dst, dst_format);
}
Status Rk3588ImageProcessor::Normalize(const Frame& src, std::vector<float>& out,
const std::vector<float>& mean,
const std::vector<float>& std) {
if (!impl_) return FailStatus("no image processor backend");
return impl_->Normalize(src, out, mean, std);
}
bool Rk3588ImageProcessor::IsUsingRga() const { return use_rga_; }
int Rk3588ImageProcessor::RgaMaxInflight() const {
#if defined(RK3588_ENABLE_RGA)
if (!use_rga_) return 0;
auto rga = std::dynamic_pointer_cast<RgaImageProcessor>(impl_);
if (!rga) return 0;
return rga->MaxInflight();
#else
return 0;
#endif
}
std::string Rk3588ImageProcessor::RgaGateKey() const {
#if defined(RK3588_ENABLE_RGA)
if (!use_rga_) return "";
auto rga = std::dynamic_pointer_cast<RgaImageProcessor>(impl_);
if (!rga) return "";
return rga->GateKey();
#else
return "";
#endif
}
} // namespace rk3588

4
tests/CMakeLists.txt
View File

@ -36,7 +36,11 @@ add_executable(rk3588_gtests
test_simple_json.cpp
test_config_expand.cpp
test_hw_factory.cpp
test_infer_backend.cpp
test_image_processor.cpp
${CMAKE_SOURCE_DIR}/src/hw_factory.cpp
${CMAKE_SOURCE_DIR}/src/hw_image_processor.cpp
${CMAKE_SOURCE_DIR}/src/ai_scheduler.cpp
${CMAKE_SOURCE_DIR}/src/utils/config_expand.cpp
)

94
tests/test_image_processor.cpp Normal file
View File

@ -0,0 +1,94 @@
#include <gtest/gtest.h>
#include <vector>
#include "frame/frame.h"
#include "hw/hw_factory.h"
namespace rk3588 {
namespace {
Frame MakeNv12Frame(int w, int h) {
Frame f;
f.width = w;
f.height = h;
f.format = PixelFormat::NV12;
const size_t size = static_cast<size_t>(w) * h * 3 / 2;
auto buf = std::make_shared<std::vector<uint8_t>>(size, 0);
const size_t y_bytes = static_cast<size_t>(w) * h;
for (size_t i = 0; i < y_bytes; ++i) {
(*buf)[i] = 80;
}
for (size_t i = y_bytes; i < size; ++i) {
(*buf)[i] = 128;
}
f.data = buf->data();
f.data_size = buf->size();
f.data_owner = buf;
f.plane_count = 2;
f.planes[0] = {f.data, w, static_cast<int>(y_bytes), 0};
f.planes[1] = {f.data + y_bytes, w, static_cast<int>(size - y_bytes), static_cast<int>(y_bytes)};
return f;
}
TEST(ImageProcessorTest, RgaVsSwscale_OutputShape) {
#if defined(RK3588_ENABLE_RGA) && defined(RK3588_ENABLE_FFMPEG)
SimpleJson rga_cfg(SimpleJson::Object{{"use_rga", true}});
SimpleJson sw_cfg(SimpleJson::Object{{"use_rga", false}});
auto rga = HwFactory::CreateImageProcessor(rga_cfg);
auto sws = HwFactory::CreateImageProcessor(sw_cfg);
ASSERT_NE(rga, nullptr);
ASSERT_NE(sws, nullptr);
Frame src = MakeNv12Frame(64, 48);
Frame dst_rga;
dst_rga.width = 128;
dst_rga.height = 96;
dst_rga.format = PixelFormat::RGB;
EXPECT_TRUE(rga->Resize(src, dst_rga));
EXPECT_EQ(dst_rga.width, 128);
EXPECT_EQ(dst_rga.height, 96);
Frame dst_sws;
dst_sws.width = 128;
dst_sws.height = 96;
dst_sws.format = PixelFormat::RGB;
EXPECT_TRUE(sws->Resize(src, dst_sws));
EXPECT_EQ(dst_sws.width, 128);
EXPECT_EQ(dst_sws.height, 96);
#else
GTEST_SKIP() << "RGA/FFmpeg not enabled";
#endif
}
TEST(ImageProcessorTest, ColorConversion_Nv12ToRgb) {
#if defined(RK3588_ENABLE_FFMPEG)
SimpleJson cfg(SimpleJson::Object{{"use_rga", false}});
auto proc = HwFactory::CreateImageProcessor(cfg);
ASSERT_NE(proc, nullptr);
Frame src = MakeNv12Frame(2, 2);
Frame dst;
dst.width = 2;
dst.height = 2;
dst.format = PixelFormat::RGB;
Status st = proc->CvtColor(src, dst, PixelFormat::RGB);
EXPECT_TRUE(st.IsOk());
EXPECT_EQ(dst.width, 2);
EXPECT_EQ(dst.height, 2);
EXPECT_EQ(dst.format, PixelFormat::RGB);
EXPECT_NE(dst.data, nullptr);
EXPECT_GT(dst.data_size, 0u);
#else
GTEST_SKIP() << "FFmpeg not enabled";
#endif
}
} // namespace
} // namespace rk3588

53
tests/test_infer_backend.cpp Normal file
View File

@ -0,0 +1,53 @@
#include <gtest/gtest.h>
#include <filesystem>
#include "hw/hw_factory.h"
#include "hw/rk3588_defaults.h"
namespace rk3588 {
namespace {
TEST(InferBackendTest, LoadModelSmoke) {
SimpleJson config(SimpleJson::Object{});
auto infer = HwFactory::CreateInferBackend(config);
ASSERT_NE(infer, nullptr);
std::filesystem::path repo_root = std::filesystem::path(__FILE__).parent_path().parent_path();
std::filesystem::path model_path = repo_root / "models" / "RetinaFace_mobile320.rknn";
if (!std::filesystem::exists(model_path)) {
GTEST_SKIP() << "model file not found: " << model_path.string();
}
std::string err;
ModelHandle handle = infer->LoadModel(model_path.string(), err);
#if defined(RK3588_ENABLE_RKNN)
EXPECT_NE(handle, kInvalidModelHandle) << err;
if (handle != kInvalidModelHandle) {
infer->UnloadModel(handle);
}
#else
EXPECT_EQ(handle, kInvalidModelHandle);
EXPECT_FALSE(err.empty());
#endif
}
TEST(InferBackendTest, BorrowedInputUsesDmaFd) {
SimpleJson config(SimpleJson::Object{});
auto infer = HwFactory::CreateInferBackend(config);
auto rk = std::dynamic_pointer_cast<Rk3588InferBackend>(infer);
ASSERT_NE(rk, nullptr);
InferInput input;
input.dma_fd = 7;
input.dma_offset = 16;
rk->InferBorrowed(kInvalidModelHandle, input);
InferInput last = rk->GetLastBorrowedInput();
EXPECT_EQ(last.dma_fd, 7);
EXPECT_EQ(last.dma_offset, 16);
}
} // namespace
} // namespace rk3588