790 lines
28 KiB
C++
790 lines
28 KiB
C++
#include "hw/rk3588_defaults.h"
|
|
|
|
#include <algorithm>
|
|
#include <atomic>
|
|
#include <condition_variable>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <mutex>
|
|
#include <unordered_map>
|
|
|
|
#include "utils/dma_alloc.h"
|
|
#include "utils/logger.h"
|
|
|
|
#if defined(RK3588_ENABLE_RGA)
|
|
#include "im2d.hpp"
|
|
#include "im2d_common.h"
|
|
#include "im2d_buffer.h"
|
|
#include "im2d_type.h"
|
|
#endif
|
|
|
|
#if defined(RK3588_ENABLE_FFMPEG)
|
|
extern "C" {
|
|
#include <libavutil/imgutils.h>
|
|
#include <libswscale/swscale.h>
|
|
}
|
|
#endif
|
|
|
|
namespace rk3588 {
|
|
namespace {
|
|
|
|
inline int Align16(int v) { return (v + 15) & ~15; }
|
|
|
|
size_t CalcImageSize(int w, int h, PixelFormat fmt) {
|
|
switch (fmt) {
|
|
case PixelFormat::NV12:
|
|
case PixelFormat::YUV420:
|
|
return static_cast<size_t>(w) * h * 3 / 2;
|
|
case PixelFormat::RGB:
|
|
case PixelFormat::BGR:
|
|
return static_cast<size_t>(w) * h * 3;
|
|
default:
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
size_t CalcImageSizeStrided(int wstride, int hstride, PixelFormat fmt) {
|
|
if (wstride <= 0 || hstride <= 0) return 0;
|
|
const size_t ws = static_cast<size_t>(wstride);
|
|
const size_t hs = static_cast<size_t>(hstride);
|
|
switch (fmt) {
|
|
case PixelFormat::NV12: {
|
|
return ws * hs + ws * (hs / 2);
|
|
}
|
|
case PixelFormat::YUV420: {
|
|
const size_t y = ws * hs;
|
|
const size_t uv = (ws / 2) * (hs / 2);
|
|
return y + uv + uv;
|
|
}
|
|
case PixelFormat::RGB:
|
|
case PixelFormat::BGR:
|
|
return ws * hs * 3;
|
|
default:
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
void SetupPlanes(Frame& f, PixelFormat fmt) {
|
|
if (!f.data) return;
|
|
|
|
if (fmt == PixelFormat::NV12) {
|
|
f.plane_count = 2;
|
|
int y_stride = f.stride > 0 ? f.stride : f.width;
|
|
if (y_stride <= 0) y_stride = f.width;
|
|
|
|
size_t y_bytes = static_cast<size_t>(y_stride) * static_cast<size_t>(f.height);
|
|
if (f.data_size > 0) {
|
|
const size_t candidate = (f.data_size * 2) / 3;
|
|
if (candidate >= y_bytes && candidate <= f.data_size &&
|
|
(candidate % static_cast<size_t>(y_stride)) == 0) {
|
|
y_bytes = candidate;
|
|
}
|
|
}
|
|
|
|
size_t uv_bytes = y_bytes / 2;
|
|
if (f.data_size > 0 && y_bytes + uv_bytes > f.data_size) {
|
|
uv_bytes = f.data_size > y_bytes ? (f.data_size - y_bytes) : 0;
|
|
}
|
|
|
|
f.planes[0] = {f.data, y_stride, static_cast<int>(y_bytes), 0};
|
|
f.planes[1] = {f.data + y_bytes, y_stride, static_cast<int>(uv_bytes), static_cast<int>(y_bytes)};
|
|
f.SyncBufferFromFrame();
|
|
return;
|
|
}
|
|
|
|
if (fmt == PixelFormat::YUV420) {
|
|
f.plane_count = 3;
|
|
int y_stride = f.stride > 0 ? f.stride : f.width;
|
|
if (y_stride <= 0) y_stride = f.width;
|
|
|
|
size_t y_bytes = static_cast<size_t>(y_stride) * static_cast<size_t>(f.height);
|
|
size_t hstride = static_cast<size_t>(f.height);
|
|
if (f.data_size > 0) {
|
|
const size_t candidate = (f.data_size * 2) / 3;
|
|
if (candidate >= y_bytes && candidate <= f.data_size &&
|
|
(candidate % static_cast<size_t>(y_stride)) == 0) {
|
|
y_bytes = candidate;
|
|
hstride = y_bytes / static_cast<size_t>(y_stride);
|
|
}
|
|
}
|
|
|
|
size_t uv_stride = static_cast<size_t>(y_stride) / 2;
|
|
size_t uv_h = hstride / 2;
|
|
size_t u_bytes = uv_stride * uv_h;
|
|
size_t v_bytes = u_bytes;
|
|
size_t need = y_bytes + u_bytes + v_bytes;
|
|
|
|
if (f.data_size > 0 && need > f.data_size) {
|
|
y_stride = f.width;
|
|
y_bytes = static_cast<size_t>(f.width) * static_cast<size_t>(f.height);
|
|
uv_stride = static_cast<size_t>(f.width) / 2;
|
|
uv_h = static_cast<size_t>(f.height) / 2;
|
|
u_bytes = uv_stride * uv_h;
|
|
v_bytes = u_bytes;
|
|
}
|
|
|
|
f.planes[0] = {f.data, y_stride, static_cast<int>(y_bytes), 0};
|
|
f.planes[1] = {f.data + y_bytes, static_cast<int>(uv_stride), static_cast<int>(u_bytes), static_cast<int>(y_bytes)};
|
|
f.planes[2] = {f.data + y_bytes + u_bytes, static_cast<int>(uv_stride), static_cast<int>(v_bytes), static_cast<int>(y_bytes + u_bytes)};
|
|
f.SyncBufferFromFrame();
|
|
return;
|
|
}
|
|
|
|
f.plane_count = 1;
|
|
int stride_bytes = f.stride > 0 ? f.stride : (f.width * 3);
|
|
f.planes[0] = {f.data, stride_bytes, static_cast<int>(f.data_size), 0};
|
|
f.SyncBufferFromFrame();
|
|
}
|
|
|
|
#if defined(RK3588_ENABLE_RGA)
|
|
int ToRgaFormat(PixelFormat fmt) {
|
|
switch (fmt) {
|
|
case PixelFormat::NV12: return RK_FORMAT_YCbCr_420_SP;
|
|
case PixelFormat::YUV420: return RK_FORMAT_YCbCr_420_P;
|
|
case PixelFormat::RGB: return RK_FORMAT_RGB_888;
|
|
case PixelFormat::BGR: return RK_FORMAT_BGR_888;
|
|
default: return RK_FORMAT_UNKNOWN;
|
|
}
|
|
}
|
|
|
|
std::atomic<int>& GlobalRgaMaxInflightRef() {
|
|
static std::atomic<int> v{2};
|
|
return v;
|
|
}
|
|
|
|
int GlobalRgaMaxInflight() {
|
|
static std::once_flag once;
|
|
std::call_once(once, []() {
|
|
const char* s = std::getenv("RK3588_RGA_MAX_INFLIGHT");
|
|
if (!s || !*s) return;
|
|
try {
|
|
int v = std::stoi(s);
|
|
if (v > 0 && v <= 32) GlobalRgaMaxInflightRef().store(v);
|
|
} catch (...) {
|
|
LogWarn(std::string("[image_processor] invalid RK3588_RGA_MAX_INFLIGHT='") + s + "'");
|
|
}
|
|
});
|
|
int v = GlobalRgaMaxInflightRef().load();
|
|
return v > 0 ? v : 1;
|
|
}
|
|
|
|
class RgaGate {
|
|
public:
|
|
explicit RgaGate(int max_inflight) : max_inflight_(max_inflight > 0 ? max_inflight : 1) {}
|
|
|
|
void Acquire() {
|
|
std::unique_lock<std::mutex> lock(mu_);
|
|
cv_.wait(lock, [&]() { return in_flight_ < max_inflight_; });
|
|
++in_flight_;
|
|
}
|
|
|
|
void Release() {
|
|
std::lock_guard<std::mutex> lock(mu_);
|
|
if (in_flight_ > 0) {
|
|
--in_flight_;
|
|
}
|
|
cv_.notify_one();
|
|
}
|
|
|
|
void SetMaxInflight(int v) {
|
|
if (v <= 0) return;
|
|
if (v > 32) v = 32;
|
|
{
|
|
std::lock_guard<std::mutex> lock(mu_);
|
|
max_inflight_ = v;
|
|
}
|
|
cv_.notify_all();
|
|
}
|
|
|
|
int MaxInflight() const {
|
|
std::lock_guard<std::mutex> lock(mu_);
|
|
return max_inflight_;
|
|
}
|
|
|
|
private:
|
|
mutable std::mutex mu_;
|
|
std::condition_variable cv_;
|
|
int in_flight_ = 0;
|
|
int max_inflight_ = 1;
|
|
};
|
|
|
|
class RgaGateRegistry {
|
|
public:
|
|
static RgaGateRegistry& Instance() {
|
|
static RgaGateRegistry* inst = new RgaGateRegistry();
|
|
return *inst;
|
|
}
|
|
|
|
RgaGate& Get(const std::string& key) {
|
|
std::lock_guard<std::mutex> lock(mu_);
|
|
auto it = gates_.find(key);
|
|
if (it != gates_.end()) return *it->second;
|
|
auto gate = std::make_unique<RgaGate>(GlobalRgaMaxInflight());
|
|
RgaGate& ref = *gate;
|
|
gates_.emplace(key, std::move(gate));
|
|
return ref;
|
|
}
|
|
|
|
private:
|
|
std::mutex mu_;
|
|
std::unordered_map<std::string, std::unique_ptr<RgaGate>> gates_;
|
|
};
|
|
|
|
RgaGate& GetRgaGate(const std::string& key) {
|
|
const std::string k = key.empty() ? "global" : key;
|
|
return RgaGateRegistry::Instance().Get(k);
|
|
}
|
|
|
|
class ScopedRgaGate {
|
|
public:
|
|
explicit ScopedRgaGate(const std::string& key) : gate_(&GetRgaGate(key)) { gate_->Acquire(); }
|
|
~ScopedRgaGate() { gate_->Release(); }
|
|
ScopedRgaGate(const ScopedRgaGate&) = delete;
|
|
ScopedRgaGate& operator=(const ScopedRgaGate&) = delete;
|
|
|
|
private:
|
|
RgaGate* gate_ = nullptr;
|
|
};
|
|
|
|
void EnsureRgaInitializedOnce() {
|
|
static std::once_flag once;
|
|
std::call_once(once, []() {
|
|
const IM_STATUS st = imcheckHeader();
|
|
if (st != IM_STATUS_NOERROR && st != IM_STATUS_SUCCESS) {
|
|
LogWarn(std::string("[image_processor] imcheckHeader failed: ") + imStrError(st));
|
|
}
|
|
});
|
|
}
|
|
|
|
bool CopyToStridedBuffer(const Frame& src, uint8_t* dst, size_t dst_size,
|
|
int dst_wstride, int dst_hstride) {
|
|
if (!dst || dst_size == 0) return false;
|
|
if (dst_wstride <= 0 || dst_hstride <= 0) return false;
|
|
|
|
std::memset(dst, 0, dst_size);
|
|
|
|
const int w = src.width;
|
|
const int h = src.height;
|
|
if (w <= 0 || h <= 0) return false;
|
|
|
|
if (src.format == PixelFormat::NV12) {
|
|
const size_t y_bytes = static_cast<size_t>(dst_wstride) * dst_hstride;
|
|
const size_t uv_bytes = static_cast<size_t>(dst_wstride) * (dst_hstride / 2);
|
|
if (y_bytes + uv_bytes > dst_size) return false;
|
|
|
|
const uint8_t* src_y = src.planes[0].data ? src.planes[0].data : src.data;
|
|
const uint8_t* src_uv = src.planes[1].data ? src.planes[1].data : nullptr;
|
|
const int src_y_stride = src.planes[0].stride > 0 ? src.planes[0].stride : w;
|
|
const int src_uv_stride = src.planes[1].stride > 0 ? src.planes[1].stride : w;
|
|
if (!src_y) return false;
|
|
if (!src_uv) {
|
|
if (!src.data) return false;
|
|
src_uv = src.data + static_cast<size_t>(src_y_stride) * static_cast<size_t>(h);
|
|
}
|
|
|
|
for (int row = 0; row < h; ++row) {
|
|
std::memcpy(dst + static_cast<size_t>(row) * dst_wstride,
|
|
src_y + static_cast<size_t>(row) * src_y_stride,
|
|
static_cast<size_t>(w));
|
|
}
|
|
|
|
uint8_t* dst_uv = dst + y_bytes;
|
|
const int uv_rows = h / 2;
|
|
for (int row = 0; row < uv_rows; ++row) {
|
|
std::memcpy(dst_uv + static_cast<size_t>(row) * dst_wstride,
|
|
src_uv + static_cast<size_t>(row) * src_uv_stride,
|
|
static_cast<size_t>(w));
|
|
}
|
|
return true;
|
|
}
|
|
|
|
if (src.format == PixelFormat::YUV420) {
|
|
const size_t y_bytes = static_cast<size_t>(dst_wstride) * dst_hstride;
|
|
const size_t uv_stride = static_cast<size_t>(dst_wstride) / 2;
|
|
const size_t uv_h = static_cast<size_t>(dst_hstride) / 2;
|
|
const size_t u_bytes = uv_stride * uv_h;
|
|
const size_t v_bytes = u_bytes;
|
|
if (y_bytes + u_bytes + v_bytes > dst_size) return false;
|
|
|
|
const uint8_t* src_y = src.planes[0].data ? src.planes[0].data : src.data;
|
|
const uint8_t* src_u = src.planes[1].data ? src.planes[1].data : nullptr;
|
|
const uint8_t* src_v = src.planes[2].data ? src.planes[2].data : nullptr;
|
|
const int src_y_stride = src.planes[0].stride > 0 ? src.planes[0].stride : w;
|
|
const int src_u_stride = src.planes[1].stride > 0 ? src.planes[1].stride : (w / 2);
|
|
const int src_v_stride = src.planes[2].stride > 0 ? src.planes[2].stride : (w / 2);
|
|
if (!src_y || !src_u || !src_v) return false;
|
|
|
|
for (int row = 0; row < h; ++row) {
|
|
std::memcpy(dst + static_cast<size_t>(row) * dst_wstride,
|
|
src_y + static_cast<size_t>(row) * src_y_stride,
|
|
static_cast<size_t>(w));
|
|
}
|
|
|
|
uint8_t* dst_u = dst + y_bytes;
|
|
uint8_t* dst_v = dst + y_bytes + u_bytes;
|
|
const int uv_rows = h / 2;
|
|
const int uv_cols = w / 2;
|
|
for (int row = 0; row < uv_rows; ++row) {
|
|
std::memcpy(dst_u + static_cast<size_t>(row) * uv_stride,
|
|
src_u + static_cast<size_t>(row) * src_u_stride,
|
|
static_cast<size_t>(uv_cols));
|
|
std::memcpy(dst_v + static_cast<size_t>(row) * uv_stride,
|
|
src_v + static_cast<size_t>(row) * src_v_stride,
|
|
static_cast<size_t>(uv_cols));
|
|
}
|
|
return true;
|
|
}
|
|
|
|
if (src.format == PixelFormat::RGB || src.format == PixelFormat::BGR) {
|
|
const size_t need = static_cast<size_t>(dst_wstride) * dst_hstride * 3;
|
|
if (need > dst_size) return false;
|
|
|
|
const uint8_t* src_rgb = src.planes[0].data ? src.planes[0].data : src.data;
|
|
const int src_stride = src.planes[0].stride > 0
|
|
? src.planes[0].stride
|
|
: (src.stride > 0 ? src.stride : w * 3);
|
|
if (!src_rgb) return false;
|
|
|
|
const size_t dst_stride = static_cast<size_t>(dst_wstride) * 3;
|
|
const size_t row_bytes = static_cast<size_t>(w) * 3;
|
|
for (int row = 0; row < h; ++row) {
|
|
std::memcpy(dst + static_cast<size_t>(row) * dst_stride,
|
|
src_rgb + static_cast<size_t>(row) * src_stride,
|
|
row_bytes);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
class RgaImageProcessor : public IImageProcessor {
|
|
public:
|
|
explicit RgaImageProcessor(const SimpleJson& config) {
|
|
rga_gate_ = config.ValueOr<std::string>("rga_gate", "global");
|
|
const int rga_max_inflight = config.ValueOr<int>("rga_max_inflight", 0);
|
|
if (rga_max_inflight > 0) {
|
|
GetRgaGate(rga_gate_).SetMaxInflight(rga_max_inflight);
|
|
}
|
|
|
|
if (config.Find("dst_packed")) {
|
|
dst_packed_ = config.ValueOr<bool>("dst_packed", false);
|
|
dst_packed_explicit_ = true;
|
|
} else {
|
|
dst_packed_ = false;
|
|
dst_packed_explicit_ = false;
|
|
}
|
|
}
|
|
|
|
Status Resize(const Frame& src, Frame& dst) override {
|
|
EnsureRgaInitializedOnce();
|
|
|
|
if (dst.width <= 0 || dst.height <= 0) return FailStatus("invalid output size");
|
|
if (dst.format == PixelFormat::UNKNOWN) return FailStatus("invalid output format");
|
|
|
|
const PixelFormat out_fmt = dst.format;
|
|
const int out_w = dst.width;
|
|
const int out_h = dst.height;
|
|
|
|
int src_fmt_rga = ToRgaFormat(src.format);
|
|
int dst_fmt_rga = ToRgaFormat(out_fmt);
|
|
bool need_cvt = (src_fmt_rga != dst_fmt_rga);
|
|
bool need_resize = (src.width != out_w || src.height != out_h);
|
|
|
|
if (src_fmt_rga == RK_FORMAT_UNKNOWN || dst_fmt_rga == RK_FORMAT_UNKNOWN) {
|
|
return FailStatus("unsupported format for RGA");
|
|
}
|
|
|
|
int src_wstride = Align16(src.width);
|
|
int src_hstride = Align16(src.height);
|
|
if (src.format == PixelFormat::NV12 || src.format == PixelFormat::YUV420) {
|
|
const int y_stride = src.planes[0].stride > 0 ? src.planes[0].stride
|
|
: (src.stride > 0 ? src.stride : src.width);
|
|
if (y_stride > 0) src_wstride = y_stride;
|
|
if (src.planes[0].size > 0 && y_stride > 0) {
|
|
const int hs = src.planes[0].size / y_stride;
|
|
if (hs >= src.height) src_hstride = hs;
|
|
}
|
|
} else if (src.format == PixelFormat::RGB || src.format == PixelFormat::BGR) {
|
|
const int stride_bytes = src.planes[0].stride > 0 ? src.planes[0].stride
|
|
: (src.stride > 0 ? src.stride : src.width * 3);
|
|
if (stride_bytes > 0 && (stride_bytes % 3) == 0) {
|
|
src_wstride = stride_bytes / 3;
|
|
}
|
|
if (src.planes[0].size > 0 && stride_bytes > 0) {
|
|
const int hs = src.planes[0].size / stride_bytes;
|
|
if (hs >= src.height) src_hstride = hs;
|
|
}
|
|
}
|
|
|
|
int dst_wstride = Align16(out_w);
|
|
const bool want_packed_rgb = (out_fmt == PixelFormat::RGB || out_fmt == PixelFormat::BGR) &&
|
|
(!dst_packed_explicit_ || dst_packed_);
|
|
if (want_packed_rgb) {
|
|
dst_wstride = out_w;
|
|
}
|
|
int dst_hstride = Align16(out_h);
|
|
|
|
size_t out_size = CalcImageSizeStrided(dst_wstride, dst_hstride, out_fmt);
|
|
if (out_size == 0) {
|
|
return FailStatus("invalid output size for RGA");
|
|
}
|
|
|
|
auto dma_buf = DmaAlloc(out_size);
|
|
if (!dma_buf || !dma_buf->valid()) {
|
|
return FailStatus("DMA alloc failed");
|
|
}
|
|
|
|
rga_buffer_t src_buf{};
|
|
rga_buffer_t dst_buf{};
|
|
DmaBufferPtr src_dma_buf;
|
|
DmaBufferPtr tmp_dma;
|
|
|
|
const bool can_cpu_read_src = (src.data != nullptr && src.data_size > 0);
|
|
auto CopySrcToDmaIfPossible = [&]() -> bool {
|
|
if (!can_cpu_read_src) return false;
|
|
const int copy_wstride = Align16(src.width);
|
|
const int copy_hstride = Align16(src.height);
|
|
const size_t src_size = CalcImageSizeStrided(copy_wstride, copy_hstride, src.format);
|
|
src_dma_buf = DmaAlloc(src_size);
|
|
if (!src_dma_buf || !src_dma_buf->valid()) {
|
|
return false;
|
|
}
|
|
|
|
if (src.DmaFd() >= 0) {
|
|
src.SyncStart();
|
|
}
|
|
|
|
DmaSyncStartFd(src_dma_buf->fd);
|
|
const bool ok = CopyToStridedBuffer(src, src_dma_buf->data(), src_dma_buf->size,
|
|
copy_wstride, copy_hstride);
|
|
DmaSyncEndFd(src_dma_buf->fd);
|
|
|
|
if (src.DmaFd() >= 0) {
|
|
src.SyncEnd();
|
|
}
|
|
|
|
if (!ok) {
|
|
src_dma_buf.reset();
|
|
return false;
|
|
}
|
|
|
|
src_wstride = copy_wstride;
|
|
src_hstride = copy_hstride;
|
|
return true;
|
|
};
|
|
|
|
if (src.DmaFd() < 0) {
|
|
if (!CopySrcToDmaIfPossible()) {
|
|
return FailStatus("no dma_fd and src copy failed");
|
|
}
|
|
}
|
|
|
|
auto RunRgaOnce = [&](int src_fd, std::string& err) -> bool {
|
|
ScopedRgaGate guard(rga_gate_);
|
|
|
|
src_buf = wrapbuffer_fd_t(src_fd, src.width, src.height,
|
|
src_wstride, src_hstride, src_fmt_rga);
|
|
dst_buf = wrapbuffer_fd_t(dma_buf->fd, out_w, out_h,
|
|
dst_wstride, dst_hstride, dst_fmt_rga);
|
|
|
|
auto Check = [&](const rga_buffer_t& s, const rga_buffer_t& d) -> bool {
|
|
const im_rect sr{0, 0, s.width, s.height};
|
|
const im_rect dr{0, 0, d.width, d.height};
|
|
rga_buffer_t pat{};
|
|
const im_rect pr{0, 0, 0, 0};
|
|
const IM_STATUS chk = imcheck_t(s, d, pat, sr, dr, pr, 0);
|
|
if (chk != IM_STATUS_NOERROR && chk != IM_STATUS_SUCCESS) {
|
|
err = std::string("RGA imcheck failed: ") + imStrError(chk);
|
|
return false;
|
|
}
|
|
return true;
|
|
};
|
|
|
|
IM_STATUS status = IM_STATUS_SUCCESS;
|
|
|
|
if (need_resize && need_cvt) {
|
|
if (Check(src_buf, dst_buf)) {
|
|
rga_buffer_t pat{};
|
|
const im_rect sr{0, 0, src_buf.width, src_buf.height};
|
|
const im_rect dr{0, 0, dst_buf.width, dst_buf.height};
|
|
const im_rect pr{0, 0, 0, 0};
|
|
status = improcess(src_buf, dst_buf, pat, sr, dr, pr,
|
|
0, nullptr, nullptr, IM_SYNC);
|
|
if (status == IM_STATUS_SUCCESS) {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
if (!tmp_dma || !tmp_dma->valid()) {
|
|
tmp_dma = DmaAlloc(CalcImageSizeStrided(dst_wstride, dst_hstride, src.format));
|
|
if (!tmp_dma || !tmp_dma->valid()) {
|
|
err = "DMA alloc for tmp failed";
|
|
return false;
|
|
}
|
|
}
|
|
|
|
rga_buffer_t tmp = wrapbuffer_fd_t(tmp_dma->fd, out_w, out_h,
|
|
dst_wstride, dst_hstride, src_fmt_rga);
|
|
if (!Check(src_buf, tmp) || !Check(tmp, dst_buf)) {
|
|
return false;
|
|
}
|
|
status = imresize(src_buf, tmp, 0, 0, 0, 1, nullptr);
|
|
if (status == IM_STATUS_SUCCESS) {
|
|
status = imcvtcolor(tmp, dst_buf, src_fmt_rga, dst_fmt_rga, IM_COLOR_SPACE_DEFAULT, 1, nullptr);
|
|
}
|
|
} else if (need_resize) {
|
|
if (!Check(src_buf, dst_buf)) {
|
|
return false;
|
|
}
|
|
status = imresize(src_buf, dst_buf, 0, 0, 0, 1, nullptr);
|
|
} else if (need_cvt) {
|
|
if (!Check(src_buf, dst_buf)) {
|
|
return false;
|
|
}
|
|
status = imcvtcolor(src_buf, dst_buf, src_fmt_rga, dst_fmt_rga, IM_COLOR_SPACE_DEFAULT, 1, nullptr);
|
|
}
|
|
|
|
if (status != IM_STATUS_SUCCESS) {
|
|
err = std::string("RGA failed: ") + imStrError(status);
|
|
return false;
|
|
}
|
|
return true;
|
|
};
|
|
|
|
std::string rga_err;
|
|
const int src_fd = (src_dma_buf && src_dma_buf->valid()) ? src_dma_buf->fd : src.DmaFd();
|
|
if (src_fd < 0 || !RunRgaOnce(src_fd, rga_err)) {
|
|
return FailStatus(rga_err.empty() ? "RGA failed" : rga_err);
|
|
}
|
|
|
|
dst.stride = (out_fmt == PixelFormat::RGB || out_fmt == PixelFormat::BGR)
|
|
? (dst_wstride * 3)
|
|
: dst_wstride;
|
|
dst.SetDmaFd(dma_buf->fd);
|
|
dst.data = dma_buf->data();
|
|
dst.data_size = dma_buf->size;
|
|
dst.SetOwner(dma_buf);
|
|
SetupPlanes(dst, out_fmt);
|
|
return OkStatus();
|
|
}
|
|
|
|
Status CvtColor(const Frame& src, Frame& dst, PixelFormat dst_format) override {
|
|
dst.width = src.width;
|
|
dst.height = src.height;
|
|
dst.format = dst_format;
|
|
return Resize(src, dst);
|
|
}
|
|
|
|
Status Normalize(const Frame& /*src*/, std::vector<float>& /*out*/,
|
|
const std::vector<float>& /*mean*/,
|
|
const std::vector<float>& /*std*/) override {
|
|
return FailStatus("not implemented");
|
|
}
|
|
|
|
int MaxInflight() const { return GetRgaGate(rga_gate_).MaxInflight(); }
|
|
const std::string& GateKey() const { return rga_gate_; }
|
|
|
|
private:
|
|
std::string rga_gate_ = "global";
|
|
bool dst_packed_ = false;
|
|
bool dst_packed_explicit_ = false;
|
|
};
|
|
#endif
|
|
|
|
#if defined(RK3588_ENABLE_FFMPEG)
|
|
AVPixelFormat ToAvFormat(PixelFormat fmt) {
|
|
switch (fmt) {
|
|
case PixelFormat::NV12: return AV_PIX_FMT_NV12;
|
|
case PixelFormat::YUV420: return AV_PIX_FMT_YUV420P;
|
|
case PixelFormat::RGB: return AV_PIX_FMT_RGB24;
|
|
case PixelFormat::BGR: return AV_PIX_FMT_BGR24;
|
|
default: return AV_PIX_FMT_NONE;
|
|
}
|
|
}
|
|
|
|
void SetupAvPlanes(const Frame* f, uint8_t* data[4], int linesize[4]) {
|
|
if (!f || !f->data) return;
|
|
if (f->format == PixelFormat::NV12) {
|
|
data[0] = f->planes[0].data ? f->planes[0].data : f->data;
|
|
data[1] = f->planes[1].data ? f->planes[1].data : (f->data + f->width * f->height);
|
|
linesize[0] = f->planes[0].stride > 0 ? f->planes[0].stride : f->width;
|
|
linesize[1] = f->planes[1].stride > 0 ? f->planes[1].stride : f->width;
|
|
} else if (f->format == PixelFormat::YUV420) {
|
|
data[0] = f->planes[0].data ? f->planes[0].data : f->data;
|
|
int y_size = f->width * f->height;
|
|
int uv_size = y_size / 4;
|
|
data[1] = f->planes[1].data ? f->planes[1].data : (f->data + y_size);
|
|
data[2] = f->planes[2].data ? f->planes[2].data : (f->data + y_size + uv_size);
|
|
linesize[0] = f->planes[0].stride > 0 ? f->planes[0].stride : f->width;
|
|
linesize[1] = f->planes[1].stride > 0 ? f->planes[1].stride : f->width / 2;
|
|
linesize[2] = f->planes[2].stride > 0 ? f->planes[2].stride : f->width / 2;
|
|
} else {
|
|
data[0] = f->data;
|
|
linesize[0] = f->stride > 0 ? f->stride : f->width * 3;
|
|
}
|
|
}
|
|
|
|
class SwscaleImageProcessor : public IImageProcessor {
|
|
public:
|
|
~SwscaleImageProcessor() override {
|
|
if (sws_ctx_) {
|
|
sws_freeContext(sws_ctx_);
|
|
sws_ctx_ = nullptr;
|
|
}
|
|
}
|
|
|
|
Status Resize(const Frame& src, Frame& dst) override {
|
|
if (dst.width <= 0 || dst.height <= 0) return FailStatus("invalid output size");
|
|
if (dst.format == PixelFormat::UNKNOWN) return FailStatus("invalid output format");
|
|
|
|
AVPixelFormat src_av_fmt = ToAvFormat(src.format);
|
|
AVPixelFormat dst_av_fmt = ToAvFormat(dst.format);
|
|
|
|
if (src_av_fmt == AV_PIX_FMT_NONE || dst_av_fmt == AV_PIX_FMT_NONE) {
|
|
return FailStatus("unsupported format");
|
|
}
|
|
|
|
if (!sws_ctx_ || src.width != last_src_w_ || src.height != last_src_h_ ||
|
|
src_av_fmt != last_src_fmt_ || dst_av_fmt != last_dst_fmt_ ||
|
|
dst.width != last_dst_w_ || dst.height != last_dst_h_) {
|
|
if (sws_ctx_) sws_freeContext(sws_ctx_);
|
|
sws_ctx_ = sws_getContext(src.width, src.height, src_av_fmt,
|
|
dst.width, dst.height, dst_av_fmt,
|
|
SWS_BILINEAR, nullptr, nullptr, nullptr);
|
|
last_src_w_ = src.width;
|
|
last_src_h_ = src.height;
|
|
last_src_fmt_ = src_av_fmt;
|
|
last_dst_fmt_ = dst_av_fmt;
|
|
last_dst_w_ = dst.width;
|
|
last_dst_h_ = dst.height;
|
|
}
|
|
|
|
if (!sws_ctx_) {
|
|
return FailStatus("sws_getContext failed");
|
|
}
|
|
|
|
size_t out_size = CalcImageSize(dst.width, dst.height, dst.format);
|
|
if (out_size == 0) return FailStatus("invalid output size");
|
|
auto buffer = std::make_shared<std::vector<uint8_t>>(out_size);
|
|
|
|
uint8_t* src_data[4] = {nullptr};
|
|
int src_linesize[4] = {0};
|
|
uint8_t* dst_data[4] = {nullptr};
|
|
int dst_linesize[4] = {0};
|
|
|
|
SetupAvPlanes(&src, src_data, src_linesize);
|
|
av_image_fill_arrays(dst_data, dst_linesize, buffer->data(),
|
|
dst_av_fmt, dst.width, dst.height, 1);
|
|
|
|
sws_scale(sws_ctx_, src_data, src_linesize, 0, src.height,
|
|
dst_data, dst_linesize);
|
|
|
|
dst.stride = (dst.format == PixelFormat::RGB || dst.format == PixelFormat::BGR)
|
|
? (dst.width * 3)
|
|
: dst.width;
|
|
dst.data = buffer->data();
|
|
dst.data_size = buffer->size();
|
|
dst.SetOwner(buffer);
|
|
SetupPlanes(dst, dst.format);
|
|
return OkStatus();
|
|
}
|
|
|
|
Status CvtColor(const Frame& src, Frame& dst, PixelFormat dst_format) override {
|
|
dst.width = src.width;
|
|
dst.height = src.height;
|
|
dst.format = dst_format;
|
|
return Resize(src, dst);
|
|
}
|
|
|
|
Status Normalize(const Frame& /*src*/, std::vector<float>& /*out*/,
|
|
const std::vector<float>& /*mean*/,
|
|
const std::vector<float>& /*std*/) override {
|
|
return FailStatus("not implemented");
|
|
}
|
|
|
|
private:
|
|
SwsContext* sws_ctx_ = nullptr;
|
|
int last_src_w_ = 0;
|
|
int last_src_h_ = 0;
|
|
int last_dst_w_ = 0;
|
|
int last_dst_h_ = 0;
|
|
AVPixelFormat last_src_fmt_ = AV_PIX_FMT_NONE;
|
|
AVPixelFormat last_dst_fmt_ = AV_PIX_FMT_NONE;
|
|
};
|
|
#else
|
|
class SwscaleImageProcessor : public IImageProcessor {
|
|
public:
|
|
Status Resize(const Frame& /*src*/, Frame& /*dst*/) override {
|
|
return FailStatus("swscale disabled");
|
|
}
|
|
Status CvtColor(const Frame& /*src*/, Frame& /*dst*/, PixelFormat /*dst_format*/) override {
|
|
return FailStatus("swscale disabled");
|
|
}
|
|
Status Normalize(const Frame& /*src*/, std::vector<float>& /*out*/,
|
|
const std::vector<float>& /*mean*/,
|
|
const std::vector<float>& /*std*/) override {
|
|
return FailStatus("swscale disabled");
|
|
}
|
|
};
|
|
#endif
|
|
|
|
} // namespace
|
|
|
|
Rk3588ImageProcessor::Rk3588ImageProcessor(const SimpleJson& config) {
|
|
use_rga_ = config.ValueOr<bool>("use_rga", true);
|
|
|
|
#if defined(RK3588_ENABLE_RGA)
|
|
if (use_rga_) {
|
|
impl_ = std::make_shared<RgaImageProcessor>(config);
|
|
}
|
|
#endif
|
|
|
|
if (!impl_) {
|
|
impl_ = std::make_shared<SwscaleImageProcessor>();
|
|
}
|
|
}
|
|
|
|
Status Rk3588ImageProcessor::Resize(const Frame& src, Frame& dst) {
|
|
if (!impl_) return FailStatus("no image processor backend");
|
|
return impl_->Resize(src, dst);
|
|
}
|
|
|
|
Status Rk3588ImageProcessor::CvtColor(const Frame& src, Frame& dst, PixelFormat dst_format) {
|
|
if (!impl_) return FailStatus("no image processor backend");
|
|
return impl_->CvtColor(src, dst, dst_format);
|
|
}
|
|
|
|
Status Rk3588ImageProcessor::Normalize(const Frame& src, std::vector<float>& out,
|
|
const std::vector<float>& mean,
|
|
const std::vector<float>& std) {
|
|
if (!impl_) return FailStatus("no image processor backend");
|
|
return impl_->Normalize(src, out, mean, std);
|
|
}
|
|
|
|
bool Rk3588ImageProcessor::IsUsingRga() const { return use_rga_; }
|
|
|
|
int Rk3588ImageProcessor::RgaMaxInflight() const {
|
|
#if defined(RK3588_ENABLE_RGA)
|
|
if (!use_rga_) return 0;
|
|
auto rga = std::dynamic_pointer_cast<RgaImageProcessor>(impl_);
|
|
if (!rga) return 0;
|
|
return rga->MaxInflight();
|
|
#else
|
|
return 0;
|
|
#endif
|
|
}
|
|
|
|
std::string Rk3588ImageProcessor::RgaGateKey() const {
|
|
#if defined(RK3588_ENABLE_RGA)
|
|
if (!use_rga_) return "";
|
|
auto rga = std::dynamic_pointer_cast<RgaImageProcessor>(impl_);
|
|
if (!rga) return "";
|
|
return rga->GateKey();
|
|
#else
|
|
return "";
|
|
#endif
|
|
}
|
|
|
|
} // namespace rk3588
|