Doni/safesight-edge
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Enhance preprocessing and detection with letterbox resizing and transformation metadata

Browse Source 
- Added `resize_mode` option to `preprocess_node` for handling image resizing strategies: stretch, keep ratio, and letterbox.
- Implemented `BuildLetterbox` function to create letterbox resized frames while maintaining aspect ratio.
- Introduced `FrameTransformMeta` structure to store transformation metadata for frames, including scaling and padding information.
- Updated detection coordinate calculations in `ai_yolo_node` to utilize transformation metadata for accurate bounding box adjustments.
- Enhanced bounding box mapping in `osd_node` to account for potential discrepancies between detection and frame sizes.
- Refactored image processing logic to support new resizing modes and ensure compatibility with existing formats.
This commit is contained in:
sladro 2026-02-27 20:05:20 +08:00
parent b4709b20cd
commit aaa794fe10
5 changed files with 518 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
configs/test_cam1_ppe11_yolo_debug.json
View File

@ -35,6 +35,7 @@
"dst_h": 768,
"dst_format": "rgb",
"dst_packed": true,
"resize_mode": "letterbox",
"keep_ratio": false,
"rga_gate": "cam_ppe11_yolo_debug",
"use_rga": true

14
include/frame/frame.h
View File

@ -44,6 +44,19 @@ struct DetectionResult {
std::string model_name;
};
struct FrameTransformMeta {
bool valid = false;
bool letterbox = false;
int src_w = 0;
int src_h = 0;
int dst_w = 0;
int dst_h = 0;
float scale_x = 1.0f;
float scale_y = 1.0f;
float pad_x = 0.0f;
float pad_y = 0.0f;
};
struct FramePlane {
uint8_t* data = nullptr;
int stride = 0; // bytes per row
@ -76,6 +89,7 @@ struct Frame {
// Face recognition pipeline meta (kept separate from user_meta to avoid conflicts with publish).
std::shared_ptr<FaceDetResult> face_det;
std::shared_ptr<FaceRecogResult> face_recog;
std::shared_ptr<FrameTransformMeta> transform_meta;
std::shared_ptr<void> user_meta;
int DmaFd() const { return buffer ? buffer->DmaFd() : dma_fd; }

85
plugins/ai_yolo/ai_yolo_node.cpp
View File

@ -54,6 +54,67 @@ inline int Clamp(float val, int min_val, int max_val) {
return val > min_val ? (val < max_val ? static_cast<int>(val) : max_val) : min_val;
}
struct DetCoordContext {
bool has_transform = false;
int out_w = 0;
int out_h = 0;
float scale_x = 1.0f;
float scale_y = 1.0f;
float pad_x = 0.0f;
float pad_y = 0.0f;
float fallback_scale_w = 1.0f;
float fallback_scale_h = 1.0f;
};
DetCoordContext BuildDetCoordContext(const Frame& frame, int model_input_w, int model_input_h) {
DetCoordContext ctx{};
ctx.fallback_scale_w = frame.width > 0 ? static_cast<float>(model_input_w) / frame.width : 1.0f;
ctx.fallback_scale_h = frame.height > 0 ? static_cast<float>(model_input_h) / frame.height : 1.0f;
ctx.out_w = frame.width;
ctx.out_h = frame.height;
if (frame.transform_meta && frame.transform_meta->valid &&
frame.transform_meta->src_w > 0 && frame.transform_meta->src_h > 0 &&
frame.transform_meta->scale_x > 1e-6f && frame.transform_meta->scale_y > 1e-6f) {
ctx.has_transform = true;
ctx.out_w = frame.transform_meta->src_w;
ctx.out_h = frame.transform_meta->src_h;
ctx.scale_x = frame.transform_meta->scale_x;
ctx.scale_y = frame.transform_meta->scale_y;
ctx.pad_x = frame.transform_meta->pad_x;
ctx.pad_y = frame.transform_meta->pad_y;
}
return ctx;
}
Rect DecodeToOutputRect(float x, float y, float w, float h, const DetCoordContext& ctx) {
float ox = x;
float oy = y;
float ow = w;
float oh = h;
if (ctx.has_transform) {
ox = (x - ctx.pad_x) / ctx.scale_x;
oy = (y - ctx.pad_y) / ctx.scale_y;
ow = w / ctx.scale_x;
oh = h / ctx.scale_y;
} else {
ox = x / ctx.fallback_scale_w;
oy = y / ctx.fallback_scale_h;
ow = w / ctx.fallback_scale_w;
oh = h / ctx.fallback_scale_h;
}
Rect r{};
const int out_w = std::max(1, ctx.out_w);
const int out_h = std::max(1, ctx.out_h);
r.x = static_cast<float>(Clamp(static_cast<int>(ox), 0, out_w));
r.y = static_cast<float>(Clamp(static_cast<int>(oy), 0, out_h));
r.w = static_cast<float>(Clamp(static_cast<int>(ow), 0, out_w - static_cast<int>(r.x)));
r.h = static_cast<float>(Clamp(static_cast<int>(oh), 0, out_h - static_cast<int>(r.y)));
return r;
}
inline int32_t ClipFloat(float val, float min_val, float max_val) {
return static_cast<int32_t>(val <= min_val ? min_val : (val >= max_val ? max_val : val));
}
@ -890,12 +951,11 @@ private:
NMS(valid_count, boxes, class_ids, indices, c, nms_thresh_);
}
float scale_w = static_cast<float>(model_input_w_) / frame->width;
float scale_h = static_cast<float>(model_input_h_) / frame->height;
const DetCoordContext coord_ctx = BuildDetCoordContext(*frame, model_input_w_, model_input_h_);
auto det_result = std::make_shared<DetectionResult>();
det_result->img_w = frame->width;
det_result->img_h = frame->height;
det_result->img_w = coord_ctx.out_w;
det_result->img_h = coord_ctx.out_h;
det_result->model_name = (yolo_version_ == YoloVersion::V5) ? "yolov5" : "yolov8";
for (int i = 0; i < valid_count && det_result->items.size() < kMaxDetections; ++i) {
@ -915,10 +975,7 @@ private:
Detection det;
det.cls_id = cls_id;
det.score = obj_probs[i];
det.bbox.x = static_cast<float>(Clamp(static_cast<int>(x1 / scale_w), 0, frame->width));
det.bbox.y = static_cast<float>(Clamp(static_cast<int>(y1 / scale_h), 0, frame->height));
det.bbox.w = static_cast<float>(Clamp(static_cast<int>(w / scale_w), 0, frame->width - static_cast<int>(det.bbox.x)));
det.bbox.h = static_cast<float>(Clamp(static_cast<int>(h / scale_h), 0, frame->height - static_cast<int>(det.bbox.y)));
det.bbox = DecodeToOutputRect(x1, y1, w, h, coord_ctx);
det.track_id = -1;
if (debug_det_ && det_result->items.size() < 5 && processed_ < 20) {
@ -1044,12 +1101,11 @@ private:
NMS(valid_count, boxes, class_ids, indices, c, nms_thresh_);
}
float scale_w = static_cast<float>(model_input_w_) / frame->width;
float scale_h = static_cast<float>(model_input_h_) / frame->height;
const DetCoordContext coord_ctx = BuildDetCoordContext(*frame, model_input_w_, model_input_h_);
auto det_result = std::make_shared<DetectionResult>();
det_result->img_w = frame->width;
det_result->img_h = frame->height;
det_result->img_w = coord_ctx.out_w;
det_result->img_h = coord_ctx.out_h;
det_result->model_name = (yolo_version_ == YoloVersion::V5) ? "yolov5" : "yolov8";
for (int i = 0; i < valid_count && det_result->items.size() < kMaxDetections; ++i) {
@ -1069,10 +1125,7 @@ private:
Detection det;
det.cls_id = cls_id;
det.score = obj_probs[i];
det.bbox.x = static_cast<float>(Clamp(static_cast<int>(x1 / scale_w), 0, frame->width));
det.bbox.y = static_cast<float>(Clamp(static_cast<int>(y1 / scale_h), 0, frame->height));
det.bbox.w = static_cast<float>(Clamp(static_cast<int>(w / scale_w), 0, frame->width - static_cast<int>(det.bbox.x)));
det.bbox.h = static_cast<float>(Clamp(static_cast<int>(h / scale_h), 0, frame->height - static_cast<int>(det.bbox.y)));
det.bbox = DecodeToOutputRect(x1, y1, w, h, coord_ctx);
det.track_id = -1;
if (debug_det_ && det_result->items.size() < 5 && processed_ < 20) {

35
plugins/osd/osd_node.cpp
View File

@ -63,6 +63,20 @@ inline int Clamp(int val, int min_val, int max_val) {
return val < min_val ? min_val : (val > max_val ? max_val : val);
}
Rect MapRectToFrame(const Rect& in, int src_w, int src_h, int dst_w, int dst_h) {
if (src_w <= 0 || src_h <= 0 || dst_w <= 0 || dst_h <= 0) return Rect{};
const float sx = static_cast<float>(dst_w) / static_cast<float>(src_w);
const float sy = static_cast<float>(dst_h) / static_cast<float>(src_h);
Rect out{};
out.x = std::max(0.0f, in.x * sx);
out.y = std::max(0.0f, in.y * sy);
out.w = std::max(0.0f, in.w * sx);
out.h = std::max(0.0f, in.h * sy);
if (out.x + out.w > static_cast<float>(dst_w)) out.w = std::max(0.0f, static_cast<float>(dst_w) - out.x);
if (out.y + out.h > static_cast<float>(dst_h)) out.h = std::max(0.0f, static_cast<float>(dst_h) - out.y);
return out;
}
#if defined(RK3588_ENABLE_RGA)
inline uint32_t PackColorArgb(const Color& c) {
return (0xFFu << 24) | (static_cast<uint32_t>(c.r) << 16) |
@ -561,6 +575,9 @@ private:
int w = frame->width;
int h = frame->height;
const int det_w = frame->det->img_w > 0 ? frame->det->img_w : w;
const int det_h = frame->det->img_h > 0 ? frame->det->img_h : h;
const bool map_det_to_frame = (det_w != w || det_h != h);
uint8_t* data = frame->planes[0].data ? frame->planes[0].data : frame->data;
PixelFormat fmt = frame->format;
@ -587,10 +604,11 @@ private:
} else {
bool ok = true;
for (const auto& det : frame->det->items) {
int x = Clamp(static_cast<int>(det.bbox.x), 0, w - 1);
int y = Clamp(static_cast<int>(det.bbox.y), 0, h - 1);
int rw = static_cast<int>(det.bbox.w);
int rh = static_cast<int>(det.bbox.h);
const Rect draw = map_det_to_frame ? MapRectToFrame(det.bbox, det_w, det_h, w, h) : det.bbox;
int x = Clamp(static_cast<int>(draw.x), 0, w - 1);
int y = Clamp(static_cast<int>(draw.y), 0, h - 1);
int rw = static_cast<int>(draw.w);
int rh = static_cast<int>(draw.h);
rw = Clamp(rw, 1, w - x);
rh = Clamp(rh, 1, h - y);
im_rect rect{x, y, rw, rh};
@ -636,10 +654,11 @@ private:
}
for (const auto& det : frame->det->items) {
int x1 = static_cast<int>(det.bbox.x);
int y1 = static_cast<int>(det.bbox.y);
int x2 = static_cast<int>(det.bbox.x + det.bbox.w);
int y2 = static_cast<int>(det.bbox.y + det.bbox.h);
const Rect draw = map_det_to_frame ? MapRectToFrame(det.bbox, det_w, det_h, w, h) : det.bbox;
int x1 = static_cast<int>(draw.x);
int y1 = static_cast<int>(draw.y);
int x2 = static_cast<int>(draw.x + draw.w);
int y2 = static_cast<int>(draw.y + draw.h);
Color color = GetClassColor(det.cls_id);

494
plugins/preprocess/preprocess_node.cpp
View File

@ -1,17 +1,27 @@
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <cstring>
#include <memory>
#include <string>
#include <vector>
#include "face/face_result.h"
#include "hw/i_image_processor.h"
#include "node.h"
#include "utils/dma_alloc.h"
#include "utils/logger.h"
namespace rk3588 {
namespace {
enum class ResizeMode {
Stretch,
KeepRatio,
Letterbox,
};
PixelFormat ParseFormat(const std::string& s) {
if (s == "nv12" || s == "NV12") return PixelFormat::NV12;
if (s == "yuv420" || s == "YUV420") return PixelFormat::YUV420;
@ -20,11 +30,199 @@ PixelFormat ParseFormat(const std::string& s) {
return PixelFormat::UNKNOWN;
}
ResizeMode ParseResizeMode(const std::string& s, bool keep_ratio) {
if (s == "stretch") return ResizeMode::Stretch;
if (s == "keep_ratio" || s == "fit") return ResizeMode::KeepRatio;
if (s == "letterbox") return ResizeMode::Letterbox;
return keep_ratio ? ResizeMode::KeepRatio : ResizeMode::Stretch;
}
inline bool IsYuvFormat(PixelFormat fmt) {
return fmt == PixelFormat::NV12 || fmt == PixelFormat::YUV420;
}
inline float ClampFloat(float v, float lo, float hi) {
return std::max(lo, std::min(v, hi));
}
void ScaleRect(Rect& r, float sx, float sy, int out_w, int out_h) {
inline int MakeEvenFloor(int v) {
if (v <= 0) return 0;
return v & ~1;
}
size_t CalcImageSize(int w, int h, PixelFormat fmt) {
if (w <= 0 || h <= 0) return 0;
switch (fmt) {
case PixelFormat::NV12:
case PixelFormat::YUV420:
return static_cast<size_t>(w) * static_cast<size_t>(h) * 3 / 2;
case PixelFormat::RGB:
case PixelFormat::BGR:
return static_cast<size_t>(w) * static_cast<size_t>(h) * 3;
default:
return 0;
}
}
void SetupPlanes(Frame& f) {
if (!f.data || f.width <= 0 || f.height <= 0) return;
if (f.format == PixelFormat::NV12) {
const int y_stride = f.width;
const int y_size = y_stride * f.height;
const int uv_size = y_stride * (f.height / 2);
f.stride = y_stride;
f.plane_count = 2;
f.planes[0] = {f.data, y_stride, y_size, 0};
f.planes[1] = {f.data + y_size, y_stride, uv_size, y_size};
} else if (f.format == PixelFormat::YUV420) {
const int y_stride = f.width;
const int y_size = y_stride * f.height;
const int uv_stride = f.width / 2;
const int u_size = uv_stride * (f.height / 2);
f.stride = y_stride;
f.plane_count = 3;
f.planes[0] = {f.data, y_stride, y_size, 0};
f.planes[1] = {f.data + y_size, uv_stride, u_size, y_size};
f.planes[2] = {f.data + y_size + u_size, uv_stride, u_size, y_size + u_size};
} else {
const int stride = f.width * 3;
f.stride = stride;
f.plane_count = 1;
f.planes[0] = {f.data, stride, static_cast<int>(f.data_size), 0};
}
f.SyncBufferFromFrame();
}
bool InitFrameStorage(Frame& f) {
const size_t need = CalcImageSize(f.width, f.height, f.format);
if (need == 0) return false;
if (auto dma = DmaAlloc(need); dma && dma->valid()) {
f.SetDmaFd(dma->fd);
f.data = dma->data();
f.data_size = dma->size;
f.SetOwner(dma);
SetupPlanes(f);
return true;
}
auto buf = std::make_shared<std::vector<uint8_t>>(need);
f.SetDmaFd(-1);
f.data = buf->data();
f.data_size = buf->size();
f.SetOwner(buf);
SetupPlanes(f);
return true;
}
void FillBlack(Frame& f) {
if (!f.data || f.data_size == 0) return;
if (f.DmaFd() >= 0) {
DmaSyncStartFd(f.DmaFd());
}
if (f.format == PixelFormat::NV12) {
const int y_size = f.width * f.height;
std::memset(f.data, 0, static_cast<size_t>(y_size));
std::memset(f.data + y_size, 128, static_cast<size_t>(f.width * f.height / 2));
} else if (f.format == PixelFormat::YUV420) {
const int y_size = f.width * f.height;
const int u_size = (f.width / 2) * (f.height / 2);
std::memset(f.data, 0, static_cast<size_t>(y_size));
std::memset(f.data + y_size, 128, static_cast<size_t>(u_size));
std::memset(f.data + y_size + u_size, 128, static_cast<size_t>(u_size));
} else {
std::memset(f.data, 0, f.data_size);
}
if (f.DmaFd() >= 0) {
DmaSyncEndFd(f.DmaFd());
}
}
bool BlitLetterbox(const Frame& src, Frame& dst, int pad_x, int pad_y) {
if (!src.data || !dst.data || src.format != dst.format) return false;
if (pad_x < 0 || pad_y < 0) return false;
if (src.width + pad_x > dst.width || src.height + pad_y > dst.height) return false;
if (src.DmaFd() >= 0) src.SyncStart();
if (dst.DmaFd() >= 0) DmaSyncStartFd(dst.DmaFd());
if (src.format == PixelFormat::RGB || src.format == PixelFormat::BGR) {
const int src_stride = src.planes[0].stride > 0 ? src.planes[0].stride : src.width * 3;
const int dst_stride = dst.planes[0].stride > 0 ? dst.planes[0].stride : dst.width * 3;
const uint8_t* src_ptr = src.planes[0].data ? src.planes[0].data : src.data;
uint8_t* dst_ptr = dst.planes[0].data ? dst.planes[0].data : dst.data;
const size_t row_bytes = static_cast<size_t>(src.width) * 3;
for (int y = 0; y < src.height; ++y) {
std::memcpy(dst_ptr + static_cast<size_t>(y + pad_y) * dst_stride + static_cast<size_t>(pad_x) * 3,
src_ptr + static_cast<size_t>(y) * src_stride,
row_bytes);
}
} else if (src.format == PixelFormat::NV12) {
const int src_y_stride = src.planes[0].stride > 0 ? src.planes[0].stride : src.width;
const int src_uv_stride = src.planes[1].stride > 0 ? src.planes[1].stride : src.width;
const int dst_y_stride = dst.planes[0].stride > 0 ? dst.planes[0].stride : dst.width;
const int dst_uv_stride = dst.planes[1].stride > 0 ? dst.planes[1].stride : dst.width;
const uint8_t* src_y = src.planes[0].data ? src.planes[0].data : src.data;
const uint8_t* src_uv = src.planes[1].data ? src.planes[1].data : (src.data + src.width * src.height);
uint8_t* dst_y = dst.planes[0].data ? dst.planes[0].data : dst.data;
uint8_t* dst_uv = dst.planes[1].data ? dst.planes[1].data : (dst.data + dst.width * dst.height);
for (int y = 0; y < src.height; ++y) {
std::memcpy(dst_y + static_cast<size_t>(y + pad_y) * dst_y_stride + pad_x,
src_y + static_cast<size_t>(y) * src_y_stride,
static_cast<size_t>(src.width));
}
const int uv_rows = src.height / 2;
for (int y = 0; y < uv_rows; ++y) {
std::memcpy(dst_uv + static_cast<size_t>(y + pad_y / 2) * dst_uv_stride + pad_x,
src_uv + static_cast<size_t>(y) * src_uv_stride,
static_cast<size_t>(src.width));
}
} else if (src.format == PixelFormat::YUV420) {
const int src_y_stride = src.planes[0].stride > 0 ? src.planes[0].stride : src.width;
const int src_u_stride = src.planes[1].stride > 0 ? src.planes[1].stride : src.width / 2;
const int src_v_stride = src.planes[2].stride > 0 ? src.planes[2].stride : src.width / 2;
const int dst_y_stride = dst.planes[0].stride > 0 ? dst.planes[0].stride : dst.width;
const int dst_u_stride = dst.planes[1].stride > 0 ? dst.planes[1].stride : dst.width / 2;
const int dst_v_stride = dst.planes[2].stride > 0 ? dst.planes[2].stride : dst.width / 2;
const uint8_t* src_y = src.planes[0].data ? src.planes[0].data : src.data;
const uint8_t* src_u = src.planes[1].data ? src.planes[1].data : (src.data + src.width * src.height);
const uint8_t* src_v = src.planes[2].data ? src.planes[2].data : (src_u + (src.width / 2) * (src.height / 2));
uint8_t* dst_y = dst.planes[0].data ? dst.planes[0].data : dst.data;
uint8_t* dst_u = dst.planes[1].data ? dst.planes[1].data : (dst.data + dst.width * dst.height);
uint8_t* dst_v = dst.planes[2].data ? dst.planes[2].data : (dst_u + (dst.width / 2) * (dst.height / 2));
for (int y = 0; y < src.height; ++y) {
std::memcpy(dst_y + static_cast<size_t>(y + pad_y) * dst_y_stride + pad_x,
src_y + static_cast<size_t>(y) * src_y_stride,
static_cast<size_t>(src.width));
}
const int uv_rows = src.height / 2;
const int uv_pad_x = pad_x / 2;
const int uv_pad_y = pad_y / 2;
const int uv_cols = src.width / 2;
for (int y = 0; y < uv_rows; ++y) {
std::memcpy(dst_u + static_cast<size_t>(y + uv_pad_y) * dst_u_stride + uv_pad_x,
src_u + static_cast<size_t>(y) * src_u_stride,
static_cast<size_t>(uv_cols));
std::memcpy(dst_v + static_cast<size_t>(y + uv_pad_y) * dst_v_stride + uv_pad_x,
src_v + static_cast<size_t>(y) * src_v_stride,
static_cast<size_t>(uv_cols));
}
} else {
if (src.DmaFd() >= 0) src.SyncEnd();
if (dst.DmaFd() >= 0) DmaSyncEndFd(dst.DmaFd());
return false;
}
if (dst.DmaFd() >= 0) DmaSyncEndFd(dst.DmaFd());
if (src.DmaFd() >= 0) src.SyncEnd();
return true;
}
void TransformRect(Rect& r, float sx, float sy, float tx, float ty, int out_w, int out_h) {
if (out_w <= 0 || out_h <= 0) {
r = Rect{};
return;
@ -32,8 +230,8 @@ void ScaleRect(Rect& r, float sx, float sy, int out_w, int out_h) {
const float fw = static_cast<float>(out_w);
const float fh = static_cast<float>(out_h);
const float x = ClampFloat(r.x * sx, 0.0f, fw);
const float y = ClampFloat(r.y * sy, 0.0f, fh);
const float x = ClampFloat(r.x * sx + tx, 0.0f, fw);
const float y = ClampFloat(r.y * sy + ty, 0.0f, fh);
float w = std::max(0.0f, r.w * sx);
float h = std::max(0.0f, r.h * sy);
@ -46,13 +244,13 @@ void ScaleRect(Rect& r, float sx, float sy, int out_w, int out_h) {
r.h = h;
}
void ScalePoint(Point2f& p, float sx, float sy, int out_w, int out_h) {
void TransformPoint(Point2f& p, float sx, float sy, float tx, float ty, int out_w, int out_h) {
if (out_w <= 0 || out_h <= 0) {
p = Point2f{};
return;
}
p.x = ClampFloat(p.x * sx, 0.0f, static_cast<float>(out_w));
p.y = ClampFloat(p.y * sy, 0.0f, static_cast<float>(out_h));
p.x = ClampFloat(p.x * sx + tx, 0.0f, static_cast<float>(out_w));
p.y = ClampFloat(p.y * sy + ty, 0.0f, static_cast<float>(out_h));
}
} // namespace
@ -67,6 +265,7 @@ public:
dst_w_ = config.ValueOr<int>("dst_w", 640);
dst_h_ = config.ValueOr<int>("dst_h", 640);
keep_ratio_ = config.ValueOr<bool>("keep_ratio", false);
resize_mode_ = ParseResizeMode(config.ValueOr<std::string>("resize_mode", ""), keep_ratio_);
std::string fmt_str = config.ValueOr<std::string>("dst_format", "");
if (!fmt_str.empty()) {
@ -117,8 +316,11 @@ public:
}
bool Start() override {
std::string mode = "stretch";
if (resize_mode_ == ResizeMode::KeepRatio) mode = "keep_ratio";
if (resize_mode_ == ResizeMode::Letterbox) mode = "letterbox";
LogInfo("[preprocess] start id=" + id_ + " dst=" + std::to_string(dst_w_) + "x" +
std::to_string(dst_h_) + (use_rga_ ? " (rga)" : " (swscale)"));
std::to_string(dst_h_) + " mode=" + mode + (use_rga_ ? " (rga)" : " (swscale)"));
return true;
}
@ -131,98 +333,80 @@ public:
PixelFormat out_fmt = (dst_fmt_ != PixelFormat::UNKNOWN) ? dst_fmt_ : frame->format;
int out_w = dst_w_;
int out_h = dst_h_;
if (out_w <= 0) out_w = frame->width;
if (out_h <= 0) out_h = frame->height;
if (keep_ratio_ && dst_w_ > 0 && dst_h_ > 0 && frame->width > 0 && frame->height > 0) {
float scale = std::min(static_cast<float>(dst_w_) / frame->width,
static_cast<float>(dst_h_) / frame->height);
out_w = static_cast<int>(frame->width * scale);
out_h = static_cast<int>(frame->height * scale);
out_w = (out_w + 1) & ~1;
out_h = (out_h + 1) & ~1;
}
const bool need_resize = (frame->width != out_w || frame->height != out_h);
const bool need_cvt = (frame->format != out_fmt);
if (need_resize) {
WarnMetaResizeOnce(frame, out_w, out_h);
}
if (!need_resize && !need_cvt) {
ProcessPassthrough(frame);
return NodeStatus::OK;
}
FrameTransformMeta tx{};
tx.valid = true;
tx.src_w = frame->width;
tx.src_h = frame->height;
Frame out;
out.width = out_w;
out.height = out_h;
out.format = out_fmt;
if (resize_mode_ == ResizeMode::Letterbox && dst_w_ > 0 && dst_h_ > 0 &&
frame->width > 0 && frame->height > 0) {
Status lb = BuildLetterbox(*frame, out_fmt, out_w, out_h, out, tx);
if (lb.Failed()) {
LogError("[preprocess] letterbox failed: " + lb.ErrMessage());
return NodeStatus::ERROR;
}
} else {
if (resize_mode_ == ResizeMode::KeepRatio && dst_w_ > 0 && dst_h_ > 0 &&
frame->width > 0 && frame->height > 0) {
float scale = std::min(static_cast<float>(dst_w_) / frame->width,
static_cast<float>(dst_h_) / frame->height);
out_w = static_cast<int>(std::round(frame->width * scale));
out_h = static_cast<int>(std::round(frame->height * scale));
if (IsYuvFormat(out_fmt)) {
out_w = std::max(2, MakeEvenFloor(out_w));
out_h = std::max(2, MakeEvenFloor(out_h));
}
if (out_w <= 0) out_w = frame->width;
if (out_h <= 0) out_h = frame->height;
}
Status st = image_processor_->Resize(*frame, out);
if (st.Failed()) {
if (!use_rga_ && st.ErrMessage().find("unsupported format") != std::string::npos) {
const bool need_resize = (frame->width != out_w || frame->height != out_h);
const bool need_cvt = (frame->format != out_fmt);
tx.letterbox = false;
tx.dst_w = out_w;
tx.dst_h = out_h;
tx.scale_x = frame->width > 0 ? static_cast<float>(out_w) / frame->width : 1.0f;
tx.scale_y = frame->height > 0 ? static_cast<float>(out_h) / frame->height : 1.0f;
tx.pad_x = 0.0f;
tx.pad_y = 0.0f;
if (!need_resize && !need_cvt) {
auto t = std::make_shared<FrameTransformMeta>(tx);
frame->transform_meta = t;
ProcessPassthrough(frame);
return NodeStatus::OK;
}
LogError("[preprocess] " + st.ErrMessage());
return NodeStatus::ERROR;
if (need_resize) {
WarnMetaResizeOnce(frame, out_w, out_h);
}
out.width = out_w;
out.height = out_h;
out.format = out_fmt;
Status st = image_processor_->Resize(*frame, out);
if (st.Failed()) {
if (!use_rga_ && st.ErrMessage().find("unsupported format") != std::string::npos) {
auto t = std::make_shared<FrameTransformMeta>(tx);
frame->transform_meta = t;
ProcessPassthrough(frame);
return NodeStatus::OK;
}
LogError("[preprocess] " + st.ErrMessage());
return NodeStatus::ERROR;
}
}
auto out_frame = std::make_shared<Frame>(out);
out_frame->pts = frame->pts;
out_frame->frame_id = frame->frame_id;
if (frame->det) {
auto det = std::make_shared<DetectionResult>(*frame->det);
if (frame->width > 0 && frame->height > 0) {
const float sx = static_cast<float>(out_w) / static_cast<float>(frame->width);
const float sy = static_cast<float>(out_h) / static_cast<float>(frame->height);
for (auto& it : det->items) {
ScaleRect(it.bbox, sx, sy, out_w, out_h);
}
}
det->img_w = out_w;
det->img_h = out_h;
out_frame->det = std::move(det);
}
if (frame->face_det) {
auto face_det = std::make_shared<FaceDetResult>(*frame->face_det);
if (frame->width > 0 && frame->height > 0) {
const float sx = static_cast<float>(out_w) / static_cast<float>(frame->width);
const float sy = static_cast<float>(out_h) / static_cast<float>(frame->height);
for (auto& it : face_det->faces) {
ScaleRect(it.bbox, sx, sy, out_w, out_h);
if (it.has_landmarks) {
for (auto& lm : it.landmarks) {
ScalePoint(lm, sx, sy, out_w, out_h);
}
}
}
}
face_det->img_w = out_w;
face_det->img_h = out_h;
out_frame->face_det = std::move(face_det);
}
if (frame->face_recog) {
auto face_recog = std::make_shared<FaceRecogResult>(*frame->face_recog);
if (frame->width > 0 && frame->height > 0) {
const float sx = static_cast<float>(out_w) / static_cast<float>(frame->width);
const float sy = static_cast<float>(out_h) / static_cast<float>(frame->height);
for (auto& it : face_recog->items) {
ScaleRect(it.bbox, sx, sy, out_w, out_h);
if (it.has_landmarks) {
for (auto& lm : it.landmarks) {
ScalePoint(lm, sx, sy, out_w, out_h);
}
}
}
}
face_recog->img_w = out_w;
face_recog->img_h = out_h;
out_frame->face_recog = std::move(face_recog);
}
out_frame->transform_meta = std::make_shared<FrameTransformMeta>(tx);
ScaleMeta(*frame, *out_frame, tx);
out_frame->user_meta = frame->user_meta;
PushToDownstream(out_frame);
@ -232,7 +416,7 @@ public:
LogInfo("[preprocess] " + std::string(use_rga_ ? "rga" : "swscale") +
" frame=" + std::to_string(out_frame->frame_id) +
" " + std::to_string(frame->width) + "x" + std::to_string(frame->height) +
" -> " + std::to_string(out_w) + "x" + std::to_string(out_h) +
" -> " + std::to_string(out_frame->width) + "x" + std::to_string(out_frame->height) +
" id=" + id_);
}
@ -240,6 +424,142 @@ public:
}
private:
Status BuildLetterbox(const Frame& input, PixelFormat out_fmt, int dst_w, int dst_h, Frame& out,
FrameTransformMeta& tx) const {
Frame src = input;
if (input.format != out_fmt) {
src.width = input.width;
src.height = input.height;
src.format = out_fmt;
Status cvt = image_processor_->CvtColor(input, src, out_fmt);
if (cvt.Failed()) return cvt;
}
if (src.width <= 0 || src.height <= 0 || dst_w <= 0 || dst_h <= 0) {
return FailStatus("invalid letterbox size");
}
float scale = std::min(static_cast<float>(dst_w) / static_cast<float>(src.width),
static_cast<float>(dst_h) / static_cast<float>(src.height));
int inner_w = std::max(1, static_cast<int>(std::round(src.width * scale)));
int inner_h = std::max(1, static_cast<int>(std::round(src.height * scale)));
if (IsYuvFormat(out_fmt)) {
inner_w = std::max(2, MakeEvenFloor(inner_w));
inner_h = std::max(2, MakeEvenFloor(inner_h));
if (((dst_w - inner_w) & 1) != 0) inner_w = std::max(2, inner_w - 2);
if (((dst_h - inner_h) & 1) != 0) inner_h = std::max(2, inner_h - 2);
}
if (inner_w <= 0 || inner_h <= 0 || inner_w > dst_w || inner_h > dst_h) {
return FailStatus("invalid inner letterbox size");
}
Frame resized;
resized.width = inner_w;
resized.height = inner_h;
resized.format = out_fmt;
Status st = image_processor_->Resize(src, resized);
if (st.Failed()) return st;
out.width = dst_w;
out.height = dst_h;
out.format = out_fmt;
if (!InitFrameStorage(out)) {
return FailStatus("alloc letterbox output failed");
}
FillBlack(out);
const int pad_x = (dst_w - inner_w) / 2;
const int pad_y = (dst_h - inner_h) / 2;
if (!BlitLetterbox(resized, out, pad_x, pad_y)) {
return FailStatus("blit letterbox failed");
}
tx.letterbox = true;
tx.src_w = input.width;
tx.src_h = input.height;
tx.dst_w = dst_w;
tx.dst_h = dst_h;
tx.scale_x = static_cast<float>(inner_w) / static_cast<float>(input.width);
tx.scale_y = static_cast<float>(inner_h) / static_cast<float>(input.height);
tx.pad_x = static_cast<float>(pad_x);
tx.pad_y = static_cast<float>(pad_y);
return OkStatus();
}
void ScaleMeta(const Frame& in_frame, Frame& out_frame, const FrameTransformMeta& tx) const {
if (in_frame.det) {
auto det = std::make_shared<DetectionResult>(*in_frame.det);
const int src_meta_w = det->img_w > 0 ? det->img_w : in_frame.width;
const int src_meta_h = det->img_h > 0 ? det->img_h : in_frame.height;
const float to_frame_x = src_meta_w > 0 ? static_cast<float>(in_frame.width) / src_meta_w : 1.0f;
const float to_frame_y = src_meta_h > 0 ? static_cast<float>(in_frame.height) / src_meta_h : 1.0f;
for (auto& it : det->items) {
TransformRect(it.bbox,
tx.scale_x * to_frame_x,
tx.scale_y * to_frame_y,
tx.pad_x, tx.pad_y,
out_frame.width, out_frame.height);
}
det->img_w = out_frame.width;
det->img_h = out_frame.height;
out_frame.det = std::move(det);
}
if (in_frame.face_det) {
auto face_det = std::make_shared<FaceDetResult>(*in_frame.face_det);
const int src_meta_w = face_det->img_w > 0 ? face_det->img_w : in_frame.width;
const int src_meta_h = face_det->img_h > 0 ? face_det->img_h : in_frame.height;
const float to_frame_x = src_meta_w > 0 ? static_cast<float>(in_frame.width) / src_meta_w : 1.0f;
const float to_frame_y = src_meta_h > 0 ? static_cast<float>(in_frame.height) / src_meta_h : 1.0f;
for (auto& it : face_det->faces) {
TransformRect(it.bbox,
tx.scale_x * to_frame_x,
tx.scale_y * to_frame_y,
tx.pad_x, tx.pad_y,
out_frame.width, out_frame.height);
if (it.has_landmarks) {
for (auto& lm : it.landmarks) {
TransformPoint(lm,
tx.scale_x * to_frame_x,
tx.scale_y * to_frame_y,
tx.pad_x, tx.pad_y,
out_frame.width, out_frame.height);
}
}
}
face_det->img_w = out_frame.width;
face_det->img_h = out_frame.height;
out_frame.face_det = std::move(face_det);
}
if (in_frame.face_recog) {
auto face_recog = std::make_shared<FaceRecogResult>(*in_frame.face_recog);
const int src_meta_w = face_recog->img_w > 0 ? face_recog->img_w : in_frame.width;
const int src_meta_h = face_recog->img_h > 0 ? face_recog->img_h : in_frame.height;
const float to_frame_x = src_meta_w > 0 ? static_cast<float>(in_frame.width) / src_meta_w : 1.0f;
const float to_frame_y = src_meta_h > 0 ? static_cast<float>(in_frame.height) / src_meta_h : 1.0f;
for (auto& it : face_recog->items) {
TransformRect(it.bbox,
tx.scale_x * to_frame_x,
tx.scale_y * to_frame_y,
tx.pad_x, tx.pad_y,
out_frame.width, out_frame.height);
if (it.has_landmarks) {
for (auto& lm : it.landmarks) {
TransformPoint(lm,
tx.scale_x * to_frame_x,
tx.scale_y * to_frame_y,
tx.pad_x, tx.pad_y,
out_frame.width, out_frame.height);
}
}
}
face_recog->img_w = out_frame.width;
face_recog->img_h = out_frame.height;
out_frame.face_recog = std::move(face_recog);
}
}
void PushToDownstream(FramePtr frame) {
for (auto& q : output_queues_) {
q->Push(frame);
@ -267,12 +587,12 @@ private:
int dst_w_ = 640;
int dst_h_ = 640;
bool keep_ratio_ = false;
ResizeMode resize_mode_ = ResizeMode::Stretch;
PixelFormat dst_fmt_ = PixelFormat::UNKNOWN;
bool use_rga_ = true;
bool stats_log_ = false;
uint64_t stats_interval_ = 100;
bool warned_meta_resize_ = false;
std::shared_ptr<SpscQueue<FramePtr>> input_queue_;