diff --git a/plugins/ai_face_det/ai_face_det_node.cpp b/plugins/ai_face_det/ai_face_det_node.cpp index 144600c..a67711a 100644 --- a/plugins/ai_face_det/ai_face_det_node.cpp +++ b/plugins/ai_face_det/ai_face_det_node.cpp @@ -330,6 +330,9 @@ public: input_dtype_ = config.ValueOr("input_dtype", input_dtype_); for (auto& c : input_dtype_) c = static_cast(std::tolower(static_cast(c))); + input_layout_ = config.ValueOr("input_layout", input_layout_); + for (auto& c : input_layout_) c = static_cast(std::tolower(static_cast(c))); + // Optional normalization when input_dtype is float. // - scale/bias: x = x*scale + bias // - mean/std: x = (x - mean[c]) / std[c] @@ -372,6 +375,13 @@ public: } } + if (const SimpleJson* dbg = config.Find("debug"); dbg && dbg->IsObject()) { + stats_log_ = dbg->ValueOr("stats", stats_log_); + stats_interval_ = std::max( + 1, static_cast(dbg->ValueOr("stats_interval", static_cast(stats_interval_)))); + log_outputs_ = dbg->ValueOr("log_outputs", log_outputs_); + } + input_queue_ = ctx.input_queue; output_queues_ = ctx.output_queues; if (!input_queue_) { @@ -434,6 +444,10 @@ public: for (auto& c : dtype) c = static_cast(std::tolower(static_cast(c))); input_dtype_ = std::move(dtype); + std::string layout = new_config.ValueOr("input_layout", input_layout_); + for (auto& c : layout) c = static_cast(std::tolower(static_cast(c))); + input_layout_ = std::move(layout); + if (const SimpleJson* norm = new_config.Find("normalize"); norm && norm->IsObject()) { bool use_ms = false; if (const SimpleJson* mean = norm->Find("mean"); mean && mean->IsArray() && mean->AsArray().size() >= 3) { @@ -545,7 +559,8 @@ private: InferInput input; input.width = in_w; input.height = in_h; - input.is_nhwc = true; + const bool want_nchw = (input_layout_ == "nchw"); + input.is_nhwc = !want_nchw; // Default: keep existing UINT8 behavior. if (input_dtype_ == "float" || input_dtype_ == "f32" || input_dtype_ == "float32") { @@ -565,12 +580,39 @@ private: } } - input.data = float_input_buf_.data(); - input.size = float_input_buf_.size() * sizeof(float); + const float* fp = float_input_buf_.data(); + if (want_nchw) { + float_input_nchw_buf_.resize(pix * 3); + for (int c = 0; c < 3; ++c) { + float* dst = float_input_nchw_buf_.data() + static_cast(c) * pix; + for (size_t i = 0; i < pix; ++i) { + dst[i] = fp[i * 3 + static_cast(c)]; + } + } + input.data = float_input_nchw_buf_.data(); + input.size = float_input_nchw_buf_.size() * sizeof(float); + } else { + input.data = float_input_buf_.data(); + input.size = float_input_buf_.size() * sizeof(float); + } input.type = RKNN_TENSOR_FLOAT32; } else { - input.data = input_ptr; - input.size = in_size; + if (want_nchw) { + const size_t pix = static_cast(in_w) * static_cast(in_h); + input_nchw_buf_.resize(pix * 3); + const uint8_t* p = reinterpret_cast(input_ptr); + for (int c = 0; c < 3; ++c) { + uint8_t* dst = input_nchw_buf_.data() + static_cast(c) * pix; + for (size_t i = 0; i < pix; ++i) { + dst[i] = p[i * 3 + static_cast(c)]; + } + } + input.data = input_nchw_buf_.data(); + input.size = input_nchw_buf_.size(); + } else { + input.data = input_ptr; + input.size = in_size; + } input.type = RKNN_TENSOR_UINT8; } @@ -580,6 +622,31 @@ private: return; } + if (log_outputs_ && !r.outputs.empty()) { + // Print basic output tensor info a few times for debugging model mismatches. + if (printed_outputs_ < 3) { + ++printed_outputs_; + for (size_t i = 0; i < r.outputs.size(); ++i) { + const auto& o = r.outputs[i]; + std::string shape; + shape.reserve(64); + shape.push_back('['); + for (size_t d = 0; d < o.dims.size(); ++d) { + shape += std::to_string(o.dims[d]); + if (d + 1 < o.dims.size()) shape += ","; + } + shape.push_back(']'); + std::cerr << "[ai_face_det] out" << i + << " type=" << static_cast(o.type) + << " zp=" << o.zp + << " scale=" << o.scale + << " dims=" << shape + << " size=" << o.size + << "\n"; + } + } + } + std::vector tensors; tensors.reserve(r.outputs.size()); for (const auto& o : r.outputs) { @@ -599,6 +666,18 @@ private: det.model_name = "retinaface"; DecodeRetinaFace(tensors, src_w, src_h, in_w, in_h, det); + + ++processed_; + if (stats_log_ && stats_interval_ > 0 && (processed_ % stats_interval_) == 0) { + const size_t n = det.faces.size(); + const float best = n > 0 ? det.faces[0].score : 0.0f; + std::cerr << "[ai_face_det] frame=" << frame->frame_id + << " faces=" << n + << " best=" << best + << " conf_thr=" << conf_thresh_ + << " nms=" << nms_thresh_ + << "\n"; + } frame->face_det = std::make_shared(std::move(det)); } @@ -608,7 +687,8 @@ private: FaceDetResult& out) { // Find loc/conf/landms tensors. std::vector locs; - std::vector confs; + std::vector confs; // Nx2 + std::vector confs1; // Nx1 (some models) std::vector landms; locs.reserve(4); confs.reserve(4); @@ -624,12 +704,16 @@ private: confs.push_back(std::move(tmp)); continue; } + if (ExtractNc(t, 1, tmp)) { + confs1.push_back(std::move(tmp)); + continue; + } if (ExtractNc(t, 10, tmp)) { landms.push_back(std::move(tmp)); continue; } } - if (locs.empty() || confs.empty()) return; + if (locs.empty() || (confs.empty() && confs1.empty())) return; // Concatenate along N. auto Concat = [](const std::vector& parts) -> NcTensor { @@ -650,7 +734,15 @@ private: }; NcTensor loc = Concat(locs); - NcTensor conf = Concat(confs); + NcTensor conf; + bool conf_is_2 = false; + if (!confs.empty()) { + conf = Concat(confs); + conf_is_2 = true; + } else { + conf = Concat(confs1); + conf_is_2 = false; + } NcTensor lmk; if (output_landmarks_ && !landms.empty()) lmk = Concat(landms); @@ -678,13 +770,19 @@ private: constexpr float var1 = 0.2f; for (int i = 0; i < n; ++i) { - const float s0 = conf.data[static_cast(i) * 2 + 0]; - const float s1 = conf.data[static_cast(i) * 2 + 1]; - float score; - if (s0 >= 0.0f && s0 <= 1.0f && s1 >= 0.0f && s1 <= 1.0f && std::fabs((s0 + s1) - 1.0f) < 0.1f) { - score = s1; + float score = 0.0f; + if (conf_is_2) { + const float s0 = conf.data[static_cast(i) * 2 + 0]; + const float s1 = conf.data[static_cast(i) * 2 + 1]; + if (s0 >= 0.0f && s0 <= 1.0f && s1 >= 0.0f && s1 <= 1.0f && std::fabs((s0 + s1) - 1.0f) < 0.1f) { + score = s1; + } else { + score = Softmax2(s0, s1); + } } else { - score = Softmax2(s0, s1); + const float s = conf.data[static_cast(i)]; + if (s >= 0.0f && s <= 1.0f) score = s; + else score = Sigmoid(s); } if (score < conf_thresh_) continue; @@ -772,6 +870,7 @@ private: // Model input dtype: "uint8" (default) or "float32". std::string input_dtype_ = "uint8"; + std::string input_layout_ = "nhwc"; float norm_scale_ = 1.0f; float norm_bias_ = 0.0f; bool norm_use_mean_std_ = false; @@ -785,12 +884,20 @@ private: std::vector>> output_queues_; std::vector input_buf_; + std::vector input_nchw_buf_; std::vector float_input_buf_; + std::vector float_input_nchw_buf_; ModelHandle model_handle_ = kInvalidModelHandle; int model_w_ = 320; int model_h_ = 320; uint32_t n_output_ = 0; + + bool stats_log_ = false; + uint64_t stats_interval_ = 100; + bool log_outputs_ = false; + uint64_t processed_ = 0; + int printed_outputs_ = 0; }; REGISTER_NODE(AiFaceDetNode, "ai_face_det"); diff --git a/plugins/ai_face_recog/ai_face_recog_node.cpp b/plugins/ai_face_recog/ai_face_recog_node.cpp index f0c5b1e..1c20833 100644 --- a/plugins/ai_face_recog/ai_face_recog_node.cpp +++ b/plugins/ai_face_recog/ai_face_recog_node.cpp @@ -572,6 +572,9 @@ public: input_dtype_ = config.ValueOr("input_dtype", input_dtype_); for (auto& c : input_dtype_) c = static_cast(std::tolower(static_cast(c))); + input_layout_ = config.ValueOr("input_layout", input_layout_); + for (auto& c : input_layout_) c = static_cast(std::tolower(static_cast(c))); + if (const SimpleJson* norm = config.Find("normalize"); norm && norm->IsObject()) { if (const SimpleJson* mean = norm->Find("mean"); mean && mean->IsArray() && mean->AsArray().size() >= 3) { for (int i = 0; i < 3; ++i) { @@ -667,6 +670,10 @@ public: for (auto& c : dtype) c = static_cast(std::tolower(static_cast(c))); input_dtype_ = std::move(dtype); + std::string layout = new_config.ValueOr("input_layout", input_layout_); + for (auto& c : layout) c = static_cast(std::tolower(static_cast(c))); + input_layout_ = std::move(layout); + if (const SimpleJson* norm = new_config.Find("normalize"); norm && norm->IsObject()) { bool use_ms = false; if (const SimpleJson* mean = norm->Find("mean"); mean && mean->IsArray() && mean->AsArray().size() >= 3) { @@ -818,7 +825,8 @@ private: InferInput in; in.width = model_w_; in.height = model_h_; - in.is_nhwc = true; + const bool want_nchw = (input_layout_ == "nchw"); + in.is_nhwc = !want_nchw; if (input_dtype_ == "float" || input_dtype_ == "f32" || input_dtype_ == "float32") { float_input_buf_.resize(static_cast(model_w_) * static_cast(model_h_) * 3); @@ -837,12 +845,39 @@ private: } } - in.data = float_input_buf_.data(); - in.size = float_input_buf_.size() * sizeof(float); + const float* fp = float_input_buf_.data(); + if (want_nchw) { + float_input_nchw_buf_.resize(pix * 3); + for (int c = 0; c < 3; ++c) { + float* dst = float_input_nchw_buf_.data() + static_cast(c) * pix; + for (size_t i = 0; i < pix; ++i) { + dst[i] = fp[i * 3 + static_cast(c)]; + } + } + in.data = float_input_nchw_buf_.data(); + in.size = float_input_nchw_buf_.size() * sizeof(float); + } else { + in.data = float_input_buf_.data(); + in.size = float_input_buf_.size() * sizeof(float); + } in.type = RKNN_TENSOR_FLOAT32; } else { - in.data = face_buf_.data(); - in.size = face_buf_.size(); + if (want_nchw) { + const size_t pix = static_cast(model_w_) * static_cast(model_h_); + input_nchw_buf_.resize(pix * 3); + const uint8_t* p = face_buf_.data(); + for (int c = 0; c < 3; ++c) { + uint8_t* dst = input_nchw_buf_.data() + static_cast(c) * pix; + for (size_t i = 0; i < pix; ++i) { + dst[i] = p[i * 3 + static_cast(c)]; + } + } + in.data = input_nchw_buf_.data(); + in.size = input_nchw_buf_.size(); + } else { + in.data = face_buf_.data(); + in.size = face_buf_.size(); + } in.type = RKNN_TENSOR_UINT8; } @@ -899,6 +934,7 @@ private: std::string model_input_format_ = "rgb"; std::string input_dtype_ = "uint8"; + std::string input_layout_ = "nhwc"; float norm_scale_ = 1.0f; float norm_bias_ = 0.0f; bool norm_use_mean_std_ = false; @@ -916,7 +952,9 @@ private: std::vector>> output_queues_; std::vector face_buf_; + std::vector input_nchw_buf_; std::vector float_input_buf_; + std::vector float_input_nchw_buf_; ModelHandle model_handle_ = kInvalidModelHandle; int model_w_ = 112; diff --git a/plugins/osd/osd_node.cpp b/plugins/osd/osd_node.cpp index 14a7cf2..2a5d76e 100644 --- a/plugins/osd/osd_node.cpp +++ b/plugins/osd/osd_node.cpp @@ -10,6 +10,7 @@ #include "face/face_result.h" #include "node.h" +#include "utils/dma_alloc.h" #include "utils/logger.h" namespace rk3588 { @@ -335,9 +336,18 @@ public: if (out.use_count() > 1) { out = CloneFrameForWrite(out); } + + // If we are drawing directly onto a DMA-BUF backed frame, we must sync CPU caches, + // otherwise downstream HW (RGA/encoder) may not see the updated pixels. + if (out->dma_fd >= 0 && out->data) { + DmaSyncStartFd(out->dma_fd); + } DrawDetections(out); DrawFaceDet(out); DrawFaceRecog(out); + if (out->dma_fd >= 0 && out->data) { + DmaSyncEndFd(out->dma_fd); + } } PushToDownstream(out); diff --git a/src/ai_scheduler.cpp b/src/ai_scheduler.cpp index bdf0646..62ec7e1 100644 --- a/src/ai_scheduler.cpp +++ b/src/ai_scheduler.cpp @@ -174,6 +174,8 @@ ModelHandle AiScheduler::LoadModel(const std::string& model_path, std::string& e " (handle=" + std::to_string(handle) + ", input=" + std::to_string(ctx->input_w) + "x" + std::to_string(ctx->input_h) + "x" + std::to_string(ctx->input_c) + + ", fmt=" + std::string(ctx->input_attrs[0].fmt == RKNN_TENSOR_NCHW ? "NCHW" : "NHWC") + + ", dtype=" + std::to_string(static_cast(ctx->input_attrs[0].type)) + ", outputs=" + std::to_string(ctx->n_output) + ")"); return handle;