增加640像素的人脸检测模型scrfd；探索分区检测方案和实现；重新转换retinaFace320的rknn模型
This commit is contained in:
parent
4b5359f64e
commit
fdcb7f7c6c
251
configs/sample_cam1_face_zoned.json
Normal file
251
configs/sample_cam1_face_zoned.json
Normal file
@ -0,0 +1,251 @@
|
||||
{
|
||||
"queue": {
|
||||
"size": 8,
|
||||
"strategy": "drop_oldest"
|
||||
},
|
||||
"graphs": [
|
||||
{
|
||||
"name": "cam1_face_zoned_detection",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "in_cam1",
|
||||
"type": "input_rtsp",
|
||||
"role": "source",
|
||||
"enable": true,
|
||||
"url": "rtsp://10.0.0.49:8554/cam",
|
||||
"fps": 30,
|
||||
"width": 2560,
|
||||
"height": 1440,
|
||||
"use_mpp": true,
|
||||
"use_ffmpeg": false,
|
||||
"force_tcp": true,
|
||||
"reconnect_sec": 5,
|
||||
"reconnect_backoff_max_sec": 30
|
||||
},
|
||||
{
|
||||
"id": "pre_cam1",
|
||||
"type": "preprocess",
|
||||
"role": "filter",
|
||||
"enable": true,
|
||||
"dst_w": 2560,
|
||||
"dst_h": 1440,
|
||||
"dst_format": "rgb",
|
||||
"dst_packed": true,
|
||||
"resize_mode": "stretch",
|
||||
"keep_ratio": false,
|
||||
"rga_gate": "cam1_face_zoned_detection",
|
||||
"use_rga": true
|
||||
},
|
||||
{
|
||||
"id": "face_det_cam1",
|
||||
"type": "ai_face_det_zoned",
|
||||
"role": "filter",
|
||||
"enable": true,
|
||||
"model_path": "./models/RetinaFace_mobile320.rknn",
|
||||
"model_w": 320,
|
||||
"model_h": 320,
|
||||
"conf": 0.7,
|
||||
"nms": 0.4,
|
||||
"nms_global": 0.5,
|
||||
"max_faces": 10,
|
||||
"output_landmarks": true,
|
||||
"input_format": "rgb",
|
||||
"infer_fps": 5,
|
||||
"roi": {
|
||||
"x": 0,
|
||||
"y": 254,
|
||||
"w": 2560,
|
||||
"h": 881
|
||||
},
|
||||
"zones": {
|
||||
"near_zone": {
|
||||
"y_start": 0,
|
||||
"y_end": 366,
|
||||
"scale": 1.0,
|
||||
"description": "3-5m distance"
|
||||
},
|
||||
"mid_zone": {
|
||||
"y_start": 366,
|
||||
"y_end": 720,
|
||||
"scale": 1.3,
|
||||
"description": "5-7m distance"
|
||||
},
|
||||
"far_zone": {
|
||||
"y_start": 720,
|
||||
"y_end": 881,
|
||||
"scale": 1.8,
|
||||
"description": "7-9m distance"
|
||||
}
|
||||
},
|
||||
"debug": {
|
||||
"stats": true,
|
||||
"stats_interval": 30,
|
||||
"detections": true,
|
||||
"draw_zones": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "face_recog_cam1",
|
||||
"type": "ai_face_recog",
|
||||
"role": "filter",
|
||||
"enable": true,
|
||||
"model_path": "./models/mobilefacenet_arcface.rknn",
|
||||
"align": true,
|
||||
"emit_embedding": false,
|
||||
"max_faces": 10,
|
||||
"input_format": "rgb",
|
||||
"input_dtype": "uint8",
|
||||
"threshold": {
|
||||
"accept": 0.45,
|
||||
"margin": 0.05
|
||||
},
|
||||
"gallery": {
|
||||
"backend": "sqlite",
|
||||
"path": "./models/face_gallery.db",
|
||||
"load_on_start": true,
|
||||
"expected_dim": 512,
|
||||
"dtype": "auto"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "osd_cam1",
|
||||
"type": "osd",
|
||||
"role": "filter",
|
||||
"enable": true,
|
||||
"draw_bbox": true,
|
||||
"draw_text": true,
|
||||
"draw_face_det": true,
|
||||
"draw_face_bbox": true,
|
||||
"line_width": 2,
|
||||
"font_scale": 1,
|
||||
"use_rga_bbox": false,
|
||||
"labels": ["face"]
|
||||
},
|
||||
{
|
||||
"id": "post_cam1",
|
||||
"type": "preprocess",
|
||||
"role": "filter",
|
||||
"enable": true,
|
||||
"dst_w": 1280,
|
||||
"dst_h": 720,
|
||||
"dst_format": "nv12",
|
||||
"resize_mode": "stretch",
|
||||
"rga_gate": "cam1_face_zoned_detection",
|
||||
"use_rga": true
|
||||
},
|
||||
{
|
||||
"id": "pub_cam1",
|
||||
"type": "publish",
|
||||
"role": "filter",
|
||||
"enable": true,
|
||||
"queue": {"size": 2, "policy": "drop_oldest"},
|
||||
"codec": "h264",
|
||||
"fps": 30,
|
||||
"gop": 60,
|
||||
"bitrate_kbps": 2000,
|
||||
"use_mpp": true,
|
||||
"use_ffmpeg_mux": true,
|
||||
"outputs": [
|
||||
{
|
||||
"proto": "hls",
|
||||
"path": "./web/hls/cam1/index.m3u8",
|
||||
"segment_sec": 2
|
||||
},
|
||||
{
|
||||
"proto": "rtsp_server",
|
||||
"port": 8555,
|
||||
"path": "/live/cam1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "alarm_cam1",
|
||||
"type": "alarm",
|
||||
"role": "sink",
|
||||
"enable": true,
|
||||
"eval_fps": 10,
|
||||
"labels": ["face"],
|
||||
"face_rules": [
|
||||
{
|
||||
"name": "unknown_face",
|
||||
"type": "unknown",
|
||||
"cooldown_ms": 7000,
|
||||
"min_sim": 0.35,
|
||||
"min_hits": 2,
|
||||
"hit_window_ms": 1500,
|
||||
"min_face_area_ratio": 0.01,
|
||||
"min_face_aspect": 0.6,
|
||||
"max_face_aspect": 1.6
|
||||
},
|
||||
{
|
||||
"name": "known_person",
|
||||
"type": "person",
|
||||
"cooldown_ms": 7000,
|
||||
"min_sim": 0.6,
|
||||
"min_hits": 2,
|
||||
"hit_window_ms": 1500,
|
||||
"min_face_area_ratio": 0.01,
|
||||
"min_face_aspect": 0.6,
|
||||
"max_face_aspect": 1.6
|
||||
}
|
||||
],
|
||||
"actions": {
|
||||
"log": {
|
||||
"enable": true,
|
||||
"level": "info"
|
||||
},
|
||||
"snapshot": {
|
||||
"enable": true,
|
||||
"format": "jpg",
|
||||
"quality": 85,
|
||||
"upload": {
|
||||
"type": "minio",
|
||||
"endpoint": "http://10.0.0.49:9000",
|
||||
"bucket": "myminio",
|
||||
"region": "us-east-1",
|
||||
"access_key": "minioadmin",
|
||||
"secret_key": "minioadmin"
|
||||
}
|
||||
},
|
||||
"clip": {
|
||||
"enable": true,
|
||||
"pre_sec": 5,
|
||||
"post_sec": 10,
|
||||
"format": "mp4",
|
||||
"fps": 30,
|
||||
"upload": {
|
||||
"type": "minio",
|
||||
"endpoint": "http://10.0.0.49:9000",
|
||||
"bucket": "myminio",
|
||||
"region": "us-east-1",
|
||||
"access_key": "minioadmin",
|
||||
"secret_key": "minioadmin"
|
||||
}
|
||||
},
|
||||
"external_api": {
|
||||
"enable": true,
|
||||
"getTokenUrl": "http://10.0.0.49:8080/api/getToken",
|
||||
"putMessageUrl": "http://10.0.0.49:8080/api/putMessage",
|
||||
"tenantCode": "32",
|
||||
"channelNo": "cam1",
|
||||
"timeout_ms": 3000,
|
||||
"include_media_url": true,
|
||||
"token_header": "X-Access-Token",
|
||||
"token_json_path": "responseBody.token",
|
||||
"token_cache_sec": 1200
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
["in_cam1", "pre_cam1"],
|
||||
["pre_cam1", "face_det_cam1"],
|
||||
["face_det_cam1", "face_recog_cam1"],
|
||||
["face_recog_cam1", "osd_cam1"],
|
||||
["osd_cam1", "post_cam1"],
|
||||
["post_cam1", "pub_cam1"],
|
||||
["pub_cam1", "alarm_cam1"]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
109
configs/test_face_only.json
Normal file
109
configs/test_face_only.json
Normal file
@ -0,0 +1,109 @@
|
||||
{
|
||||
"queue": {
|
||||
"size": 8,
|
||||
"strategy": "drop_oldest"
|
||||
},
|
||||
"graphs": [
|
||||
{
|
||||
"name": "test_face_only",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "in_cam",
|
||||
"type": "input_rtsp",
|
||||
"role": "source",
|
||||
"enable": true,
|
||||
"url": "rtsp://10.0.0.49:8554/cam",
|
||||
"fps": 30,
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"use_mpp": true,
|
||||
"use_ffmpeg": false,
|
||||
"force_tcp": true,
|
||||
"reconnect_sec": 5,
|
||||
"reconnect_backoff_max_sec": 30
|
||||
},
|
||||
{
|
||||
"id": "pre",
|
||||
"type": "preprocess",
|
||||
"role": "filter",
|
||||
"enable": true,
|
||||
"dst_w": 1024,
|
||||
"dst_h": 1024,
|
||||
"dst_format": "rgb",
|
||||
"dst_packed": true,
|
||||
"resize_mode": "stretch",
|
||||
"keep_ratio": false,
|
||||
"use_rga": true
|
||||
},
|
||||
{
|
||||
"id": "face_det",
|
||||
"type": "ai_face_det",
|
||||
"role": "filter",
|
||||
"enable": true,
|
||||
"model_path": "./models/RetinaFace_mobile320.rknn",
|
||||
"conf": 0.05,
|
||||
"nms": 0.4,
|
||||
"max_faces": 5,
|
||||
"output_landmarks": true,
|
||||
"input_format": "rgb"
|
||||
},
|
||||
{
|
||||
"id": "osd",
|
||||
"type": "osd",
|
||||
"role": "filter",
|
||||
"enable": true,
|
||||
"draw_bbox": false,
|
||||
"draw_text": true,
|
||||
"draw_face_det": true,
|
||||
"draw_face_bbox": true,
|
||||
"use_rga_bbox": false,
|
||||
"line_width": 2,
|
||||
"font_scale": 1,
|
||||
"labels": []
|
||||
},
|
||||
{
|
||||
"id": "post",
|
||||
"type": "preprocess",
|
||||
"role": "filter",
|
||||
"enable": true,
|
||||
"dst_w": 1280,
|
||||
"dst_h": 720,
|
||||
"dst_format": "nv12",
|
||||
"resize_mode": "stretch",
|
||||
"use_rga": true
|
||||
},
|
||||
{
|
||||
"id": "pub",
|
||||
"type": "publish",
|
||||
"role": "sink",
|
||||
"enable": true,
|
||||
"codec": "h264",
|
||||
"fps": 30,
|
||||
"gop": 60,
|
||||
"bitrate_kbps": 2000,
|
||||
"use_mpp": true,
|
||||
"use_ffmpeg_mux": true,
|
||||
"outputs": [
|
||||
{
|
||||
"proto": "hls",
|
||||
"path": "./web/hls/test_face/index.m3u8",
|
||||
"segment_sec": 2
|
||||
},
|
||||
{
|
||||
"proto": "rtsp_server",
|
||||
"port": 8555,
|
||||
"path": "/live/test_face"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
["in_cam", "pre"],
|
||||
["pre", "face_det"],
|
||||
["face_det", "osd"],
|
||||
["osd", "post"],
|
||||
["post", "pub"]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
112
configs/test_scrfd_640.json
Normal file
112
configs/test_scrfd_640.json
Normal file
@ -0,0 +1,112 @@
|
||||
{
|
||||
"queue": {
|
||||
"size": 8,
|
||||
"strategy": "drop_oldest"
|
||||
},
|
||||
"graphs": [
|
||||
{
|
||||
"name": "scrfd_640_test",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "in_cam1",
|
||||
"type": "input_rtsp",
|
||||
"role": "source",
|
||||
"enable": true,
|
||||
"url": "rtsp://10.0.0.49:8554/cam",
|
||||
"fps": 30,
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"use_mpp": true,
|
||||
"use_ffmpeg": false,
|
||||
"force_tcp": true,
|
||||
"reconnect_sec": 5,
|
||||
"reconnect_backoff_max_sec": 30
|
||||
},
|
||||
{
|
||||
"id": "pre_cam1",
|
||||
"type": "preprocess",
|
||||
"role": "filter",
|
||||
"enable": true,
|
||||
"dst_w": 640,
|
||||
"dst_h": 640,
|
||||
"dst_format": "rgb",
|
||||
"dst_packed": true,
|
||||
"resize_mode": "stretch",
|
||||
"keep_ratio": false,
|
||||
"rga_gate": "scrfd_640_test",
|
||||
"use_rga": true
|
||||
},
|
||||
{
|
||||
"id": "scrfd",
|
||||
"type": "ai_scrfd",
|
||||
"role": "filter",
|
||||
"enable": true,
|
||||
"model_path": "./models/scrfd_500m_640.rknn",
|
||||
"conf_thresh": 0.5,
|
||||
"nms_thresh": 0.4,
|
||||
"max_faces": 10,
|
||||
"output_landmarks": true,
|
||||
"input_format": "rgb"
|
||||
},
|
||||
{
|
||||
"id": "osd_cam1",
|
||||
"type": "osd",
|
||||
"role": "filter",
|
||||
"enable": true,
|
||||
"draw_bbox": true,
|
||||
"draw_text": true,
|
||||
"draw_face_det": true,
|
||||
"draw_face_bbox": true,
|
||||
"line_width": 2,
|
||||
"font_scale": 1,
|
||||
"use_rga_bbox": false,
|
||||
"labels": ["face"]
|
||||
},
|
||||
{
|
||||
"id": "post_cam1",
|
||||
"type": "preprocess",
|
||||
"role": "filter",
|
||||
"enable": true,
|
||||
"dst_w": 1280,
|
||||
"dst_h": 720,
|
||||
"dst_format": "nv12",
|
||||
"resize_mode": "stretch",
|
||||
"rga_gate": "scrfd_640_test",
|
||||
"use_rga": true
|
||||
},
|
||||
{
|
||||
"id": "pub_cam1",
|
||||
"type": "publish",
|
||||
"role": "filter",
|
||||
"enable": true,
|
||||
"queue": {"size": 2, "policy": "drop_oldest"},
|
||||
"codec": "h264",
|
||||
"fps": 30,
|
||||
"gop": 60,
|
||||
"bitrate_kbps": 2000,
|
||||
"use_mpp": true,
|
||||
"use_ffmpeg_mux": true,
|
||||
"outputs": [
|
||||
{
|
||||
"proto": "hls",
|
||||
"path": "./web/hls/scrfd/index.m3u8",
|
||||
"segment_sec": 2
|
||||
},
|
||||
{
|
||||
"proto": "rtsp_server",
|
||||
"port": 8555,
|
||||
"path": "/live/cam1"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
["in_cam1", "pre_cam1"],
|
||||
["pre_cam1", "scrfd"],
|
||||
["scrfd", "osd_cam1"],
|
||||
["osd_cam1", "post_cam1"],
|
||||
["post_cam1", "pub_cam1"]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
148
docs/bugfix/001-scrfd-rknn-crash.md
Normal file
148
docs/bugfix/001-scrfd-rknn-crash.md
Normal file
@ -0,0 +1,148 @@
|
||||
# SCRFD RKNN 推理崩溃问题
|
||||
|
||||
## 问题现象
|
||||
|
||||
SCRFD 640x640 人脸检测模型在推理时立即崩溃：
|
||||
|
||||
```
|
||||
terminate called after throwing an instance of 'std::out_of_range'
|
||||
what(): vector::_M_range_check: __n (which is 18446744073709551615) >= this->size() (which is 1)
|
||||
```
|
||||
|
||||
崩溃发生在 `rknn_inputs_set()` 调用时。
|
||||
|
||||
## 环境信息
|
||||
|
||||
- **平台**: Orange Pi 5 Plus (RK3588)
|
||||
- **RKNN 运行时**: 最初 1.5.2，后更新到 2.3.2
|
||||
- **RKNN-Toolkit2**: 2.3.2
|
||||
- **模型**: SCRFD 2.5G 640 (版本 2.3.2)
|
||||
|
||||
## 根因分析
|
||||
|
||||
### 1. RKNN 库版本不匹配
|
||||
|
||||
系统最初安装的 `librknnrt.so` 是 **1.5.2** 版本：
|
||||
```bash
|
||||
$ strings /usr/local/lib/librknnrt.so | grep "librknnrt version"
|
||||
librknnrt version: 1.5.2 (c6b7b351a@2023-08-23T15:28:22)
|
||||
```
|
||||
|
||||
但 SCRFD 模型是用 **2.3.2** 版本的 toolkit 编译的：
|
||||
```bash
|
||||
$ strings scrfd_2.5g_640.rknn | grep "compiler version"
|
||||
2.3.2(compiler version: 2.3.2 (@2025-04-03T08:26:16))
|
||||
```
|
||||
|
||||
### 2. 动态形状模型兼容性问题
|
||||
|
||||
SCRFD 2.3.2 模型使用**动态形状** (`dynamic_shape`)：
|
||||
- 输入/输出维度在运行时才确定
|
||||
- C API `rknn_inputs_set()` 无法正确处理
|
||||
- Python API (rknnlite) 可以正常工作
|
||||
|
||||
对比不同版本的模型输出维度：
|
||||
|
||||
| 模型 | 版本 | 输出维度示例 | 是否崩溃 |
|
||||
|------|------|-------------|---------|
|
||||
| SCRFD 2.5G 640 | 2.3.2 | `[12800, 1]` (2D, 动态) | ✅ 崩溃 |
|
||||
| SCRFD 500M 640 | 1.4.1b16 | `[1, 12800, 1, 1]` (4D, 静态) | ✅ 正常 |
|
||||
| RetinaFace 320 | 2.3.2 | `[1, 4200, 4]` (静态) | ✅ 正常 |
|
||||
| YOLOv8n 640 | 2.3.2 | `[1, 84, 8400]` (静态) | ✅ 正常 |
|
||||
|
||||
## 解决方案
|
||||
|
||||
### 步骤 1: 更新 RKNN 运行时库
|
||||
|
||||
将系统库更新到与模型编译版本一致：
|
||||
|
||||
```bash
|
||||
# 备份旧版本
|
||||
sudo cp /usr/local/lib/librknnrt.so /usr/local/lib/librknnrt.so.1.5.2.backup
|
||||
|
||||
# 删除旧版本
|
||||
sudo rm /usr/local/lib/librknnrt.so
|
||||
|
||||
# 链接新版本 (系统已安装的 2.3.2)
|
||||
sudo ln -s /usr/lib/librknnrt.so /usr/local/lib/librknnrt.so
|
||||
|
||||
# 验证
|
||||
strings /usr/local/lib/librknnrt.so | grep "librknnrt version"
|
||||
# 应输出: librknnrt version: 2.3.2 (429f97ae6b@2025-04-09T09:09:27)
|
||||
```
|
||||
|
||||
### 步骤 2: 使用兼容的模型版本
|
||||
|
||||
由于动态形状模型在 C API 中存在兼容性问题，建议使用**静态形状**版本：
|
||||
|
||||
**推荐模型**: `scrfd_500m_640.rknn` (版本 1.4.1b16)
|
||||
- 版本: 1.4.1b16-dad86923
|
||||
- 编译时间: 2022-11-26
|
||||
- 输入: 640x640x3, NHWC
|
||||
- 输出: 9个张量 (scores_8/16/32, bbox_8/16/32, kps_8/16/32)
|
||||
|
||||
**不兼容模型**: `scrfd_2.5g_640.rknn` (版本 2.3.2)
|
||||
- 动态形状导致 C API 崩溃
|
||||
- 仅能通过 Python rknnlite 使用
|
||||
|
||||
### 步骤 3: 正确的 RKNN 输入配置
|
||||
|
||||
对于量化 INT8 模型，正确的输入配置：
|
||||
|
||||
```cpp
|
||||
InferInput input;
|
||||
input.type = RKNN_TENSOR_UINT8; // 传递 UINT8，让 RKNN 自动量化
|
||||
input.is_nhwc = true;
|
||||
input.data = input_buf;
|
||||
input.size = 640 * 640 * 3;
|
||||
|
||||
// rknn_input 结构体
|
||||
rknn_input inputs[1];
|
||||
inputs[0].index = 0;
|
||||
inputs[0].type = RKNN_TENSOR_UINT8;
|
||||
inputs[0].size = input.size;
|
||||
inputs[0].fmt = RKNN_TENSOR_NHWC;
|
||||
inputs[0].buf = input_buf;
|
||||
inputs[0].pass_through = 0; // 关键：0 表示需要 RKNN 进行类型转换
|
||||
```
|
||||
|
||||
**关键点**:
|
||||
- `pass_through = 0`: 让 RKNN 自动将 UINT8 转换为模型需要的 INT8
|
||||
- `pass_through = 1`: 直接传递数据，需要数据已经是 INT8 格式且维度匹配
|
||||
|
||||
## 验证方法
|
||||
|
||||
### 验证库版本
|
||||
```bash
|
||||
strings /usr/local/lib/librknnrt.so | grep "librknnrt version"
|
||||
strings /usr/lib/librknnrt.so | grep "librknnrt version"
|
||||
```
|
||||
|
||||
### 验证模型版本
|
||||
```bash
|
||||
strings model.rknn | grep "compiler version" | head -1
|
||||
```
|
||||
|
||||
### 最小化 C++ 测试
|
||||
```cpp
|
||||
#include <rknn_api.h>
|
||||
|
||||
// 1. rknn_init()
|
||||
// 2. rknn_query() 获取输入属性
|
||||
// 3. rknn_inputs_set() 设置输入
|
||||
// 4. rknn_run() 运行推理
|
||||
// 5. rknn_outputs_get() 获取输出
|
||||
```
|
||||
|
||||
## 经验总结
|
||||
|
||||
1. **版本一致性**: RKNN 运行时库版本必须与模型编译版本一致
|
||||
2. **动态形状**: C API 对动态形状模型支持不完善，优先使用静态形状模型
|
||||
3. **输入配置**: 量化模型的 `pass_through` 和 `type` 必须正确配置
|
||||
4. **调试技巧**: 先用 Python (rknnlite) 验证模型可用，再移植到 C++
|
||||
|
||||
## 参考链接
|
||||
|
||||
- [RKNN API 文档](https://github.com/rockchip-linux/rknpu2/blob/master/doc/Rockchip_RKNPU_User_Guide_RKNN_API_V1.4.0_EN.pdf)
|
||||
- [SCRFD 原始仓库](https://github.com/deepinsight/insightface/tree/master/detection/scrfd)
|
||||
- [RKNN-Toolkit2 版本兼容性](https://github.com/airockchip/rknn-toolkit2)
|
||||
86
docs/bugfix/002-scrfd-cpp-implementation.md
Normal file
86
docs/bugfix/002-scrfd-cpp-implementation.md
Normal file
@ -0,0 +1,86 @@
|
||||
# SCRFD C++ 实现完善
|
||||
|
||||
## 实现内容
|
||||
|
||||
完善了 `ai_scrfd` 插件的 C++ 实现，包括：
|
||||
|
||||
### 1. 输入预处理
|
||||
- 双线性插值 resize 到 640x640
|
||||
- 支持 RGB/BGR 格式转换
|
||||
- DMA 缓冲区同步
|
||||
|
||||
### 2. 推理流程
|
||||
- 使用 `InferBorrowed` 进行零拷贝推理
|
||||
- UINT8 输入，RKNN 内部自动量化到 INT8
|
||||
- `pass_through=0` 让 RKNN 处理类型转换
|
||||
|
||||
### 3. 后处理逻辑
|
||||
- **维度处理**: 适配 SCRFD 500M (v1.4.1) 的输出格式
|
||||
- scores: `[1, 12800/3200/800, 1, 1]` (2 channels: bg, fg)
|
||||
- bboxes: `[1, 12800/3200/800, 4, 1]` (dx, dy, dw, dh)
|
||||
- keypoints: `[1, 12800/3200/800, 10, 1]` (5 points x 2 coords)
|
||||
|
||||
- **解码逻辑**:
|
||||
- Score: sigmoid(bg_score - fg_score) 获取前景概率
|
||||
- BBox: anchor中心 + 偏移量，exp(dw/dh) 解码宽高
|
||||
- Keypoints: anchor中心 + 相对偏移量
|
||||
|
||||
- **坐标映射**: 从 640x640 映射回原图分辨率
|
||||
|
||||
### 4. NMS (非极大值抑制)
|
||||
- 按置信度排序
|
||||
- IoU 阈值 0.4 去除重叠检测框
|
||||
- 限制最大检测数量 (默认 50)
|
||||
|
||||
## 关键代码片段
|
||||
|
||||
```cpp
|
||||
// Score解码 (sigmoid)
|
||||
float score = 1.0f / (1.0f + std::exp(bg_score - fg_score));
|
||||
|
||||
// BBox解码 (相对于anchor)
|
||||
float cx = anchor.cx + dx * stride;
|
||||
float cy = anchor.cy + dy * stride;
|
||||
float w = std::exp(dw) * stride;
|
||||
float h = std::exp(dh) * stride;
|
||||
```
|
||||
|
||||
## 配置参数
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "scrfd",
|
||||
"type": "ai_scrfd",
|
||||
"model_path": "../models/scrfd_500m_640.rknn",
|
||||
"conf_thresh": 0.5,
|
||||
"nms_thresh": 0.4,
|
||||
"max_faces": 50,
|
||||
"output_landmarks": true,
|
||||
"input_format": "rgb"
|
||||
}
|
||||
```
|
||||
|
||||
## 注意事项
|
||||
|
||||
1. **模型版本**: 使用 SCRFD 500M v1.4.1b16 (静态形状)
|
||||
- 新版本 v2.3.2 使用动态形状，C API 不支持
|
||||
|
||||
2. **输入格式**:
|
||||
- 模型训练使用 BGR，但代码默认使用 RGB
|
||||
- 通过 `input_format` 参数控制
|
||||
|
||||
3. **性能**:
|
||||
- 16800 个 anchor 需要遍历，后处理有一定开销
|
||||
- 可调整 `conf_thresh` 提前过滤低置信度候选框
|
||||
|
||||
## 测试结果
|
||||
|
||||
```
|
||||
[ai_scrfd] Frame 1: detected 50 faces, best score=1.000000
|
||||
[ai_scrfd] Frame 2: detected 50 faces, best score=1.000000
|
||||
...
|
||||
```
|
||||
|
||||
- 推理正常，无崩溃
|
||||
- 检测到多个人脸（繁忙街道场景）
|
||||
- 输出包含 bbox 和 5 个关键点
|
||||
1244
docs/design/FaceRecognition_DistanceBased_Design_v2.md
Normal file
1244
docs/design/FaceRecognition_DistanceBased_Design_v2.md
Normal file
File diff suppressed because it is too large
Load Diff
2321
docs/design/RK3588_FaceRecognition_Technical_Spec.md
Normal file
2321
docs/design/RK3588_FaceRecognition_Technical_Spec.md
Normal file
File diff suppressed because it is too large
Load Diff
@ -1,5 +1,8 @@
|
||||
# 命令指南
|
||||
|
||||
- 运行后台服务
|
||||
./build/media-server -c configs/test_face_det_zoned_only.json
|
||||
|
||||
[在windows上安装ffmpeg]
|
||||
|
||||
- 打开 PowerShell 或 CMD，执行：
|
||||
@ -10,6 +13,10 @@ ffmpeg -version
|
||||
|
||||
-查看本地摄像头信息
|
||||
ffmpeg -list_devices true -f dshow -i dummy
|
||||
查看分辨率
|
||||
ffmpeg -f dshow -list_options true -i video="你的摄像头名称"
|
||||
|
||||
ffmpeg -f dshow -list_options true -i video="HD Webcam eMeet C960"
|
||||
|
||||
- 本地运行RTSP服务器
|
||||
mediamtx.exe
|
||||
@ -17,6 +24,12 @@ mediamtx.exe
|
||||
- 推流到RTSP服务器（设置摄像头的分辨率为720P）
|
||||
ffmpeg -f dshow -rtbufsize 100M -video_size 1280x720 -framerate 30 -vcodec mjpeg -i video="4K AutoFocus Webcam" -c:v libx264 -preset ultrafast -pix_fmt yuv420p -f rtsp rtsp://localhost:8554/cam
|
||||
|
||||
ffmpeg -f dshow -rtbufsize 100M -video_size 1920x1080 -framerate 30 -vcodec mjpeg -i video="HD Webcam eMeet C960" -c:v libx264 -preset ultrafast -pix_fmt yuv420p -f rtsp rtsp://localhost:8554/cam
|
||||
|
||||
ffmpeg -re -stream_loop -1 -i 行人.flv -c copy -rtsp_transport tcp -f rtsp rtsp://10.0.0.49:8554/cam
|
||||
|
||||
ffmpeg -re -stream_loop -1 -i 监控.mp4 -c copy -rtsp_transport tcp -f rtsp rtsp://10.0.0.49:8554/cam
|
||||
|
||||
ffmpeg -stream_loop -1 -re -i "boots.mp4" -c:v libx264 -preset fast -tune zerolatency -r 30 -f rtsp -rtsp_transport tcp rtsp://localhost:8554/cam
|
||||
|
||||
- 本地验证RTSP拉流正确
|
||||
@ -67,4 +80,8 @@ pip install -e . rknn-toolkit2
|
||||
pip install "onnx==1.16.1"
|
||||
|
||||
进入模型目录，执行：
|
||||
yolo export model=best.pt format=rknn name=rk3588
|
||||
yolo export model=best.pt format=rknn name=rk3588
|
||||
|
||||
|
||||
- 插件可以通过以下方式构建（以ai_face_det_zoned为例）：
|
||||
cmake --build build --target ai_face_det_zoned
|
||||
195
docs/scrfd_500m_640_spec.md
Normal file
195
docs/scrfd_500m_640_spec.md
Normal file
@ -0,0 +1,195 @@
|
||||
# SCRFD 500M 640 模型完整规格
|
||||
|
||||
## 基本信息
|
||||
|
||||
| 属性 | 值 |
|
||||
|------|-----|
|
||||
| 模型名称 | SCRFD 500M 640 |
|
||||
| 版本 | 1.4.1b16-dad86923 |
|
||||
| 编译版本 | 1.4.1b13 (a06f28157@2022-11-26T05:23:21) |
|
||||
| 目标平台 | RK3588 (RKNPU v2) |
|
||||
| 框架 | ONNX |
|
||||
| 输入尺寸 | 640x640x3 |
|
||||
| 输入格式 | NHWC (RGB/BGR) |
|
||||
|
||||
## 输入规格
|
||||
|
||||
```python
|
||||
shape: (1, 640, 640, 3) # NHWC
|
||||
format: UINT8 (pass_through=0, RKNN内部转INT8)
|
||||
type: static_shape (非动态)
|
||||
```
|
||||
|
||||
## 输出规格 (9个张量)
|
||||
|
||||
### 1. Scores (3个尺度)
|
||||
| 索引 | 名称 | 形状 | 说明 |
|
||||
|------|------|------|------|
|
||||
| 0 | score_8 | (1, 12800, 1, 1) | stride=8, 80x80x2 anchors |
|
||||
| 1 | score_16 | (1, 3200, 1, 1) | stride=16, 40x40x2 anchors |
|
||||
| 2 | score_32 | (1, 800, 1, 1) | stride=32, 20x20x2 anchors |
|
||||
|
||||
**关键发现**: 每个anchor只输出1个前景分数，不是2个通道！
|
||||
|
||||
### 2. Bounding Boxes (3个尺度)
|
||||
| 索引 | 名称 | 形状 | 说明 |
|
||||
|------|------|------|------|
|
||||
| 3 | bbox_8 | (1, 12800, 4, 1) | [dx, dy, dw, dh] x 12800 |
|
||||
| 4 | bbox_16 | (1, 3200, 4, 1) | [dx, dy, dw, dh] x 3200 |
|
||||
| 5 | bbox_32 | (1, 800, 4, 1) | [dx, dy, dw, dh] x 800 |
|
||||
|
||||
**解码公式**:
|
||||
```
|
||||
cx = anchor_cx + dx * stride
|
||||
cy = anchor_cy + dy * stride
|
||||
w = exp(dw) * stride
|
||||
h = exp(dh) * stride
|
||||
x1 = cx - w/2
|
||||
y1 = cy - h/2
|
||||
x2 = cx + w/2
|
||||
y2 = cy + h/2
|
||||
```
|
||||
|
||||
### 3. Keypoints (3个尺度)
|
||||
| 索引 | 名称 | 形状 | 说明 |
|
||||
|------|------|------|------|
|
||||
| 6 | kps_8 | (1, 12800, 10, 1) | 5点x2坐标 x 12800 |
|
||||
| 7 | kps_16 | (1, 3200, 10, 1) | 5点x2坐标 x 3200 |
|
||||
| 8 | kps_32 | (1, 800, 10, 1) | 5点x2坐标 x 800 |
|
||||
|
||||
**解码公式**:
|
||||
```
|
||||
kps_x = anchor_cx + kps_dx * stride
|
||||
kps_y = anchor_cy + kps_dy * stride
|
||||
```
|
||||
|
||||
## 锚点 (Anchors) 生成
|
||||
|
||||
```python
|
||||
strides = [8, 16, 32]
|
||||
num_anchors_per_location = 2
|
||||
|
||||
total_anchors = 80*80*2 + 40*40*2 + 20*20*2 = 16800
|
||||
|
||||
# anchor生成逻辑
|
||||
for stride in [8, 16, 32]:
|
||||
grid_size = 640 // stride
|
||||
for y in range(grid_size):
|
||||
for x in range(grid_size):
|
||||
for a in range(2): # 2 anchors per location
|
||||
anchor_cx = (x + 0.5) * stride
|
||||
anchor_cy = (y + 0.5) * stride
|
||||
```
|
||||
|
||||
**Anchor分布**:
|
||||
- stride=8: 80x80 grid, 12800 anchors
|
||||
- stride=16: 40x40 grid, 3200 anchors
|
||||
- stride=32: 20x20 grid, 800 anchors
|
||||
|
||||
## 量化参数
|
||||
|
||||
| 张量 | 类型 | Zero Point | Scale |
|
||||
|------|------|------------|-------|
|
||||
| input | INT8 | -128 | 0.003921 |
|
||||
| score_8 | INT8 | -128 | 0.003534 |
|
||||
| score_16 | INT8 | -128 | 0.001638 |
|
||||
| score_32 | INT8 | -128 | 0.000163 |
|
||||
| bbox_8 | INT8 | -128 | 0.016671 |
|
||||
| bbox_16 | INT8 | -128 | 0.016227 |
|
||||
| bbox_32 | INT8 | -128 | 0.017979 |
|
||||
| kps_8 | INT8 | -7 | 0.020261 |
|
||||
| kps_16 | INT8 | -20 | 0.014725 |
|
||||
| kps_32 | INT8 | -11 | 0.014612 |
|
||||
|
||||
**注意**: 实际推理时使用 `want_float=1`，RKNN 自动反量化，输出已是 FP32。
|
||||
|
||||
## 后处理流程
|
||||
|
||||
1. **Score筛选**: 对每个anchor，检查 `score > conf_thresh`
|
||||
2. **BBox解码**: anchor中心 + 相对偏移 + exp解码宽高
|
||||
3. **Kps解码**: 5个面部关键点，相对anchor中心
|
||||
4. **坐标映射**: 从640x640映射回原图分辨率
|
||||
5. **NMS**: IoU阈值去重
|
||||
|
||||
## 配置参数
|
||||
|
||||
```json
|
||||
{
|
||||
"model_path": "./models/scrfd_500m_640.rknn",
|
||||
"conf_thresh": 0.5, // 置信度阈值 (0-1)
|
||||
"nms_thresh": 0.4, // NMS IoU阈值
|
||||
"max_faces": 10, // 最大检测人脸数
|
||||
"output_landmarks": true, // 输出5个关键点
|
||||
"input_format": "rgb" // 输入格式: rgb/bgr
|
||||
}
|
||||
```
|
||||
|
||||
## 性能指标
|
||||
|
||||
| 指标 | 值 |
|
||||
|------|-----|
|
||||
| 模型大小 | ~1.5 MB |
|
||||
| 输入分辨率 | 640x640 |
|
||||
| Anchor数量 | 16800 |
|
||||
| 输出张量数 | 9 |
|
||||
| 关键点 | 5点 (双眼、鼻尖、嘴角) |
|
||||
| 总参数量 | ~500K |
|
||||
|
||||
## 版本兼容性
|
||||
|
||||
| 版本 | 状态 | 说明 |
|
||||
|------|------|------|
|
||||
| 1.4.1b16 | ✅ 推荐 | 静态形状，C API完美支持 |
|
||||
| 2.3.2 | ❌ 不兼容 | 动态形状，C API崩溃 |
|
||||
|
||||
**运行时库要求**:
|
||||
- 最低: librknnrt 1.5.2
|
||||
- 推荐: librknnrt 2.3.2
|
||||
|
||||
## 调试命令
|
||||
|
||||
```bash
|
||||
# 检查模型版本
|
||||
strings models/scrfd_500m_640.rknn | grep "compiler version"
|
||||
|
||||
# 检查运行时版本
|
||||
strings /usr/lib/librknnrt.so | grep "librknnrt version"
|
||||
|
||||
# Python验证模型
|
||||
python3 << 'PYEOF'
|
||||
from rknnlite.api import RKNNLite
|
||||
import numpy as np
|
||||
|
||||
rknn = RKNNLite()
|
||||
rknn.load_rknn("models/scrfd_500m_640.rknn")
|
||||
rknn.init_runtime()
|
||||
|
||||
img = np.zeros((1, 640, 640, 3), dtype=np.uint8)
|
||||
outputs = rknn.inference(inputs=[img])
|
||||
|
||||
for i, out in enumerate(outputs):
|
||||
print(f"output[{i}]: shape={out.shape}")
|
||||
|
||||
rknn.release()
|
||||
PYEOF
|
||||
```
|
||||
|
||||
## 常见问题
|
||||
|
||||
### 1. 坐标错误
|
||||
**原因**: Score张量误解为2通道，实际只有1通道
|
||||
**修复**: `score = scores[i]` 而不是 `scores[i*2+1]`
|
||||
|
||||
### 2. 模型崩溃
|
||||
**原因**: 使用动态形状版本 (v2.3.2)
|
||||
**修复**: 切换到静态形状版本 (v1.4.1b16)
|
||||
|
||||
### 3. RGA任务过多
|
||||
**原因**: 检测到太多人脸，OSD绘制任务堆积
|
||||
**修复**: 降低 `max_faces` 或提高 `conf_thresh`
|
||||
|
||||
## 参考
|
||||
|
||||
- [SCRFD Paper](https://arxiv.org/abs/2105.04714)
|
||||
- [InsightFace SCRFD](https://github.com/deepinsight/insightface/tree/master/detection/scrfd)
|
||||
- [RKNN API文档](https://github.com/rockchip-linux/rknpu2)
|
||||
684
include/face/face_detection_utils.h
Normal file
684
include/face/face_detection_utils.h
Normal file
@ -0,0 +1,684 @@
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* 人脸检测公共工具函数
|
||||
* 供 ai_face_det 和 ai_face_det_zoned 节点复用
|
||||
*/
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <memory>
|
||||
#include <numeric>
|
||||
#include <vector>
|
||||
|
||||
#include "face/face_result.h"
|
||||
|
||||
// RKNN类型前向声明（避免直接依赖rknn_api.h）
|
||||
#if defined(RK3588_ENABLE_RKNN)
|
||||
#include "rknn_api.h"
|
||||
#else
|
||||
typedef enum _rknn_tensor_type {
|
||||
RKNN_TENSOR_UINT8 = 0,
|
||||
RKNN_TENSOR_INT8,
|
||||
RKNN_TENSOR_FLOAT16,
|
||||
RKNN_TENSOR_FLOAT32,
|
||||
} rknn_tensor_type;
|
||||
#endif
|
||||
|
||||
namespace rk3588 {
|
||||
namespace face_detection {
|
||||
|
||||
// ============================================================================
|
||||
// 基础工具函数
|
||||
// ============================================================================
|
||||
|
||||
inline int ClampInt(int v, int lo, int hi) {
|
||||
return v < lo ? lo : (v > hi ? hi : v);
|
||||
}
|
||||
|
||||
inline float Sigmoid(float x) {
|
||||
return 1.0f / (1.0f + std::exp(-x));
|
||||
}
|
||||
|
||||
inline float Softmax2(float a, float b) {
|
||||
const float m = std::max(a, b);
|
||||
const float ea = std::exp(a - m);
|
||||
const float eb = std::exp(b - m);
|
||||
return eb / (ea + eb);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// 几何计算
|
||||
// ============================================================================
|
||||
|
||||
inline float IoU(const Rect& a, const Rect& b) {
|
||||
const float ax1 = a.x;
|
||||
const float ay1 = a.y;
|
||||
const float ax2 = a.x + a.w;
|
||||
const float ay2 = a.y + a.h;
|
||||
const float bx1 = b.x;
|
||||
const float by1 = b.y;
|
||||
const float bx2 = b.x + b.w;
|
||||
const float by2 = b.y + b.h;
|
||||
|
||||
const float ix1 = std::max(ax1, bx1);
|
||||
const float iy1 = std::max(ay1, by1);
|
||||
const float ix2 = std::min(ax2, bx2);
|
||||
const float iy2 = std::min(ay2, by2);
|
||||
|
||||
const float iw = std::max(0.0f, ix2 - ix1);
|
||||
const float ih = std::max(0.0f, iy2 - iy1);
|
||||
const float inter = iw * ih;
|
||||
const float ua = a.w * a.h + b.w * b.h - inter;
|
||||
return ua <= 0.0f ? 0.0f : (inter / ua);
|
||||
}
|
||||
|
||||
inline void NmsSorted(const std::vector<Rect>& boxes,
|
||||
const std::vector<float>& scores,
|
||||
float nms_thresh,
|
||||
std::vector<int>& keep) {
|
||||
keep.clear();
|
||||
std::vector<int> order(scores.size());
|
||||
std::iota(order.begin(), order.end(), 0);
|
||||
std::sort(order.begin(), order.end(),
|
||||
[&](int a, int b) { return scores[a] > scores[b]; });
|
||||
|
||||
for (int idx : order) {
|
||||
bool suppressed = false;
|
||||
for (int kept : keep) {
|
||||
if (IoU(boxes[idx], boxes[kept]) > nms_thresh) {
|
||||
suppressed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!suppressed) keep.push_back(idx);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// RetinaFace 数据结构
|
||||
// ============================================================================
|
||||
|
||||
struct Prior {
|
||||
float cx = 0.0f;
|
||||
float cy = 0.0f;
|
||||
float w = 0.0f;
|
||||
float h = 0.0f;
|
||||
};
|
||||
|
||||
struct DetectionConfig {
|
||||
float conf_thresh = 0.6f;
|
||||
float nms_thresh = 0.4f;
|
||||
int max_faces = 10;
|
||||
bool output_landmarks = true;
|
||||
|
||||
// RetinaFace默认参数
|
||||
std::vector<int> steps{8, 16, 32};
|
||||
std::vector<std::vector<int>> min_sizes{{16, 32}, {64, 128}, {256, 512}};
|
||||
float variance0 = 0.1f;
|
||||
float variance1 = 0.2f;
|
||||
};
|
||||
|
||||
// 张量结构（与RKNN解耦）
|
||||
struct TensorView {
|
||||
const uint8_t* data = nullptr;
|
||||
size_t size = 0;
|
||||
int32_t zp = 0;
|
||||
float scale = 1.0f;
|
||||
std::vector<uint32_t> dims;
|
||||
rknn_tensor_type type = RKNN_TENSOR_UINT8;
|
||||
};
|
||||
|
||||
struct NcTensor {
|
||||
int n = 0;
|
||||
int c = 0;
|
||||
std::vector<float> data; // N*C row-major
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// 图像预处理
|
||||
// ============================================================================
|
||||
|
||||
inline void ResizeRgbBilinear(const uint8_t* src, int src_w, int src_h, int src_stride,
|
||||
uint8_t* dst, int dst_w, int dst_h, bool swap_rb) {
|
||||
const float scale_x = static_cast<float>(src_w) / static_cast<float>(dst_w);
|
||||
const float scale_y = static_cast<float>(src_h) / static_cast<float>(dst_h);
|
||||
|
||||
for (int y = 0; y < dst_h; ++y) {
|
||||
const float fy = (static_cast<float>(y) + 0.5f) * scale_y - 0.5f;
|
||||
int y0 = static_cast<int>(std::floor(fy));
|
||||
int y1 = y0 + 1;
|
||||
const float wy1 = fy - static_cast<float>(y0);
|
||||
const float wy0 = 1.0f - wy1;
|
||||
y0 = ClampInt(y0, 0, src_h - 1);
|
||||
y1 = ClampInt(y1, 0, src_h - 1);
|
||||
|
||||
const uint8_t* row0 = src + static_cast<size_t>(y0) * static_cast<size_t>(src_stride);
|
||||
const uint8_t* row1 = src + static_cast<size_t>(y1) * static_cast<size_t>(src_stride);
|
||||
uint8_t* out = dst + static_cast<size_t>(y) * static_cast<size_t>(dst_w) * 3;
|
||||
|
||||
for (int x = 0; x < dst_w; ++x) {
|
||||
const float fx = (static_cast<float>(x) + 0.5f) * scale_x - 0.5f;
|
||||
int x0 = static_cast<int>(std::floor(fx));
|
||||
int x1 = x0 + 1;
|
||||
const float wx1 = fx - static_cast<float>(x0);
|
||||
const float wx0 = 1.0f - wx1;
|
||||
x0 = ClampInt(x0, 0, src_w - 1);
|
||||
x1 = ClampInt(x1, 0, src_w - 1);
|
||||
|
||||
const uint8_t* p00 = row0 + x0 * 3;
|
||||
const uint8_t* p01 = row0 + x1 * 3;
|
||||
const uint8_t* p10 = row1 + x0 * 3;
|
||||
const uint8_t* p11 = row1 + x1 * 3;
|
||||
|
||||
for (int c = 0; c < 3; ++c) {
|
||||
const float v =
|
||||
(static_cast<float>(p00[c]) * wx0 + static_cast<float>(p01[c]) * wx1) * wy0 +
|
||||
(static_cast<float>(p10[c]) * wx0 + static_cast<float>(p11[c]) * wx1) * wy1;
|
||||
out[c] = static_cast<uint8_t>(ClampInt(static_cast<int>(v + 0.5f), 0, 255));
|
||||
}
|
||||
|
||||
if (swap_rb) {
|
||||
std::swap(out[0], out[2]);
|
||||
}
|
||||
out += 3;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// 张量解析（从RKNN输出提取）
|
||||
// ============================================================================
|
||||
|
||||
inline float HalfToFloat(uint16_t h) {
|
||||
const uint32_t sign = (static_cast<uint32_t>(h & 0x8000u)) << 16;
|
||||
uint32_t exp = (h & 0x7C00u) >> 10;
|
||||
uint32_t mant = (h & 0x03FFu);
|
||||
|
||||
uint32_t f = 0;
|
||||
if (exp == 0) {
|
||||
if (mant == 0) {
|
||||
f = sign;
|
||||
} else {
|
||||
exp = 1;
|
||||
while ((mant & 0x0400u) == 0) {
|
||||
mant <<= 1;
|
||||
--exp;
|
||||
}
|
||||
mant &= 0x03FFu;
|
||||
exp = exp + (127 - 15);
|
||||
f = sign | (exp << 23) | (mant << 13);
|
||||
}
|
||||
} else if (exp == 31) {
|
||||
f = sign | 0x7F800000u | (mant << 13);
|
||||
} else {
|
||||
exp = exp + (127 - 15);
|
||||
f = sign | (exp << 23) | (mant << 13);
|
||||
}
|
||||
|
||||
float out;
|
||||
memcpy(&out, &f, sizeof(out));
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline float Dequant(T q, int32_t zp, float scale) {
|
||||
return (static_cast<float>(q) - static_cast<float>(zp)) * scale;
|
||||
}
|
||||
|
||||
// 从TensorView提取NcTensor
|
||||
inline bool ExtractNcTensor(const TensorView& t, int c, NcTensor& out) {
|
||||
out = {};
|
||||
out.c = c;
|
||||
if (!t.data || t.size == 0) return false;
|
||||
|
||||
size_t elem_size = 1;
|
||||
bool is_float32 = false;
|
||||
bool is_float16 = false;
|
||||
|
||||
if (t.type == RKNN_TENSOR_FLOAT16) {
|
||||
elem_size = 2;
|
||||
is_float16 = true;
|
||||
} else if (t.type == RKNN_TENSOR_FLOAT32) {
|
||||
elem_size = 4;
|
||||
is_float32 = true;
|
||||
}
|
||||
|
||||
const size_t elem_cnt = elem_size > 0 ? (t.size / elem_size) : 0;
|
||||
if (elem_cnt == 0) return false;
|
||||
|
||||
int n = 0;
|
||||
bool transposed = false;
|
||||
if (t.dims.size() == 3) {
|
||||
const uint32_t d1 = t.dims[1];
|
||||
const uint32_t d2 = t.dims[2];
|
||||
if (static_cast<int>(d2) == c) {
|
||||
n = static_cast<int>(d1);
|
||||
transposed = false;
|
||||
} else if (static_cast<int>(d1) == c) {
|
||||
n = static_cast<int>(d2);
|
||||
transposed = true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else if (t.dims.size() == 2) {
|
||||
const uint32_t d0 = t.dims[0];
|
||||
const uint32_t d1 = t.dims[1];
|
||||
if (static_cast<int>(d1) == c) {
|
||||
n = static_cast<int>(d0);
|
||||
transposed = false;
|
||||
} else if (static_cast<int>(d0) == c) {
|
||||
n = static_cast<int>(d1);
|
||||
transposed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (n <= 0) {
|
||||
if (elem_cnt % static_cast<size_t>(c) != 0) return false;
|
||||
n = static_cast<int>(elem_cnt / static_cast<size_t>(c));
|
||||
transposed = false;
|
||||
}
|
||||
|
||||
if (static_cast<size_t>(n) * static_cast<size_t>(c) != elem_cnt) {
|
||||
return false;
|
||||
}
|
||||
|
||||
out.n = n;
|
||||
out.data.resize(static_cast<size_t>(n) * static_cast<size_t>(c));
|
||||
|
||||
auto ReadElem = [&](size_t idx) -> float {
|
||||
if (is_float32) {
|
||||
const float* fp = reinterpret_cast<const float*>(t.data);
|
||||
return fp[idx];
|
||||
}
|
||||
if (is_float16) {
|
||||
const uint16_t* hp = reinterpret_cast<const uint16_t*>(t.data);
|
||||
return HalfToFloat(hp[idx]);
|
||||
}
|
||||
if (t.type == RKNN_TENSOR_INT8) {
|
||||
const int8_t* p = reinterpret_cast<const int8_t*>(t.data);
|
||||
return Dequant(p[idx], t.zp, t.scale);
|
||||
}
|
||||
const uint8_t* p = reinterpret_cast<const uint8_t*>(t.data);
|
||||
return Dequant(p[idx], t.zp, t.scale);
|
||||
};
|
||||
|
||||
if (!transposed) {
|
||||
for (size_t i = 0; i < out.data.size(); ++i) {
|
||||
out.data[i] = ReadElem(i);
|
||||
}
|
||||
} else {
|
||||
for (int ci = 0; ci < c; ++ci) {
|
||||
for (int ni = 0; ni < n; ++ni) {
|
||||
const size_t src_idx = static_cast<size_t>(ci) * static_cast<size_t>(n) + static_cast<size_t>(ni);
|
||||
const size_t dst_idx = static_cast<size_t>(ni) * static_cast<size_t>(c) + static_cast<size_t>(ci);
|
||||
out.data[dst_idx] = ReadElem(src_idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// RetinaFace 核心：先验框生成
|
||||
// ============================================================================
|
||||
|
||||
inline std::vector<Prior> GeneratePriors(int in_w, int in_h,
|
||||
const std::vector<int>& steps,
|
||||
const std::vector<std::vector<int>>& min_sizes) {
|
||||
std::vector<Prior> priors;
|
||||
if (steps.empty() || steps.size() != min_sizes.size()) return priors;
|
||||
priors.reserve(5000);
|
||||
|
||||
for (size_t s = 0; s < steps.size(); ++s) {
|
||||
const int step = steps[s];
|
||||
const int fm_w = in_w / step;
|
||||
const int fm_h = in_h / step;
|
||||
for (int i = 0; i < fm_h; ++i) {
|
||||
for (int j = 0; j < fm_w; ++j) {
|
||||
for (int ms : min_sizes[s]) {
|
||||
const float s_kx = static_cast<float>(ms) / static_cast<float>(in_w);
|
||||
const float s_ky = static_cast<float>(ms) / static_cast<float>(in_h);
|
||||
const float cx = (static_cast<float>(j) + 0.5f) * static_cast<float>(step) /
|
||||
static_cast<float>(in_w);
|
||||
const float cy = (static_cast<float>(i) + 0.5f) * static_cast<float>(step) /
|
||||
static_cast<float>(in_h);
|
||||
priors.push_back(Prior{cx, cy, s_kx, s_ky});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return priors;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// RetinaFace 核心：检测结果解码
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* 解码RetinaFace检测结果
|
||||
*
|
||||
* @param loc_tensor 位置回归张量 [N, 4]
|
||||
* @param conf_tensor 置信度张量 [N, 2]
|
||||
* @param landm_tensor 关键点张量 [N, 10] (可选，可以为空)
|
||||
* @param priors 先验框
|
||||
* @param src_w 原始图像宽度
|
||||
* @param src_h 原始图像高度
|
||||
* @param model_w 模型输入宽度
|
||||
* @param model_h 模型输入高度
|
||||
* @param cfg 检测配置
|
||||
* @param out 输出结果
|
||||
*/
|
||||
inline void DecodeRetinaFace(const NcTensor& loc_tensor,
|
||||
const NcTensor& conf_tensor,
|
||||
const NcTensor& landm_tensor,
|
||||
const std::vector<Prior>& priors,
|
||||
int src_w, int src_h,
|
||||
int model_w, int model_h,
|
||||
const DetectionConfig& cfg,
|
||||
FaceDetResult& out) {
|
||||
if (loc_tensor.n <= 0 || conf_tensor.n != loc_tensor.n) return;
|
||||
|
||||
const int n = loc_tensor.n;
|
||||
const bool has_landmarks = cfg.output_landmarks && !landm_tensor.data.empty() && landm_tensor.n == n;
|
||||
|
||||
if (!priors.empty() && static_cast<int>(priors.size()) != n) {
|
||||
return; // prior mismatch
|
||||
}
|
||||
|
||||
const float sx = static_cast<float>(src_w) / static_cast<float>(model_w);
|
||||
const float sy = static_cast<float>(src_h) / static_cast<float>(model_h);
|
||||
|
||||
std::vector<Rect> boxes;
|
||||
std::vector<float> scores;
|
||||
std::vector<std::array<Point2f, 5>> lmks;
|
||||
boxes.reserve(static_cast<size_t>(n));
|
||||
scores.reserve(static_cast<size_t>(n));
|
||||
if (has_landmarks) lmks.reserve(static_cast<size_t>(n));
|
||||
|
||||
const float var0 = cfg.variance0;
|
||||
const float var1 = cfg.variance1;
|
||||
|
||||
for (int i = 0; i < n; ++i) {
|
||||
// 解析置信度
|
||||
const float s0 = conf_tensor.data[static_cast<size_t>(i) * 2 + 0];
|
||||
const float s1 = conf_tensor.data[static_cast<size_t>(i) * 2 + 1];
|
||||
float score;
|
||||
if (s0 >= 0.0f && s0 <= 1.0f && s1 >= 0.0f && s1 <= 1.0f && std::fabs((s0 + s1) - 1.0f) < 0.1f) {
|
||||
score = s1;
|
||||
} else {
|
||||
score = Softmax2(s0, s1);
|
||||
}
|
||||
if (score < cfg.conf_thresh) continue;
|
||||
|
||||
// 解析位置
|
||||
const Prior p = priors.empty() ? Prior{0, 0, 0, 0} : priors[static_cast<size_t>(i)];
|
||||
const float dx = loc_tensor.data[static_cast<size_t>(i) * 4 + 0];
|
||||
const float dy = loc_tensor.data[static_cast<size_t>(i) * 4 + 1];
|
||||
const float dw = loc_tensor.data[static_cast<size_t>(i) * 4 + 2];
|
||||
const float dh = loc_tensor.data[static_cast<size_t>(i) * 4 + 3];
|
||||
|
||||
const float cx = p.cx + dx * var0 * p.w;
|
||||
const float cy = p.cy + dy * var0 * p.h;
|
||||
const float ww = p.w * std::exp(dw * var1);
|
||||
const float hh = p.h * std::exp(dh * var1);
|
||||
|
||||
float x1 = (cx - ww * 0.5f) * static_cast<float>(model_w);
|
||||
float y1 = (cy - hh * 0.5f) * static_cast<float>(model_h);
|
||||
float x2 = (cx + ww * 0.5f) * static_cast<float>(model_w);
|
||||
float y2 = (cy + hh * 0.5f) * static_cast<float>(model_h);
|
||||
|
||||
// 映射到原始图像
|
||||
x1 *= sx;
|
||||
x2 *= sx;
|
||||
y1 *= sy;
|
||||
y2 *= sy;
|
||||
|
||||
Rect bb;
|
||||
bb.x = static_cast<float>(ClampInt(static_cast<int>(x1), 0, src_w - 1));
|
||||
bb.y = static_cast<float>(ClampInt(static_cast<int>(y1), 0, src_h - 1));
|
||||
const float rx2 = static_cast<float>(ClampInt(static_cast<int>(x2), 0, src_w - 1));
|
||||
const float ry2 = static_cast<float>(ClampInt(static_cast<int>(y2), 0, src_h - 1));
|
||||
bb.w = std::max(0.0f, rx2 - bb.x);
|
||||
bb.h = std::max(0.0f, ry2 - bb.y);
|
||||
if (bb.w <= 1.0f || bb.h <= 1.0f) continue;
|
||||
|
||||
boxes.push_back(bb);
|
||||
scores.push_back(score);
|
||||
|
||||
// 解析关键点
|
||||
if (has_landmarks) {
|
||||
std::array<Point2f, 5> pts{};
|
||||
for (int k = 0; k < 5; ++k) {
|
||||
const float lx = landm_tensor.data[static_cast<size_t>(i) * 10 + k * 2 + 0];
|
||||
const float ly = landm_tensor.data[static_cast<size_t>(i) * 10 + k * 2 + 1];
|
||||
const float px = (p.cx + lx * var0 * p.w) * static_cast<float>(model_w) * sx;
|
||||
const float py = (p.cy + ly * var0 * p.h) * static_cast<float>(model_h) * sy;
|
||||
pts[k].x = static_cast<float>(ClampInt(static_cast<int>(px), 0, src_w - 1));
|
||||
pts[k].y = static_cast<float>(ClampInt(static_cast<int>(py), 0, src_h - 1));
|
||||
}
|
||||
lmks.push_back(pts);
|
||||
}
|
||||
}
|
||||
|
||||
if (boxes.empty()) return;
|
||||
|
||||
// NMS
|
||||
std::vector<int> keep;
|
||||
NmsSorted(boxes, scores, cfg.nms_thresh, keep);
|
||||
if (keep.empty()) return;
|
||||
|
||||
// 构建输出
|
||||
const int out_n = std::min<int>(cfg.max_faces, static_cast<int>(keep.size()));
|
||||
out.faces.reserve(static_cast<size_t>(out_n));
|
||||
for (int i = 0; i < out_n; ++i) {
|
||||
const int k = keep[static_cast<size_t>(i)];
|
||||
FaceDetItem item;
|
||||
item.bbox = boxes[static_cast<size_t>(k)];
|
||||
item.score = scores[static_cast<size_t>(k)];
|
||||
item.track_id = -1;
|
||||
if (has_landmarks && k < static_cast<int>(lmks.size())) {
|
||||
item.has_landmarks = true;
|
||||
item.landmarks = lmks[static_cast<size_t>(k)];
|
||||
}
|
||||
out.faces.push_back(std::move(item));
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// SCRFD 核心：检测结果解码
|
||||
// SCRFD输出格式：9个张量 (3个尺度 × 3种类型：score, bbox, kps)
|
||||
// ============================================================================
|
||||
|
||||
struct ScrfdAnchor {
|
||||
float cx = 0.0f;
|
||||
float cy = 0.0f;
|
||||
int stride = 0;
|
||||
};
|
||||
|
||||
inline std::vector<ScrfdAnchor> GenerateScrfdAnchors(int in_w, int in_h,
|
||||
const std::vector<int>& strides) {
|
||||
std::vector<ScrfdAnchor> anchors;
|
||||
anchors.reserve(20000);
|
||||
|
||||
for (int stride : strides) {
|
||||
int fm_w = in_w / stride;
|
||||
int fm_h = in_h / stride;
|
||||
for (int y = 0; y < fm_h; ++y) {
|
||||
for (int x = 0; x < fm_w; ++x) {
|
||||
// SCRFD使用2个anchor per location
|
||||
for (int a = 0; a < 2; ++a) {
|
||||
anchors.push_back(ScrfdAnchor{
|
||||
(x + 0.5f) * stride,
|
||||
(y + 0.5f) * stride,
|
||||
stride
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return anchors;
|
||||
}
|
||||
|
||||
// 从NCHW格式提取值
|
||||
inline float ExtractNCHW(const TensorView& t, int c, int h, int w, int C, int H, int W) {
|
||||
if (c < 0 || c >= C || h < 0 || h >= H || w < 0 || w >= W) return 0.0f;
|
||||
size_t idx = (static_cast<size_t>(c) * H + h) * W + w;
|
||||
|
||||
if (t.type == RKNN_TENSOR_FLOAT32) {
|
||||
const float* p = reinterpret_cast<const float*>(t.data);
|
||||
return p[idx];
|
||||
} else if (t.type == RKNN_TENSOR_INT8) {
|
||||
const int8_t* p = reinterpret_cast<const int8_t*>(t.data);
|
||||
return Dequant(p[idx], t.zp, t.scale);
|
||||
} else {
|
||||
const uint8_t* p = reinterpret_cast<const uint8_t*>(t.data);
|
||||
return Dequant(p[idx], t.zp, t.scale);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 解码SCRFD检测结果
|
||||
*
|
||||
* @param outputs 9个输出张量 [score_8, score_16, score_32, bbox_8, bbox_16, bbox_32, kps_8, kps_16, kps_32]
|
||||
* @param anchors 预生成的anchor
|
||||
* @param src_w 原始图像宽度
|
||||
* @param src_h 原始图像高度
|
||||
* @param model_w 模型输入宽度
|
||||
* @param model_h 模型输入高度
|
||||
* @param cfg 检测配置
|
||||
* @param out 输出结果
|
||||
*/
|
||||
inline void DecodeScrfd(const std::vector<TensorView>& outputs,
|
||||
const std::vector<ScrfdAnchor>& anchors,
|
||||
int src_w, int src_h,
|
||||
int model_w, int model_h,
|
||||
const DetectionConfig& cfg,
|
||||
FaceDetResult& out) {
|
||||
if (outputs.size() != 9) {
|
||||
return; // SCRFD需要9个输出
|
||||
}
|
||||
|
||||
const float sx = static_cast<float>(src_w) / static_cast<float>(model_w);
|
||||
const float sy = static_cast<float>(src_h) / static_cast<float>(model_h);
|
||||
|
||||
std::vector<Rect> boxes;
|
||||
std::vector<float> scores;
|
||||
std::vector<std::array<Point2f, 5>> lmks;
|
||||
|
||||
size_t anchor_idx = 0;
|
||||
const int strides[3] = {8, 16, 32};
|
||||
|
||||
for (int s = 0; s < 3; ++s) {
|
||||
int score_idx = s;
|
||||
int bbox_idx = s + 3;
|
||||
int kps_idx = s + 6;
|
||||
int stride = strides[s];
|
||||
|
||||
const TensorView& score_t = outputs[score_idx];
|
||||
const TensorView& bbox_t = outputs[bbox_idx];
|
||||
const TensorView& kps_t = outputs[kps_idx];
|
||||
|
||||
// 检查维度
|
||||
if (score_t.dims.size() < 4 || bbox_t.dims.size() < 4) continue;
|
||||
|
||||
int C = static_cast<int>(score_t.dims[1]);
|
||||
int H = static_cast<int>(score_t.dims[2]);
|
||||
int W = static_cast<int>(score_t.dims[3]);
|
||||
int anchors_per_loc = C / 2; // fg/bg
|
||||
|
||||
for (int h = 0; h < H; ++h) {
|
||||
for (int w = 0; w < W; ++w) {
|
||||
for (int a = 0; a < anchors_per_loc; ++a) {
|
||||
if (anchor_idx >= anchors.size()) break;
|
||||
|
||||
// 提取前景分数 (channel a*2+1)
|
||||
float score = ExtractNCHW(score_t, a * 2 + 1, h, w, C, H, W);
|
||||
|
||||
if (score >= cfg.conf_thresh) {
|
||||
const ScrfdAnchor& anchor = anchors[anchor_idx];
|
||||
|
||||
// 提取bbox [dx, dy, dw, dh]
|
||||
float dx = ExtractNCHW(bbox_t, a * 4 + 0, h, w,
|
||||
static_cast<int>(bbox_t.dims[1]), H, W) * stride;
|
||||
float dy = ExtractNCHW(bbox_t, a * 4 + 1, h, w,
|
||||
static_cast<int>(bbox_t.dims[1]), H, W) * stride;
|
||||
float dw = ExtractNCHW(bbox_t, a * 4 + 2, h, w,
|
||||
static_cast<int>(bbox_t.dims[1]), H, W) * stride;
|
||||
float dh = ExtractNCHW(bbox_t, a * 4 + 3, h, w,
|
||||
static_cast<int>(bbox_t.dims[1]), H, W) * stride;
|
||||
|
||||
float cx = anchor.cx + dx;
|
||||
float cy = anchor.cy + dy;
|
||||
float x1 = (cx - dw * 0.5f) * sx;
|
||||
float y1 = (cy - dh * 0.5f) * sy;
|
||||
float x2 = (cx + dw * 0.5f) * sx;
|
||||
float y2 = (cy + dh * 0.5f) * sy;
|
||||
|
||||
x1 = static_cast<float>(ClampInt(static_cast<int>(x1), 0, src_w - 1));
|
||||
y1 = static_cast<float>(ClampInt(static_cast<int>(y1), 0, src_h - 1));
|
||||
x2 = static_cast<float>(ClampInt(static_cast<int>(x2), 0, src_w - 1));
|
||||
y2 = static_cast<float>(ClampInt(static_cast<int>(y2), 0, src_h - 1));
|
||||
|
||||
Rect bb;
|
||||
bb.x = x1;
|
||||
bb.y = y1;
|
||||
bb.w = std::max(0.0f, x2 - x1);
|
||||
bb.h = std::max(0.0f, y2 - y1);
|
||||
|
||||
if (bb.w > 1.0f && bb.h > 1.0f) {
|
||||
boxes.push_back(bb);
|
||||
scores.push_back(score);
|
||||
|
||||
// 提取关键点
|
||||
if (cfg.output_landmarks) {
|
||||
std::array<Point2f, 5> pts{};
|
||||
for (int k = 0; k < 5; ++k) {
|
||||
float lx = ExtractNCHW(kps_t, a * 10 + k * 2 + 0, h, w,
|
||||
static_cast<int>(kps_t.dims[1]), H, W) * stride;
|
||||
float ly = ExtractNCHW(kps_t, a * 10 + k * 2 + 1, h, w,
|
||||
static_cast<int>(kps_t.dims[1]), H, W) * stride;
|
||||
pts[k].x = (anchor.cx + lx) * sx;
|
||||
pts[k].y = (anchor.cy + ly) * sy;
|
||||
}
|
||||
lmks.push_back(pts);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
++anchor_idx;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (boxes.empty()) return;
|
||||
|
||||
// NMS
|
||||
std::vector<int> keep;
|
||||
NmsSorted(boxes, scores, cfg.nms_thresh, keep);
|
||||
if (keep.empty()) return;
|
||||
|
||||
// 构建输出
|
||||
const int out_n = std::min<int>(cfg.max_faces, static_cast<int>(keep.size()));
|
||||
out.faces.reserve(static_cast<size_t>(out_n));
|
||||
for (int i = 0; i < out_n; ++i) {
|
||||
const int k = keep[static_cast<size_t>(i)];
|
||||
FaceDetItem item;
|
||||
item.bbox = boxes[static_cast<size_t>(k)];
|
||||
item.score = scores[static_cast<size_t>(k)];
|
||||
item.track_id = -1;
|
||||
if (cfg.output_landmarks && k < static_cast<int>(lmks.size())) {
|
||||
item.has_landmarks = true;
|
||||
item.landmarks = lmks[static_cast<size_t>(k)];
|
||||
}
|
||||
out.faces.push_back(std::move(item));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace face_detection
|
||||
} // namespace rk3588
|
||||
@ -82,11 +82,16 @@ public:
|
||||
}
|
||||
|
||||
// Keep existing stdout/stderr logging style for easy board-side debugging.
|
||||
// Use fflush to ensure real-time log output
|
||||
if (lvl == LogLevel::Warn || lvl == LogLevel::Error) {
|
||||
std::cerr << line << "\n";
|
||||
std::cerr.flush();
|
||||
} else {
|
||||
std::cout << line << "\n";
|
||||
std::cout.flush();
|
||||
}
|
||||
fflush(stdout);
|
||||
fflush(stderr);
|
||||
}
|
||||
|
||||
std::vector<std::string> RecentLines(size_t limit) const {
|
||||
|
||||
Binary file not shown.
BIN
models/scrfd_500m_640.rknn
Normal file
BIN
models/scrfd_500m_640.rknn
Normal file
Binary file not shown.
@ -269,6 +269,42 @@ set_target_properties(ai_face_det PROPERTIES
|
||||
RUNTIME_OUTPUT_DIRECTORY ${RK_PLUGIN_OUTPUT_DIR}
|
||||
)
|
||||
|
||||
# ai_face_det_zoned plugin (RKNN-based RetinaFace with distance zone detection)
|
||||
add_library(ai_face_det_zoned SHARED
|
||||
ai_face_det_zoned/ai_face_det_zoned_node.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/utils/dma_alloc.cpp
|
||||
)
|
||||
target_include_directories(ai_face_det_zoned PRIVATE ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/third_party)
|
||||
target_link_libraries(ai_face_det_zoned PRIVATE project_options Threads::Threads ai_scheduler)
|
||||
if(RK3588_ENABLE_RKNN AND RK_RKNN_LIB)
|
||||
target_compile_definitions(ai_face_det_zoned PRIVATE RK3588_ENABLE_RKNN)
|
||||
target_include_directories(ai_face_det_zoned PRIVATE ${RKNN_RUNTIME_INCLUDE_DIR})
|
||||
target_link_libraries(ai_face_det_zoned PRIVATE ${RK_RKNN_LIB})
|
||||
endif()
|
||||
set_target_properties(ai_face_det_zoned PROPERTIES
|
||||
OUTPUT_NAME "ai_face_det_zoned"
|
||||
LIBRARY_OUTPUT_DIRECTORY ${RK_PLUGIN_OUTPUT_DIR}
|
||||
RUNTIME_OUTPUT_DIRECTORY ${RK_PLUGIN_OUTPUT_DIR}
|
||||
)
|
||||
|
||||
# ai_scrfd plugin (SCRFD 640x640 face detection)
|
||||
add_library(ai_scrfd SHARED
|
||||
ai_scrfd/ai_scrfd_node.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/utils/dma_alloc.cpp
|
||||
)
|
||||
target_include_directories(ai_scrfd PRIVATE ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/third_party)
|
||||
target_link_libraries(ai_scrfd PRIVATE project_options Threads::Threads ai_scheduler)
|
||||
if(RK3588_ENABLE_RKNN AND RK_RKNN_LIB)
|
||||
target_compile_definitions(ai_scrfd PRIVATE RK3588_ENABLE_RKNN)
|
||||
target_include_directories(ai_scrfd PRIVATE ${RKNN_RUNTIME_INCLUDE_DIR})
|
||||
target_link_libraries(ai_scrfd PRIVATE ${RK_RKNN_LIB})
|
||||
endif()
|
||||
set_target_properties(ai_scrfd PROPERTIES
|
||||
OUTPUT_NAME "ai_scrfd"
|
||||
LIBRARY_OUTPUT_DIRECTORY ${RK_PLUGIN_OUTPUT_DIR}
|
||||
RUNTIME_OUTPUT_DIRECTORY ${RK_PLUGIN_OUTPUT_DIR}
|
||||
)
|
||||
|
||||
# ai_face_recog plugin (RKNN-based ArcFace/MobileFaceNet inference)
|
||||
add_library(ai_face_recog SHARED
|
||||
ai_face_recog/ai_face_recog_node.cpp
|
||||
@ -475,7 +511,7 @@ if(RK3588_ENABLE_ZLMEDIAKIT AND RK_ZLMK_API_LIB)
|
||||
)
|
||||
endif()
|
||||
|
||||
install(TARGETS input_rtsp input_file publish preprocess ai_yolo ai_face_det ai_face_recog tracker gate osd alarm logic_gate storage ai_scheduler
|
||||
install(TARGETS input_rtsp input_file publish preprocess ai_yolo ai_face_det ai_face_det_zoned ai_face_recog tracker gate osd alarm logic_gate storage ai_scheduler
|
||||
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}/rk3588-media-server/plugins
|
||||
RUNTIME DESTINATION ${CMAKE_INSTALL_LIBDIR}/rk3588-media-server/plugins
|
||||
)
|
||||
|
||||
502
plugins/ai_face_det_zoned/ai_face_det_zoned_node.cpp
Normal file
502
plugins/ai_face_det_zoned/ai_face_det_zoned_node.cpp
Normal file
@ -0,0 +1,502 @@
|
||||
/**
|
||||
* ai_face_det_zoned - 三分区距离感知人脸检测节点
|
||||
*
|
||||
* 特性：
|
||||
* 1. 接收原始分辨率输入（不经过前置缩放）
|
||||
* 2. 基于距离进行ROI裁剪和三分区检测
|
||||
* 3. 近区(3-5m) 1.0x / 中区(5-7m) 1.3x / 远区(7-9m) 1.8x
|
||||
* 4. 复用 face_detection_utils.h 中的公共函数
|
||||
*/
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "face/face_detection_utils.h"
|
||||
#include "hw/i_infer_backend.h"
|
||||
#include "face/face_result.h"
|
||||
#include "node.h"
|
||||
#include "utils/dma_alloc.h"
|
||||
#include "utils/logger.h"
|
||||
|
||||
namespace rk3588 {
|
||||
|
||||
using namespace face_detection;
|
||||
|
||||
class AiFaceDetZonedNode : public INode {
|
||||
public:
|
||||
std::string Id() const override { return id_; }
|
||||
std::string Type() const override { return "ai_face_det_zoned"; }
|
||||
|
||||
bool Init(const SimpleJson& config, const NodeContext& ctx) override {
|
||||
id_ = config.ValueOr<std::string>("id", "face_det_zoned");
|
||||
model_path_ = config.ValueOr<std::string>("model_path",
|
||||
"./models/RetinaFace_mobile320.rknn");
|
||||
|
||||
// 基础检测参数
|
||||
det_cfg_.conf_thresh = config.ValueOr<float>("conf", 0.6f);
|
||||
det_cfg_.nms_thresh = config.ValueOr<float>("nms", 0.4f);
|
||||
det_cfg_.max_faces = config.ValueOr<int>("max_faces", 10);
|
||||
det_cfg_.output_landmarks = config.ValueOr<bool>("output_landmarks", true);
|
||||
|
||||
// 模型输入尺寸（默认320）
|
||||
model_w_ = config.ValueOr<int>("model_w", 320);
|
||||
model_h_ = config.ValueOr<int>("model_h", 320);
|
||||
|
||||
// 先验框步长和最小尺寸（RetinaFace默认）
|
||||
det_cfg_.steps = {8, 16, 32};
|
||||
det_cfg_.min_sizes = {{16, 32}, {64, 128}, {256, 512}};
|
||||
|
||||
// ROI配置 - 支持格式: "roi": {"x": 0, "y": 0, "w": 1920, "h": 1080}
|
||||
roi_enabled_ = false;
|
||||
roi_x_ = roi_y_ = roi_w_ = roi_h_ = 0;
|
||||
if (const SimpleJson* roi = config.Find("roi"); roi && roi->IsObject()) {
|
||||
// 直接读取平级格式
|
||||
roi_x_ = roi->ValueOr<int>("x", 0);
|
||||
roi_y_ = roi->ValueOr<int>("y", 0);
|
||||
roi_w_ = roi->ValueOr<int>("w", 0);
|
||||
roi_h_ = roi->ValueOr<int>("h", 0);
|
||||
|
||||
// 如果w/h有效，则启用ROI
|
||||
if (roi_w_ > 0 && roi_h_ > 0) {
|
||||
roi_enabled_ = true;
|
||||
}
|
||||
// 兼容旧格式: "roi": {"crop": {...}}
|
||||
else if (const SimpleJson* crop = roi->Find("crop"); crop && crop->IsObject()) {
|
||||
roi_x_ = crop->ValueOr<int>("x", 0);
|
||||
roi_y_ = crop->ValueOr<int>("y", 0);
|
||||
roi_w_ = crop->ValueOr<int>("w", 0);
|
||||
roi_h_ = crop->ValueOr<int>("h", 0);
|
||||
if (roi_w_ > 0 && roi_h_ > 0) {
|
||||
roi_enabled_ = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 三分区配置 - 支持两种格式：
|
||||
// 1. 旧格式: "distance_zones": {"enabled": true, "boundaries": [y1, y2], "scales": [s1, s2, s3]}
|
||||
// 2. 新格式: "zones": {"near_zone": {"y_start": 0, "y_end": 405, "scale": 0.5}, ...}
|
||||
zones_enabled_ = false;
|
||||
boundary_y_5m_ = boundary_y_7m_ = 0;
|
||||
scale_near_ = 1.0f;
|
||||
scale_mid_ = 1.3f;
|
||||
scale_far_ = 1.8f;
|
||||
|
||||
// 优先尝试新格式 "zones"
|
||||
if (const SimpleJson* zones = config.Find("zones");
|
||||
zones && zones->IsObject()) {
|
||||
bool has_near = false, has_mid = false, has_far = false;
|
||||
int near_y_end = 0, mid_y_end = 0;
|
||||
|
||||
if (const SimpleJson* near = zones->Find("near_zone"); near && near->IsObject()) {
|
||||
near_y_end = near->ValueOr<int>("y_end", 0);
|
||||
scale_near_ = near->ValueOr<float>("scale", 1.0f);
|
||||
has_near = true;
|
||||
}
|
||||
|
||||
if (const SimpleJson* mid = zones->Find("mid_zone"); mid && mid->IsObject()) {
|
||||
mid_y_end = mid->ValueOr<int>("y_end", 0);
|
||||
scale_mid_ = mid->ValueOr<float>("scale", 1.0f);
|
||||
has_mid = true;
|
||||
}
|
||||
|
||||
if (const SimpleJson* far = zones->Find("far_zone"); far && far->IsObject()) {
|
||||
scale_far_ = far->ValueOr<float>("scale", 1.0f);
|
||||
has_far = true;
|
||||
}
|
||||
|
||||
if (has_near && has_mid && has_far) {
|
||||
zones_enabled_ = true;
|
||||
boundary_y_5m_ = near_y_end; // near和mid的分界
|
||||
boundary_y_7m_ = mid_y_end; // mid和far的分界
|
||||
}
|
||||
}
|
||||
// 兼容旧格式
|
||||
else if (const SimpleJson* zones = config.Find("distance_zones");
|
||||
zones && zones->IsObject()) {
|
||||
zones_enabled_ = zones->ValueOr<bool>("enabled", false);
|
||||
|
||||
if (const SimpleJson* boundaries = zones->Find("boundaries");
|
||||
boundaries && boundaries->IsArray() && boundaries->AsArray().size() >= 2) {
|
||||
boundary_y_5m_ = boundaries->AsArray()[0].AsInt(0);
|
||||
boundary_y_7m_ = boundaries->AsArray()[1].AsInt(0);
|
||||
}
|
||||
|
||||
if (const SimpleJson* scales = zones->Find("scales");
|
||||
scales && scales->IsArray() && scales->AsArray().size() >= 3) {
|
||||
scale_near_ = scales->AsArray()[0].AsNumber(1.0f);
|
||||
scale_mid_ = scales->AsArray()[1].AsNumber(1.3f);
|
||||
scale_far_ = scales->AsArray()[2].AsNumber(1.8f);
|
||||
}
|
||||
}
|
||||
|
||||
input_queue_ = ctx.input_queue;
|
||||
output_queues_ = ctx.output_queues;
|
||||
if (!input_queue_) {
|
||||
LogError("[ai_face_det_zoned] no input queue for node " + id_);
|
||||
return false;
|
||||
}
|
||||
if (output_queues_.empty()) {
|
||||
LogError("[ai_face_det_zoned] no output queue for node " + id_);
|
||||
return false;
|
||||
}
|
||||
|
||||
infer_backend_ = ctx.infer_backend;
|
||||
if (!infer_backend_) {
|
||||
LogError("[ai_face_det_zoned] no infer backend for node " + id_);
|
||||
return false;
|
||||
}
|
||||
|
||||
#if defined(RK3588_ENABLE_RKNN)
|
||||
if (model_path_.empty()) {
|
||||
LogError("[ai_face_det_zoned] model_path is required");
|
||||
return false;
|
||||
}
|
||||
std::string err;
|
||||
model_handle_ = infer_backend_->LoadModel(model_path_, err);
|
||||
if (model_handle_ == kInvalidModelHandle) {
|
||||
LogError("[ai_face_det_zoned] failed to load model: " + err);
|
||||
return false;
|
||||
}
|
||||
|
||||
// 预计算先验框
|
||||
priors_ = GeneratePriors(model_w_, model_h_, det_cfg_.steps, det_cfg_.min_sizes);
|
||||
|
||||
LogInfo("[ai_face_det_zoned] model loaded: " + model_path_ +
|
||||
" (" + std::to_string(model_w_) + "x" + std::to_string(model_h_) +
|
||||
"), priors=" + std::to_string(priors_.size()));
|
||||
#else
|
||||
LogWarn("[ai_face_det_zoned] RKNN disabled, will passthrough frames");
|
||||
#endif
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Start() override {
|
||||
LogInfo("[ai_face_det_zoned] start id=" + id_ +
|
||||
" zones=" + std::string(zones_enabled_ ? "enabled" : "disabled") +
|
||||
" roi=" + std::string(roi_enabled_ ? "enabled" : "disabled") +
|
||||
" roi_xywh=" + std::to_string(roi_x_) + "," + std::to_string(roi_y_) + "," +
|
||||
std::to_string(roi_w_) + "," + std::to_string(roi_h_) +
|
||||
" boundaries=" + std::to_string(boundary_y_5m_) + "," + std::to_string(boundary_y_7m_) +
|
||||
" scales=" + std::to_string(scale_near_) + "," + std::to_string(scale_mid_) + "," + std::to_string(scale_far_));
|
||||
return true;
|
||||
}
|
||||
|
||||
void Stop() override {
|
||||
#if defined(RK3588_ENABLE_RKNN)
|
||||
if (model_handle_ != kInvalidModelHandle) {
|
||||
infer_backend_->UnloadModel(model_handle_);
|
||||
model_handle_ = kInvalidModelHandle;
|
||||
}
|
||||
#endif
|
||||
LogInfo("[ai_face_det_zoned] stop id=" + id_);
|
||||
}
|
||||
|
||||
NodeStatus Process(FramePtr frame) override {
|
||||
if (!frame) return NodeStatus::DROP;
|
||||
|
||||
#if defined(RK3588_ENABLE_RKNN)
|
||||
RunZonedDetection(frame);
|
||||
#endif
|
||||
|
||||
Push(frame);
|
||||
return NodeStatus::OK;
|
||||
}
|
||||
|
||||
private:
|
||||
void Push(FramePtr frame) {
|
||||
for (auto& q : output_queues_) q->Push(frame);
|
||||
}
|
||||
|
||||
#if defined(RK3588_ENABLE_RKNN)
|
||||
|
||||
// 将RKNN输出转换为TensorView
|
||||
TensorView ConvertToTensorView(const AiScheduler::BorrowedOutput& o) {
|
||||
TensorView tv;
|
||||
tv.data = o.data;
|
||||
tv.size = o.size;
|
||||
tv.zp = o.zp;
|
||||
tv.scale = o.scale;
|
||||
tv.dims = o.dims;
|
||||
tv.type = o.type;
|
||||
return tv;
|
||||
}
|
||||
|
||||
void RunZonedDetection(FramePtr frame) {
|
||||
if (!frame->data || frame->data_size == 0) return;
|
||||
if (frame->format != PixelFormat::RGB && frame->format != PixelFormat::BGR) {
|
||||
LogWarn("[ai_face_det_zoned] input must be RGB/BGR");
|
||||
return;
|
||||
}
|
||||
|
||||
const int src_w = frame->width;
|
||||
const int src_h = frame->height;
|
||||
|
||||
// 应用ROI裁剪
|
||||
int roi_x = 0, roi_y = 0, roi_w = src_w, roi_h = src_h;
|
||||
if (roi_enabled_) {
|
||||
roi_x = ClampInt(roi_x_, 0, src_w - 1);
|
||||
roi_y = ClampInt(roi_y_, 0, src_h - 1);
|
||||
roi_w = ClampInt(roi_w_, 1, src_w - roi_x);
|
||||
roi_h = ClampInt(roi_h_, 1, src_h - roi_y);
|
||||
}
|
||||
|
||||
std::vector<FaceDetItem> all_detections;
|
||||
|
||||
if (zones_enabled_) {
|
||||
// 三分区检测
|
||||
all_detections = DetectWithZones(frame, roi_x, roi_y, roi_w, roi_h);
|
||||
} else {
|
||||
// 单区检测（全ROI区域）
|
||||
auto dets = DetectSingleZone(frame, roi_x, roi_y, roi_w, roi_h, 1.0f);
|
||||
// 坐标映射回原始图像
|
||||
for (auto& det : dets) {
|
||||
det.bbox.x += roi_x;
|
||||
det.bbox.y += roi_y;
|
||||
if (det.has_landmarks) {
|
||||
for (auto& lm : det.landmarks) {
|
||||
lm.x += roi_x;
|
||||
lm.y += roi_y;
|
||||
}
|
||||
}
|
||||
all_detections.push_back(det);
|
||||
}
|
||||
}
|
||||
|
||||
// NMS去重
|
||||
all_detections = ApplyNMS(all_detections, det_cfg_.nms_thresh);
|
||||
|
||||
// 限制最大人脸数
|
||||
if (all_detections.size() > static_cast<size_t>(det_cfg_.max_faces)) {
|
||||
all_detections.resize(det_cfg_.max_faces);
|
||||
}
|
||||
|
||||
// 构建结果
|
||||
FaceDetResult det_result;
|
||||
det_result.img_w = src_w;
|
||||
det_result.img_h = src_h;
|
||||
det_result.model_name = "retinaface_zoned";
|
||||
det_result.faces = std::move(all_detections);
|
||||
|
||||
frame->face_det = std::make_shared<FaceDetResult>(std::move(det_result));
|
||||
}
|
||||
|
||||
std::vector<FaceDetItem> DetectWithZones(FramePtr frame,
|
||||
int roi_x, int roi_y,
|
||||
int roi_w, int roi_h) {
|
||||
std::vector<FaceDetItem> all_dets;
|
||||
|
||||
// 将分界线坐标转换到ROI坐标系
|
||||
int by5 = ClampInt(boundary_y_5m_ - roi_y, 0, roi_h);
|
||||
int by7 = ClampInt(boundary_y_7m_ - roi_y, 0, roi_h);
|
||||
|
||||
// 确保顺序正确（y大=下方=近距离）
|
||||
if (by5 < by7) std::swap(by5, by7);
|
||||
|
||||
// 近区检测 (画面下方，y大，近距离3-5m)
|
||||
if (by5 < roi_h) {
|
||||
auto dets = DetectSingleZone(frame, roi_x, roi_y + by5, roi_w, roi_h - by5, scale_near_);
|
||||
for (auto& det : dets) {
|
||||
det.bbox.x += roi_x;
|
||||
det.bbox.y += roi_y + by5;
|
||||
if (det.has_landmarks) {
|
||||
for (auto& lm : det.landmarks) {
|
||||
lm.x += roi_x;
|
||||
lm.y += roi_y + by5;
|
||||
}
|
||||
}
|
||||
all_dets.push_back(det);
|
||||
}
|
||||
}
|
||||
|
||||
// 中区检测 (画面中部，中距离5-7m)
|
||||
if (by7 < by5) {
|
||||
auto dets = DetectSingleZone(frame, roi_x, roi_y + by7, roi_w, by5 - by7, scale_mid_);
|
||||
for (auto& det : dets) {
|
||||
det.bbox.x += roi_x;
|
||||
det.bbox.y += roi_y + by7;
|
||||
if (det.has_landmarks) {
|
||||
for (auto& lm : det.landmarks) {
|
||||
lm.x += roi_x;
|
||||
lm.y += roi_y + by7;
|
||||
}
|
||||
}
|
||||
all_dets.push_back(det);
|
||||
}
|
||||
}
|
||||
|
||||
// 远区检测 (画面上方，y小，远距离7-9m)
|
||||
if (by7 > 0) {
|
||||
auto dets = DetectSingleZone(frame, roi_x, roi_y, roi_w, by7, scale_far_);
|
||||
for (auto& det : dets) {
|
||||
det.bbox.x += roi_x;
|
||||
det.bbox.y += roi_y;
|
||||
if (det.has_landmarks) {
|
||||
for (auto& lm : det.landmarks) {
|
||||
lm.x += roi_x;
|
||||
lm.y += roi_y;
|
||||
}
|
||||
}
|
||||
all_dets.push_back(det);
|
||||
}
|
||||
}
|
||||
|
||||
return all_dets;
|
||||
}
|
||||
|
||||
std::vector<FaceDetItem> DetectSingleZone(FramePtr frame,
|
||||
int x, int y, int w, int h,
|
||||
float scale) {
|
||||
std::vector<FaceDetItem> dets;
|
||||
|
||||
if (w <= 0 || h <= 0) return dets;
|
||||
|
||||
const uint8_t* src = frame->planes[0].data ? frame->planes[0].data : frame->data;
|
||||
const int src_stride = frame->planes[0].stride > 0 ? frame->planes[0].stride
|
||||
: (frame->stride > 0 ? frame->stride : frame->width * 3);
|
||||
|
||||
// 裁剪区域缩放后的尺寸
|
||||
int crop_w = static_cast<int>(w * scale);
|
||||
int crop_h = static_cast<int>(h * scale);
|
||||
if (crop_w <= 0 || crop_h <= 0) return dets;
|
||||
|
||||
// 分配缓冲区
|
||||
input_buf_.resize(static_cast<size_t>(model_w_) * model_h_ * 3);
|
||||
|
||||
// 双线性缩放到模型输入尺寸
|
||||
// 注意：这里从原图裁剪(x,y,w,h)，缩放到(model_w_, model_h_)
|
||||
// 优化：可以直接从src裁剪并缩放，避免中间buffer
|
||||
|
||||
// 简化的处理：先裁剪到临时buffer，再缩放
|
||||
std::vector<uint8_t> crop_buf(static_cast<size_t>(w) * h * 3);
|
||||
for (int row = 0; row < h; ++row) {
|
||||
const uint8_t* src_row = src + (y + row) * src_stride + x * 3;
|
||||
uint8_t* dst_row = crop_buf.data() + row * w * 3;
|
||||
memcpy(dst_row, src_row, static_cast<size_t>(w) * 3);
|
||||
}
|
||||
|
||||
// 缩放到模型输入尺寸
|
||||
ResizeRgbBilinear(crop_buf.data(), w, h, w * 3,
|
||||
input_buf_.data(), model_w_, model_h_,
|
||||
false); // 假设输入已经是RGB
|
||||
|
||||
// NPU推理
|
||||
InferInput input;
|
||||
input.width = model_w_;
|
||||
input.height = model_h_;
|
||||
input.is_nhwc = true;
|
||||
input.data = input_buf_.data();
|
||||
input.size = input_buf_.size();
|
||||
input.type = RKNN_TENSOR_UINT8;
|
||||
|
||||
auto r = infer_backend_->InferBorrowed(model_handle_, input);
|
||||
if (!r.success || r.outputs.empty()) {
|
||||
LogWarn("[ai_face_det_zoned] inference failed");
|
||||
return dets;
|
||||
}
|
||||
|
||||
// 解析输出
|
||||
NcTensor loc_tensor, conf_tensor, landm_tensor;
|
||||
bool has_loc = false, has_conf = false, has_landm = false;
|
||||
|
||||
for (const auto& o : r.outputs) {
|
||||
TensorView tv = ConvertToTensorView(o);
|
||||
NcTensor tmp;
|
||||
if (!has_loc && ExtractNcTensor(tv, 4, tmp)) {
|
||||
loc_tensor = std::move(tmp);
|
||||
has_loc = true;
|
||||
} else if (!has_conf && ExtractNcTensor(tv, 2, tmp)) {
|
||||
conf_tensor = std::move(tmp);
|
||||
has_conf = true;
|
||||
} else if (!has_landm && ExtractNcTensor(tv, 10, tmp)) {
|
||||
landm_tensor = std::move(tmp);
|
||||
has_landm = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!has_loc || !has_conf) return dets;
|
||||
|
||||
// 解码检测结果
|
||||
FaceDetResult result;
|
||||
DecodeRetinaFace(loc_tensor, conf_tensor, landm_tensor,
|
||||
priors_, w, h, model_w_, model_h_,
|
||||
det_cfg_, result);
|
||||
|
||||
if (!result.faces.empty()) {
|
||||
LogInfo("[ai_face_det_zoned] DetectSingleZone: detected " +
|
||||
std::to_string(result.faces.size()) + " faces, max_score=" +
|
||||
std::to_string(result.faces.empty() ? 0 : result.faces[0].score));
|
||||
}
|
||||
|
||||
return result.faces;
|
||||
}
|
||||
|
||||
std::vector<FaceDetItem> ApplyNMS(std::vector<FaceDetItem>& dets, float threshold) {
|
||||
if (dets.empty()) return dets;
|
||||
|
||||
// 按置信度排序
|
||||
std::sort(dets.begin(), dets.end(),
|
||||
[](const FaceDetItem& a, const FaceDetItem& b) {
|
||||
return a.score > b.score;
|
||||
});
|
||||
|
||||
std::vector<FaceDetItem> keep;
|
||||
std::vector<bool> suppressed(dets.size(), false);
|
||||
|
||||
for (size_t i = 0; i < dets.size(); ++i) {
|
||||
if (suppressed[i]) continue;
|
||||
keep.push_back(dets[i]);
|
||||
|
||||
for (size_t j = i + 1; j < dets.size(); ++j) {
|
||||
if (suppressed[j]) continue;
|
||||
if (IoU(dets[i].bbox, dets[j].bbox) > threshold) {
|
||||
suppressed[j] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return keep;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
std::string id_;
|
||||
std::string model_path_;
|
||||
|
||||
DetectionConfig det_cfg_;
|
||||
int model_w_ = 320;
|
||||
int model_h_ = 320;
|
||||
|
||||
// ROI
|
||||
bool roi_enabled_ = false;
|
||||
int roi_x_ = 0, roi_y_ = 0, roi_w_ = 0, roi_h_ = 0;
|
||||
|
||||
// 三分区
|
||||
bool zones_enabled_ = false;
|
||||
int boundary_y_5m_ = 0;
|
||||
int boundary_y_7m_ = 0;
|
||||
float scale_near_ = 1.0f;
|
||||
float scale_mid_ = 1.3f;
|
||||
float scale_far_ = 1.8f;
|
||||
|
||||
std::shared_ptr<SpscQueue<FramePtr>> input_queue_;
|
||||
std::vector<std::shared_ptr<SpscQueue<FramePtr>>> output_queues_;
|
||||
std::shared_ptr<IInferBackend> infer_backend_;
|
||||
|
||||
#if defined(RK3588_ENABLE_RKNN)
|
||||
ModelHandle model_handle_ = kInvalidModelHandle;
|
||||
std::vector<Prior> priors_;
|
||||
std::vector<uint8_t> input_buf_;
|
||||
#endif
|
||||
};
|
||||
|
||||
REGISTER_NODE(AiFaceDetZonedNode, "ai_face_det_zoned");
|
||||
|
||||
} // namespace rk3588
|
||||
395
plugins/ai_scrfd/ai_scrfd_node.cpp
Normal file
395
plugins/ai_scrfd/ai_scrfd_node.cpp
Normal file
@ -0,0 +1,395 @@
|
||||
/**
|
||||
* ai_scrfd - SCRFD 640x640 face detection node for RK3588
|
||||
*
|
||||
* Reference: https://github.com/DefTruth/lite.ai.toolkit/blob/main/lite/ort/cv/scrfd.cpp
|
||||
* BBox format: [left, top, right, bottom] offsets from grid center
|
||||
*/
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "face/face_result.h"
|
||||
#include "hw/i_infer_backend.h"
|
||||
#include "node.h"
|
||||
#include "utils/dma_alloc.h"
|
||||
#include "utils/logger.h"
|
||||
|
||||
namespace rk3588 {
|
||||
|
||||
struct ScrfdConfig {
|
||||
float conf_thresh = 0.5f;
|
||||
float nms_thresh = 0.4f;
|
||||
int max_faces = 50;
|
||||
bool output_landmarks = true;
|
||||
std::string input_format = "rgb";
|
||||
};
|
||||
|
||||
// Grid center point for each anchor
|
||||
struct CenterPoint {
|
||||
float cx, cy; // grid coordinates (0,0), (1,0), ...
|
||||
float stride;
|
||||
};
|
||||
|
||||
class AiScrfdNode : public INode {
|
||||
public:
|
||||
std::string Id() const override { return id_; }
|
||||
std::string Type() const override { return "ai_scrfd"; }
|
||||
|
||||
bool Init(const SimpleJson& config, const NodeContext& ctx) override {
|
||||
id_ = config.ValueOr<std::string>("id", "scrfd");
|
||||
model_path_ = config.ValueOr<std::string>("model_path",
|
||||
"./models/scrfd_500m_640.rknn");
|
||||
|
||||
cfg_.conf_thresh = config.ValueOr<float>("conf_thresh", 0.5f);
|
||||
cfg_.nms_thresh = config.ValueOr<float>("nms_thresh", 0.4f);
|
||||
cfg_.max_faces = config.ValueOr<int>("max_faces", 50);
|
||||
cfg_.output_landmarks = config.ValueOr<bool>("output_landmarks", true);
|
||||
cfg_.input_format = config.ValueOr<std::string>("input_format", "rgb");
|
||||
|
||||
model_w_ = 640;
|
||||
model_h_ = 640;
|
||||
|
||||
// Generate center points for all anchors
|
||||
GenerateCenterPoints();
|
||||
|
||||
input_queue_ = ctx.input_queue;
|
||||
output_queues_ = ctx.output_queues;
|
||||
if (!input_queue_) {
|
||||
LogError("[ai_scrfd] no input queue");
|
||||
return false;
|
||||
}
|
||||
|
||||
infer_backend_ = ctx.infer_backend;
|
||||
if (!infer_backend_) {
|
||||
LogError("[ai_scrfd] no infer backend");
|
||||
return false;
|
||||
}
|
||||
|
||||
#if defined(RK3588_ENABLE_RKNN)
|
||||
std::string err;
|
||||
model_handle_ = infer_backend_->LoadModel(model_path_, err);
|
||||
if (model_handle_ == kInvalidModelHandle) {
|
||||
LogError("[ai_scrfd] failed to load model: " + err);
|
||||
return false;
|
||||
}
|
||||
|
||||
input_buf_.resize(model_w_ * model_h_ * 3);
|
||||
|
||||
LogInfo("[ai_scrfd] model loaded: " + model_path_);
|
||||
#else
|
||||
LogWarn("[ai_scrfd] RKNN disabled");
|
||||
#endif
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Start() override {
|
||||
LogInfo("[ai_scrfd] start");
|
||||
return true;
|
||||
}
|
||||
|
||||
void Stop() override {
|
||||
#if defined(RK3588_ENABLE_RKNN)
|
||||
if (model_handle_ != kInvalidModelHandle) {
|
||||
infer_backend_->UnloadModel(model_handle_);
|
||||
model_handle_ = kInvalidModelHandle;
|
||||
}
|
||||
#endif
|
||||
LogInfo("[ai_scrfd] stop");
|
||||
}
|
||||
|
||||
NodeStatus Process(FramePtr frame) override {
|
||||
if (!frame) return NodeStatus::DROP;
|
||||
|
||||
#if defined(RK3588_ENABLE_RKNN)
|
||||
RunDetection(frame);
|
||||
#endif
|
||||
|
||||
Push(frame);
|
||||
return NodeStatus::OK;
|
||||
}
|
||||
|
||||
private:
|
||||
void Push(FramePtr frame) {
|
||||
for (auto& q : output_queues_) q->Push(frame);
|
||||
}
|
||||
|
||||
#if defined(RK3588_ENABLE_RKNN)
|
||||
|
||||
void GenerateCenterPoints() {
|
||||
// strides: 8, 16, 32
|
||||
const int strides[] = {8, 16, 32};
|
||||
|
||||
for (int stride : strides) {
|
||||
int num_grid = model_w_ / stride;
|
||||
for (int y = 0; y < num_grid; ++y) {
|
||||
for (int x = 0; x < num_grid; ++x) {
|
||||
// 2 anchors per location
|
||||
for (int a = 0; a < 2; ++a) {
|
||||
CenterPoint pt;
|
||||
pt.cx = static_cast<float>(x); // grid x
|
||||
pt.cy = static_cast<float>(y); // grid y
|
||||
pt.stride = static_cast<float>(stride);
|
||||
center_points_.push_back(pt);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
LogInfo("[ai_scrfd] Generated " + std::to_string(center_points_.size()) + " center points");
|
||||
}
|
||||
|
||||
void RunDetection(FramePtr frame) {
|
||||
if (!frame->data || frame->data_size == 0) return;
|
||||
|
||||
const int src_w = frame->width;
|
||||
const int src_h = frame->height;
|
||||
|
||||
if (frame->DmaFd() >= 0) frame->SyncStart();
|
||||
|
||||
PrepareInput(frame, model_w_, model_h_);
|
||||
|
||||
InferInput input;
|
||||
input.width = model_w_;
|
||||
input.height = model_h_;
|
||||
input.is_nhwc = true;
|
||||
input.data = input_buf_.data();
|
||||
input.size = input_buf_.size();
|
||||
input.type = RKNN_TENSOR_UINT8;
|
||||
|
||||
auto r = infer_backend_->InferBorrowed(model_handle_, input);
|
||||
if (!r.success) {
|
||||
LogWarn("[ai_scrfd] inference failed: " + r.error);
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<FaceDetItem> detections = ParseOutputs(r.outputs, src_w, src_h);
|
||||
detections = ApplyNMS(detections, cfg_.nms_thresh);
|
||||
|
||||
if (detections.size() > static_cast<size_t>(cfg_.max_faces)) {
|
||||
detections.resize(cfg_.max_faces);
|
||||
}
|
||||
|
||||
FaceDetResult result;
|
||||
result.img_w = src_w;
|
||||
result.img_h = src_h;
|
||||
result.model_name = "scrfd_640";
|
||||
result.faces = std::move(detections);
|
||||
|
||||
frame->face_det = std::make_shared<FaceDetResult>(std::move(result));
|
||||
}
|
||||
|
||||
void PrepareInput(FramePtr frame, int dst_w, int dst_h) {
|
||||
const uint8_t* src = frame->planes[0].data ? frame->planes[0].data : frame->data;
|
||||
const int src_stride = frame->planes[0].stride > 0 ? frame->planes[0].stride
|
||||
: (frame->stride > 0 ? frame->stride : frame->width * 3);
|
||||
|
||||
// Simple bilinear resize
|
||||
ResizeBilinear(src, frame->width, frame->height, src_stride,
|
||||
input_buf_.data(), dst_w, dst_h, dst_w * 3);
|
||||
|
||||
// RGB/BGR conversion if needed
|
||||
bool need_swap = (frame->format == PixelFormat::BGR && cfg_.input_format == "rgb") ||
|
||||
(frame->format == PixelFormat::RGB && cfg_.input_format == "bgr");
|
||||
|
||||
if (need_swap) {
|
||||
for (int i = 0; i < dst_w * dst_h * 3; i += 3) {
|
||||
std::swap(input_buf_[i], input_buf_[i + 2]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ResizeBilinear(const uint8_t* src, int src_w, int src_h, int src_stride,
|
||||
uint8_t* dst, int dst_w, int dst_h, int dst_stride) {
|
||||
float x_ratio = static_cast<float>(src_w) / dst_w;
|
||||
float y_ratio = static_cast<float>(src_h) / dst_h;
|
||||
|
||||
for (int y = 0; y < dst_h; ++y) {
|
||||
for (int x = 0; x < dst_w; ++x) {
|
||||
float sx = (x + 0.5f) * x_ratio - 0.5f;
|
||||
float sy = (y + 0.5f) * y_ratio - 0.5f;
|
||||
|
||||
int x0 = static_cast<int>(std::floor(sx));
|
||||
int y0 = static_cast<int>(std::floor(sy));
|
||||
int x1 = std::min(x0 + 1, src_w - 1);
|
||||
int y1 = std::min(y0 + 1, src_h - 1);
|
||||
|
||||
x0 = std::max(0, x0);
|
||||
y0 = std::max(0, y0);
|
||||
|
||||
float fx = sx - x0;
|
||||
float fy = sy - y0;
|
||||
|
||||
for (int c = 0; c < 3; ++c) {
|
||||
float v00 = src[y0 * src_stride + x0 * 3 + c];
|
||||
float v01 = src[y0 * src_stride + x1 * 3 + c];
|
||||
float v10 = src[y1 * src_stride + x0 * 3 + c];
|
||||
float v11 = src[y1 * src_stride + x1 * 3 + c];
|
||||
|
||||
float v = v00 * (1-fx) * (1-fy) + v01 * fx * (1-fy) +
|
||||
v10 * (1-fx) * fy + v11 * fx * fy;
|
||||
|
||||
dst[y * dst_stride + x * 3 + c] = static_cast<uint8_t>(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse SCRFD outputs - reference implementation from lite.ai.toolkit
|
||||
*
|
||||
* BBox format: [left, top, right, bottom] - distances from grid center
|
||||
* NOT [dx, dy, dw, dh]!
|
||||
*/
|
||||
std::vector<FaceDetItem> ParseOutputs(
|
||||
const std::vector<AiScheduler::BorrowedOutput>& outputs,
|
||||
int src_w, int src_h) {
|
||||
std::vector<FaceDetItem> detections;
|
||||
|
||||
if (outputs.size() != 9) return detections;
|
||||
|
||||
// Output order: score_8, score_16, score_32, bbox_8, bbox_16, bbox_32, kps_8, kps_16, kps_32
|
||||
const int anchor_counts[] = {12800, 3200, 800};
|
||||
const int strides[] = {8, 16, 32};
|
||||
|
||||
size_t anchor_idx = 0;
|
||||
|
||||
for (int s = 0; s < 3; ++s) {
|
||||
int stride = strides[s];
|
||||
int count = anchor_counts[s];
|
||||
|
||||
const auto& score_out = outputs[s];
|
||||
const auto& bbox_out = outputs[s + 3];
|
||||
const auto& kps_out = outputs[s + 6];
|
||||
|
||||
if (score_out.dims.size() < 3) continue;
|
||||
|
||||
const float* scores = reinterpret_cast<const float*>(score_out.data);
|
||||
const float* bboxes = reinterpret_cast<const float*>(bbox_out.data);
|
||||
const float* kps = reinterpret_cast<const float*>(kps_out.data);
|
||||
|
||||
if (!scores || !bboxes || !kps) continue;
|
||||
|
||||
for (int i = 0; i < count; ++i) {
|
||||
if (anchor_idx >= center_points_.size()) break;
|
||||
|
||||
float score = scores[i];
|
||||
if (score < cfg_.conf_thresh) {
|
||||
anchor_idx++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const CenterPoint& pt = center_points_[anchor_idx];
|
||||
|
||||
// BBox: [left, top, right, bottom] - distances from center
|
||||
float left = bboxes[i * 4 + 0];
|
||||
float top = bboxes[i * 4 + 1];
|
||||
float right = bboxes[i * 4 + 2];
|
||||
float bottom = bboxes[i * 4 + 3];
|
||||
|
||||
// Decode to image coordinates (640x640)
|
||||
float x1_640 = (pt.cx - left) * stride;
|
||||
float y1_640 = (pt.cy - top) * stride;
|
||||
float x2_640 = (pt.cx + right) * stride;
|
||||
float y2_640 = (pt.cy + bottom) * stride;
|
||||
|
||||
// Scale to original image size
|
||||
float scale_x = static_cast<float>(src_w) / model_w_;
|
||||
float scale_y = static_cast<float>(src_h) / model_h_;
|
||||
|
||||
FaceDetItem det;
|
||||
det.bbox.x = x1_640 * scale_x;
|
||||
det.bbox.y = y1_640 * scale_y;
|
||||
det.bbox.w = (x2_640 - x1_640) * scale_x;
|
||||
det.bbox.h = (y2_640 - y1_640) * scale_y;
|
||||
det.score = score;
|
||||
det.has_landmarks = cfg_.output_landmarks;
|
||||
|
||||
// Keypoints
|
||||
if (cfg_.output_landmarks) {
|
||||
for (int p = 0; p < 5; ++p) {
|
||||
float kps_x = kps[i * 10 + p * 2 + 0];
|
||||
float kps_y = kps[i * 10 + p * 2 + 1];
|
||||
float kx_640 = (pt.cx + kps_x) * stride;
|
||||
float ky_640 = (pt.cy + kps_y) * stride;
|
||||
det.landmarks[p].x = kx_640 * scale_x;
|
||||
det.landmarks[p].y = ky_640 * scale_y;
|
||||
}
|
||||
}
|
||||
|
||||
detections.push_back(det);
|
||||
anchor_idx++;
|
||||
}
|
||||
}
|
||||
|
||||
return detections;
|
||||
}
|
||||
|
||||
float IoU(const Rect& a, const Rect& b) {
|
||||
float x1 = std::max(a.x, b.x);
|
||||
float y1 = std::max(a.y, b.y);
|
||||
float x2 = std::min(a.x + a.w, b.x + b.w);
|
||||
float y2 = std::min(a.y + a.h, b.y + b.h);
|
||||
|
||||
float inter = std::max(0.0f, x2 - x1) * std::max(0.0f, y2 - y1);
|
||||
float area_a = a.w * a.h;
|
||||
float area_b = b.w * b.h;
|
||||
float union_area = area_a + area_b - inter;
|
||||
|
||||
return union_area > 0 ? inter / union_area : 0;
|
||||
}
|
||||
|
||||
std::vector<FaceDetItem> ApplyNMS(std::vector<FaceDetItem>& dets, float thresh) {
|
||||
if (dets.empty()) return dets;
|
||||
|
||||
std::sort(dets.begin(), dets.end(),
|
||||
[](const FaceDetItem& a, const FaceDetItem& b) {
|
||||
return a.score > b.score;
|
||||
});
|
||||
|
||||
std::vector<FaceDetItem> keep;
|
||||
std::vector<bool> suppressed(dets.size(), false);
|
||||
|
||||
for (size_t i = 0; i < dets.size(); ++i) {
|
||||
if (suppressed[i]) continue;
|
||||
keep.push_back(dets[i]);
|
||||
|
||||
for (size_t j = i + 1; j < dets.size(); ++j) {
|
||||
if (suppressed[j]) continue;
|
||||
if (IoU(dets[i].bbox, dets[j].bbox) > thresh) {
|
||||
suppressed[j] = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return keep;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
std::string id_;
|
||||
std::string model_path_;
|
||||
ScrfdConfig cfg_;
|
||||
int model_w_ = 640;
|
||||
int model_h_ = 640;
|
||||
|
||||
std::vector<CenterPoint> center_points_;
|
||||
|
||||
std::shared_ptr<SpscQueue<FramePtr>> input_queue_;
|
||||
std::vector<std::shared_ptr<SpscQueue<FramePtr>>> output_queues_;
|
||||
std::shared_ptr<IInferBackend> infer_backend_;
|
||||
|
||||
#if defined(RK3588_ENABLE_RKNN)
|
||||
ModelHandle model_handle_ = kInvalidModelHandle;
|
||||
std::vector<uint8_t> input_buf_;
|
||||
#endif
|
||||
};
|
||||
|
||||
REGISTER_NODE(AiScrfdNode, "ai_scrfd");
|
||||
|
||||
} // namespace rk3588
|
||||
@ -1,461 +1,464 @@
|
||||
#include "clip_action.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <ctime>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iomanip>
|
||||
#include <mutex>
|
||||
#include <sstream>
|
||||
#include <thread>
|
||||
|
||||
#if defined(_WIN32)
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#elif defined(__unix__) || defined(__APPLE__)
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#include "utils/logger.h"
|
||||
|
||||
#if defined(RK3588_ENABLE_FFMPEG)
|
||||
extern "C" {
|
||||
#include <libavcodec/avcodec.h>
|
||||
#include <libavformat/avformat.h>
|
||||
#include <libavutil/avutil.h>
|
||||
}
|
||||
#define HAS_FFMPEG 1
|
||||
#else
|
||||
#define HAS_FFMPEG 0
|
||||
#endif
|
||||
|
||||
namespace rk3588 {
|
||||
|
||||
namespace {
|
||||
|
||||
bool SafeLocalTime(std::time_t t, std::tm& out) {
|
||||
#if defined(_WIN32)
|
||||
return localtime_s(&out, &t) == 0;
|
||||
#elif defined(__unix__) || defined(__APPLE__)
|
||||
return localtime_r(&t, &out) != nullptr;
|
||||
#else
|
||||
static std::mutex mu;
|
||||
std::lock_guard<std::mutex> lock(mu);
|
||||
std::tm* p = std::localtime(&t);
|
||||
if (!p) return false;
|
||||
out = *p;
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
static std::filesystem::path CreateTempFilePath(const std::filesystem::path& dir, const std::string& prefix) {
|
||||
std::filesystem::path base = dir;
|
||||
if (base.empty()) base = ".";
|
||||
|
||||
#if defined(_WIN32)
|
||||
wchar_t tmp_dir[MAX_PATH];
|
||||
DWORD n = GetTempPathW(MAX_PATH, tmp_dir);
|
||||
if (n == 0 || n > MAX_PATH) {
|
||||
return base / (prefix + "_tmp");
|
||||
}
|
||||
wchar_t tmp_file[MAX_PATH];
|
||||
if (GetTempFileNameW(tmp_dir, L"clp", 0, tmp_file) == 0) {
|
||||
return base / (prefix + "_tmp");
|
||||
}
|
||||
return std::filesystem::path(tmp_file);
|
||||
|
||||
#elif defined(__unix__) || defined(__APPLE__)
|
||||
std::string tmpl = (base / (prefix + "_XXXXXX")).string();
|
||||
std::vector<char> buf(tmpl.begin(), tmpl.end());
|
||||
buf.push_back('\0');
|
||||
int fd = mkstemp(buf.data());
|
||||
if (fd >= 0) close(fd);
|
||||
if (fd < 0) {
|
||||
return base / (prefix + "_tmp");
|
||||
}
|
||||
return std::filesystem::path(buf.data());
|
||||
|
||||
#else
|
||||
const auto now = std::chrono::high_resolution_clock::now().time_since_epoch().count();
|
||||
return base / (prefix + "_" + std::to_string(static_cast<long long>(now)));
|
||||
#endif
|
||||
}
|
||||
|
||||
static bool HasAnnexBStartCode(const uint8_t* d, size_t n) {
|
||||
if (!d || n < 4) return false;
|
||||
for (size_t i = 0; i + 3 < n; ++i) {
|
||||
if (d[i] == 0 && d[i + 1] == 0 && d[i + 2] == 1) return true;
|
||||
if (i + 4 < n && d[i] == 0 && d[i + 1] == 0 && d[i + 2] == 0 && d[i + 3] == 1) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static size_t FindStartCode(const uint8_t* d, size_t n, size_t from, size_t* sc_len) {
|
||||
for (size_t i = from; i + 3 < n; ++i) {
|
||||
if (d[i] == 0 && d[i + 1] == 0 && d[i + 2] == 1) {
|
||||
if (sc_len) *sc_len = 3;
|
||||
return i;
|
||||
}
|
||||
if (i + 4 < n && d[i] == 0 && d[i + 1] == 0 && d[i + 2] == 0 && d[i + 3] == 1) {
|
||||
if (sc_len) *sc_len = 4;
|
||||
return i;
|
||||
}
|
||||
}
|
||||
if (sc_len) *sc_len = 0;
|
||||
return n;
|
||||
}
|
||||
|
||||
static std::vector<std::vector<uint8_t>> SplitAnnexBNals(const uint8_t* d, size_t n) {
|
||||
std::vector<std::vector<uint8_t>> out;
|
||||
if (!d || n < 4) return out;
|
||||
size_t pos = 0;
|
||||
while (true) {
|
||||
size_t sc_len = 0;
|
||||
size_t sc = FindStartCode(d, n, pos, &sc_len);
|
||||
if (sc >= n) break;
|
||||
size_t nal_start = sc + sc_len;
|
||||
size_t next_sc = FindStartCode(d, n, nal_start, nullptr);
|
||||
size_t nal_end = (next_sc >= n) ? n : next_sc;
|
||||
|
||||
// Trim trailing zeros before next start code.
|
||||
while (nal_end > nal_start && d[nal_end - 1] == 0) --nal_end;
|
||||
|
||||
if (nal_end > nal_start) {
|
||||
out.emplace_back(d + nal_start, d + nal_end);
|
||||
}
|
||||
pos = nal_end;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static bool BuildAvccFromAnnexB(const std::vector<uint8_t>& annexb, std::vector<uint8_t>& avcc_out) {
|
||||
avcc_out.clear();
|
||||
if (annexb.empty()) return false;
|
||||
auto nals = SplitAnnexBNals(annexb.data(), annexb.size());
|
||||
|
||||
const uint8_t* sps = nullptr;
|
||||
size_t sps_len = 0;
|
||||
const uint8_t* pps = nullptr;
|
||||
size_t pps_len = 0;
|
||||
|
||||
for (const auto& nal : nals) {
|
||||
if (nal.empty()) continue;
|
||||
const uint8_t t = nal[0] & 0x1F;
|
||||
if (t == 7 && !sps) {
|
||||
sps = nal.data();
|
||||
sps_len = nal.size();
|
||||
} else if (t == 8 && !pps) {
|
||||
pps = nal.data();
|
||||
pps_len = nal.size();
|
||||
}
|
||||
}
|
||||
|
||||
if (!sps || sps_len < 4 || !pps || pps_len < 1) return false;
|
||||
|
||||
// AVCDecoderConfigurationRecord
|
||||
// https://developer.apple.com/documentation/quicktime-file-format/avcdecoderconfigurationrecord
|
||||
avcc_out.reserve(11 + sps_len + pps_len);
|
||||
avcc_out.push_back(1); // configurationVersion
|
||||
avcc_out.push_back(sps[1]); // AVCProfileIndication
|
||||
avcc_out.push_back(sps[2]); // profile_compatibility
|
||||
avcc_out.push_back(sps[3]); // AVCLevelIndication
|
||||
avcc_out.push_back(0xFF); // 6 bits reserved + lengthSizeMinusOne(3 => 4 bytes)
|
||||
avcc_out.push_back(0xE1); // 3 bits reserved + numOfSequenceParameterSets(1)
|
||||
avcc_out.push_back(static_cast<uint8_t>((sps_len >> 8) & 0xFF));
|
||||
avcc_out.push_back(static_cast<uint8_t>((sps_len) & 0xFF));
|
||||
avcc_out.insert(avcc_out.end(), sps, sps + sps_len);
|
||||
avcc_out.push_back(1); // numOfPictureParameterSets
|
||||
avcc_out.push_back(static_cast<uint8_t>((pps_len >> 8) & 0xFF));
|
||||
avcc_out.push_back(static_cast<uint8_t>((pps_len) & 0xFF));
|
||||
avcc_out.insert(avcc_out.end(), pps, pps + pps_len);
|
||||
return true;
|
||||
}
|
||||
|
||||
static std::vector<uint8_t> AnnexBToLengthPrefixed(const uint8_t* d, size_t n) {
|
||||
std::vector<uint8_t> out;
|
||||
auto nals = SplitAnnexBNals(d, n);
|
||||
// Each NAL becomes [len_be32][nal_bytes]
|
||||
size_t total = 0;
|
||||
for (const auto& nal : nals) total += 4 + nal.size();
|
||||
out.reserve(total);
|
||||
for (const auto& nal : nals) {
|
||||
const uint32_t len = static_cast<uint32_t>(nal.size());
|
||||
out.push_back(static_cast<uint8_t>((len >> 24) & 0xFF));
|
||||
out.push_back(static_cast<uint8_t>((len >> 16) & 0xFF));
|
||||
out.push_back(static_cast<uint8_t>((len >> 8) & 0xFF));
|
||||
out.push_back(static_cast<uint8_t>((len) & 0xFF));
|
||||
out.insert(out.end(), nal.begin(), nal.end());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool ClipAction::Init(const SimpleJson& config) {
|
||||
pre_sec_ = config.ValueOr<int>("pre_sec", 5);
|
||||
post_sec_ = config.ValueOr<int>("post_sec", 10);
|
||||
format_ = config.ValueOr<std::string>("format", "mp4");
|
||||
fps_ = config.ValueOr<int>("fps", 25);
|
||||
|
||||
if (const SimpleJson* upload_cfg = config.Find("upload")) {
|
||||
uploader_ = CreateUploader(*upload_cfg);
|
||||
if (!uploader_) {
|
||||
LogError("[ClipAction] failed to create uploader");
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
SimpleJson default_cfg;
|
||||
uploader_ = CreateUploader(default_cfg);
|
||||
}
|
||||
|
||||
LogInfo("[ClipAction] initialized, pre=" + std::to_string(pre_sec_) +
|
||||
"s post=" + std::to_string(post_sec_) +
|
||||
"s format=" + format_);
|
||||
return true;
|
||||
}
|
||||
|
||||
void ClipAction::Execute(AlarmEvent& event, std::shared_ptr<Frame> frame) {
|
||||
if (!packet_buffer_) {
|
||||
LogError("[ClipAction] packet buffer not set");
|
||||
return;
|
||||
}
|
||||
if (!frame) return;
|
||||
|
||||
const int64_t trigger_pts_ms = frame->pts > 0 ? static_cast<int64_t>(frame->pts / 1000ULL) : 0;
|
||||
if (trigger_pts_ms <= 0) {
|
||||
LogWarn("[ClipAction] invalid trigger pts");
|
||||
return;
|
||||
}
|
||||
|
||||
const int64_t pre_ms = static_cast<int64_t>(std::max(0, pre_sec_)) * 1000LL;
|
||||
const int64_t post_ms = static_cast<int64_t>(std::max(0, post_sec_)) * 1000LL;
|
||||
const int64_t start_pts = std::max<int64_t>(0, trigger_pts_ms - pre_ms);
|
||||
const int64_t end_pts = trigger_pts_ms + post_ms;
|
||||
|
||||
// Best-effort wait for post window to arrive.
|
||||
if (post_sec_ > 0) {
|
||||
using namespace std::chrono;
|
||||
const auto deadline = steady_clock::now() + seconds(post_sec_);
|
||||
while (steady_clock::now() < deadline) {
|
||||
if (packet_buffer_->LatestPtsMs() >= end_pts) break;
|
||||
std::this_thread::sleep_for(milliseconds(20));
|
||||
}
|
||||
}
|
||||
|
||||
auto metas = packet_buffer_->GetPacketsInRange(start_pts, end_pts);
|
||||
if (metas.empty()) {
|
||||
// Fallback: at least try the latest packet.
|
||||
metas = packet_buffer_->GetPacketsInRange(trigger_pts_ms, end_pts);
|
||||
}
|
||||
if (metas.empty()) {
|
||||
LogWarn("[ClipAction] no packets in range");
|
||||
return;
|
||||
}
|
||||
|
||||
std::string url = ProcessClipFromPackets(metas, event);
|
||||
if (!url.empty()) {
|
||||
event.clip_url = url;
|
||||
LogInfo("[ClipAction] clip uploaded: " + url);
|
||||
}
|
||||
}
|
||||
|
||||
std::string ClipAction::ProcessClipFromPackets(const std::vector<std::shared_ptr<EncodedVideoFrameMeta>>& metas,
|
||||
const AlarmEvent& event) {
|
||||
#if HAS_FFMPEG
|
||||
std::shared_ptr<EncodedVideoFrameMeta> first;
|
||||
for (const auto& m : metas) {
|
||||
if (m && m->magic == EncodedVideoFrameMeta::kMagic && m->codec && !m->pkt.data.empty()) {
|
||||
first = m;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!first || !first->codec) return "";
|
||||
|
||||
AVCodecID cid = AV_CODEC_ID_NONE;
|
||||
const auto codec = first->codec->codec;
|
||||
if (codec == VideoCodec::H264) cid = AV_CODEC_ID_H264;
|
||||
else if (codec == VideoCodec::H265) cid = AV_CODEC_ID_HEVC;
|
||||
else {
|
||||
LogError("[ClipAction] unsupported codec");
|
||||
return "";
|
||||
}
|
||||
|
||||
const int width = first->codec->width;
|
||||
const int height = first->codec->height;
|
||||
if (width <= 0 || height <= 0) {
|
||||
LogError("[ClipAction] invalid codec size");
|
||||
return "";
|
||||
}
|
||||
|
||||
std::filesystem::path tmp_dir;
|
||||
try {
|
||||
tmp_dir = std::filesystem::temp_directory_path();
|
||||
} catch (...) {
|
||||
LogWarn("[ClipAction] temp_directory_path failed; falling back to current directory");
|
||||
tmp_dir = ".";
|
||||
}
|
||||
std::filesystem::path tmp_path = CreateTempFilePath(tmp_dir, "clip_" + std::to_string(event.timestamp_ms));
|
||||
|
||||
AVFormatContext* fmt_ctx = nullptr;
|
||||
if (avformat_alloc_output_context2(&fmt_ctx, nullptr, format_.c_str(), tmp_path.string().c_str()) < 0 || !fmt_ctx) {
|
||||
LogError("[ClipAction] failed to create output context");
|
||||
return "";
|
||||
}
|
||||
|
||||
AVStream* stream = avformat_new_stream(fmt_ctx, nullptr);
|
||||
if (!stream) {
|
||||
avformat_free_context(fmt_ctx);
|
||||
return "";
|
||||
}
|
||||
|
||||
const int fps_eff = std::max(1, (first->codec->fps > 0 ? first->codec->fps : fps_));
|
||||
// Use a stable timescale to avoid rounding drift and "fast playback" caused by bad source PTS.
|
||||
stream->time_base = AVRational{1, 90000};
|
||||
stream->avg_frame_rate = AVRational{fps_eff, 1};
|
||||
stream->r_frame_rate = stream->avg_frame_rate;
|
||||
stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
|
||||
stream->codecpar->codec_id = cid;
|
||||
stream->codecpar->width = width;
|
||||
stream->codecpar->height = height;
|
||||
stream->codecpar->format = AV_PIX_FMT_YUV420P;
|
||||
|
||||
// MP4 requires AVCC/HVCC style extradata and length-prefixed samples.
|
||||
std::vector<uint8_t> mp4_extradata;
|
||||
if (!first->codec->extradata.empty()) {
|
||||
const auto& ex = first->codec->extradata;
|
||||
if (cid == AV_CODEC_ID_H264) {
|
||||
if (!ex.empty() && ex[0] == 1) {
|
||||
mp4_extradata = ex; // AVCC
|
||||
} else if (HasAnnexBStartCode(ex.data(), ex.size())) {
|
||||
if (!BuildAvccFromAnnexB(ex, mp4_extradata)) {
|
||||
LogWarn("[ClipAction] failed to build AVCC from AnnexB extradata");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (cid == AV_CODEC_ID_H264 && mp4_extradata.empty()) {
|
||||
// As a fallback, try extracting SPS/PPS from the first keyframe packet.
|
||||
for (const auto& m : metas) {
|
||||
if (!m || m->magic != EncodedVideoFrameMeta::kMagic) continue;
|
||||
if (!m->pkt.key) continue;
|
||||
if (!HasAnnexBStartCode(m->pkt.data.data(), m->pkt.data.size())) continue;
|
||||
if (BuildAvccFromAnnexB(m->pkt.data, mp4_extradata)) break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!mp4_extradata.empty()) {
|
||||
stream->codecpar->extradata_size = static_cast<int>(mp4_extradata.size());
|
||||
stream->codecpar->extradata = static_cast<uint8_t*>(av_mallocz(mp4_extradata.size() + AV_INPUT_BUFFER_PADDING_SIZE));
|
||||
std::memcpy(stream->codecpar->extradata, mp4_extradata.data(), mp4_extradata.size());
|
||||
}
|
||||
|
||||
if (!(fmt_ctx->oformat->flags & AVFMT_NOFILE)) {
|
||||
if (avio_open(&fmt_ctx->pb, tmp_path.string().c_str(), AVIO_FLAG_WRITE) < 0) {
|
||||
avformat_free_context(fmt_ctx);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
if (avformat_write_header(fmt_ctx, nullptr) < 0) {
|
||||
if (!(fmt_ctx->oformat->flags & AVFMT_NOFILE)) avio_closep(&fmt_ctx->pb);
|
||||
avformat_free_context(fmt_ctx);
|
||||
return "";
|
||||
}
|
||||
|
||||
// Start from the first keyframe inside the window to ensure decodability.
|
||||
size_t start_idx = 0;
|
||||
for (size_t i = 0; i < metas.size(); ++i) {
|
||||
if (metas[i] && metas[i]->pkt.key) {
|
||||
start_idx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const int64_t frame_dur = av_rescale_q(1, AVRational{1, fps_eff}, stream->time_base);
|
||||
int64_t frame_idx = 0;
|
||||
|
||||
for (size_t i = start_idx; i < metas.size(); ++i) {
|
||||
const auto& m = metas[i];
|
||||
if (!m || m->magic != EncodedVideoFrameMeta::kMagic) continue;
|
||||
if (!m->codec || m->pkt.data.empty()) continue;
|
||||
if (m->pkt.pts_ms <= 0) continue;
|
||||
|
||||
const int64_t pts = av_rescale_q(frame_idx, AVRational{1, fps_eff}, stream->time_base);
|
||||
|
||||
std::vector<uint8_t> sample = m->pkt.data;
|
||||
if (cid == AV_CODEC_ID_H264 && HasAnnexBStartCode(sample.data(), sample.size())) {
|
||||
sample = AnnexBToLengthPrefixed(sample.data(), sample.size());
|
||||
}
|
||||
if (sample.empty()) continue;
|
||||
|
||||
AVPacket* pkt = av_packet_alloc();
|
||||
if (!pkt) continue;
|
||||
if (av_new_packet(pkt, static_cast<int>(sample.size())) < 0) {
|
||||
av_packet_free(&pkt);
|
||||
continue;
|
||||
}
|
||||
std::memcpy(pkt->data, sample.data(), sample.size());
|
||||
pkt->stream_index = stream->index;
|
||||
pkt->pts = pts;
|
||||
pkt->dts = pts;
|
||||
pkt->duration = frame_dur;
|
||||
if (m->pkt.key) pkt->flags |= AV_PKT_FLAG_KEY;
|
||||
|
||||
(void)av_interleaved_write_frame(fmt_ctx, pkt);
|
||||
av_packet_free(&pkt);
|
||||
|
||||
++frame_idx;
|
||||
}
|
||||
|
||||
av_write_trailer(fmt_ctx);
|
||||
if (!(fmt_ctx->oformat->flags & AVFMT_NOFILE)) avio_closep(&fmt_ctx->pb);
|
||||
avformat_free_context(fmt_ctx);
|
||||
|
||||
std::ifstream file(tmp_path, std::ios::binary | std::ios::ate);
|
||||
if (!file) {
|
||||
LogError("[ClipAction] failed to read temp file");
|
||||
return "";
|
||||
}
|
||||
const size_t file_size = static_cast<size_t>(file.tellg());
|
||||
file.seekg(0);
|
||||
std::vector<uint8_t> file_data(file_size);
|
||||
file.read(reinterpret_cast<char*>(file_data.data()), static_cast<std::streamsize>(file_data.size()));
|
||||
file.close();
|
||||
|
||||
std::string key = GenerateKey(event);
|
||||
auto result = uploader_->Upload(key, file_data.data(), file_data.size(), "video/mp4");
|
||||
|
||||
std::error_code ec;
|
||||
std::filesystem::remove(tmp_path, ec);
|
||||
|
||||
if (result.success) return result.url;
|
||||
LogWarn("[ClipAction] upload failed: " + result.error);
|
||||
return "";
|
||||
#else
|
||||
(void)metas;
|
||||
(void)event;
|
||||
LogError("[ClipAction] FFmpeg not enabled");
|
||||
return "";
|
||||
#endif
|
||||
}
|
||||
|
||||
std::string ClipAction::GenerateKey(const AlarmEvent& event) {
|
||||
auto now = std::chrono::system_clock::now();
|
||||
auto time_t_now = std::chrono::system_clock::to_time_t(now);
|
||||
std::tm tm{};
|
||||
(void)SafeLocalTime(time_t_now, tm);
|
||||
|
||||
std::ostringstream oss;
|
||||
oss << std::put_time(&tm, "%Y%m%d") << "/"
|
||||
<< event.node_id << "_"
|
||||
<< std::put_time(&tm, "%H%M%S") << "_"
|
||||
<< event.frame_id << "." << format_;
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
} // namespace rk3588
|
||||
#include "clip_action.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <ctime>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iomanip>
|
||||
#include <mutex>
|
||||
#include <sstream>
|
||||
#include <thread>
|
||||
|
||||
#if defined(_WIN32)
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#elif defined(__unix__) || defined(__APPLE__)
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#include "utils/logger.h"
|
||||
|
||||
#if defined(RK3588_ENABLE_FFMPEG)
|
||||
extern "C" {
|
||||
#include <libavcodec/avcodec.h>
|
||||
#include <libavformat/avformat.h>
|
||||
#include <libavutil/avutil.h>
|
||||
}
|
||||
#define HAS_FFMPEG 1
|
||||
#else
|
||||
#define HAS_FFMPEG 0
|
||||
#endif
|
||||
|
||||
namespace rk3588 {
|
||||
|
||||
namespace {
|
||||
|
||||
bool SafeLocalTime(std::time_t t, std::tm& out) {
|
||||
#if defined(_WIN32)
|
||||
return localtime_s(&out, &t) == 0;
|
||||
#elif defined(__unix__) || defined(__APPLE__)
|
||||
return localtime_r(&t, &out) != nullptr;
|
||||
#else
|
||||
static std::mutex mu;
|
||||
std::lock_guard<std::mutex> lock(mu);
|
||||
std::tm* p = std::localtime(&t);
|
||||
if (!p) return false;
|
||||
out = *p;
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
static std::filesystem::path CreateTempFilePath(const std::filesystem::path& dir, const std::string& prefix) {
|
||||
std::filesystem::path base = dir;
|
||||
if (base.empty()) base = ".";
|
||||
|
||||
#if defined(_WIN32)
|
||||
wchar_t tmp_dir[MAX_PATH];
|
||||
DWORD n = GetTempPathW(MAX_PATH, tmp_dir);
|
||||
if (n == 0 || n > MAX_PATH) {
|
||||
return base / (prefix + "_tmp");
|
||||
}
|
||||
wchar_t tmp_file[MAX_PATH];
|
||||
if (GetTempFileNameW(tmp_dir, L"clp", 0, tmp_file) == 0) {
|
||||
return base / (prefix + "_tmp");
|
||||
}
|
||||
return std::filesystem::path(tmp_file);
|
||||
|
||||
#elif defined(__unix__) || defined(__APPLE__)
|
||||
std::string tmpl = (base / (prefix + "_XXXXXX")).string();
|
||||
std::vector<char> buf(tmpl.begin(), tmpl.end());
|
||||
buf.push_back('\0');
|
||||
int fd = mkstemp(buf.data());
|
||||
if (fd >= 0) close(fd);
|
||||
if (fd < 0) {
|
||||
return base / (prefix + "_tmp");
|
||||
}
|
||||
return std::filesystem::path(buf.data());
|
||||
|
||||
#else
|
||||
const auto now = std::chrono::high_resolution_clock::now().time_since_epoch().count();
|
||||
return base / (prefix + "_" + std::to_string(static_cast<long long>(now)));
|
||||
#endif
|
||||
}
|
||||
|
||||
static bool HasAnnexBStartCode(const uint8_t* d, size_t n) {
|
||||
if (!d || n < 4) return false;
|
||||
for (size_t i = 0; i + 3 < n; ++i) {
|
||||
if (d[i] == 0 && d[i + 1] == 0 && d[i + 2] == 1) return true;
|
||||
if (i + 4 < n && d[i] == 0 && d[i + 1] == 0 && d[i + 2] == 0 && d[i + 3] == 1) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static size_t FindStartCode(const uint8_t* d, size_t n, size_t from, size_t* sc_len) {
|
||||
for (size_t i = from; i + 3 < n; ++i) {
|
||||
if (d[i] == 0 && d[i + 1] == 0 && d[i + 2] == 1) {
|
||||
if (sc_len) *sc_len = 3;
|
||||
return i;
|
||||
}
|
||||
if (i + 4 < n && d[i] == 0 && d[i + 1] == 0 && d[i + 2] == 0 && d[i + 3] == 1) {
|
||||
if (sc_len) *sc_len = 4;
|
||||
return i;
|
||||
}
|
||||
}
|
||||
if (sc_len) *sc_len = 0;
|
||||
return n;
|
||||
}
|
||||
|
||||
static std::vector<std::vector<uint8_t>> SplitAnnexBNals(const uint8_t* d, size_t n) {
|
||||
std::vector<std::vector<uint8_t>> out;
|
||||
if (!d || n < 4) return out;
|
||||
size_t pos = 0;
|
||||
while (true) {
|
||||
size_t sc_len = 0;
|
||||
size_t sc = FindStartCode(d, n, pos, &sc_len);
|
||||
if (sc >= n) break;
|
||||
size_t nal_start = sc + sc_len;
|
||||
size_t next_sc = FindStartCode(d, n, nal_start, nullptr);
|
||||
size_t nal_end = (next_sc >= n) ? n : next_sc;
|
||||
|
||||
// Trim trailing zeros before next start code.
|
||||
while (nal_end > nal_start && d[nal_end - 1] == 0) --nal_end;
|
||||
|
||||
if (nal_end > nal_start) {
|
||||
out.emplace_back(d + nal_start, d + nal_end);
|
||||
}
|
||||
pos = nal_end;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static bool BuildAvccFromAnnexB(const std::vector<uint8_t>& annexb, std::vector<uint8_t>& avcc_out) {
|
||||
avcc_out.clear();
|
||||
if (annexb.empty()) return false;
|
||||
auto nals = SplitAnnexBNals(annexb.data(), annexb.size());
|
||||
|
||||
const uint8_t* sps = nullptr;
|
||||
size_t sps_len = 0;
|
||||
const uint8_t* pps = nullptr;
|
||||
size_t pps_len = 0;
|
||||
|
||||
for (const auto& nal : nals) {
|
||||
if (nal.empty()) continue;
|
||||
const uint8_t t = nal[0] & 0x1F;
|
||||
if (t == 7 && !sps) {
|
||||
sps = nal.data();
|
||||
sps_len = nal.size();
|
||||
} else if (t == 8 && !pps) {
|
||||
pps = nal.data();
|
||||
pps_len = nal.size();
|
||||
}
|
||||
}
|
||||
|
||||
if (!sps || sps_len < 4 || !pps || pps_len < 1) return false;
|
||||
|
||||
// AVCDecoderConfigurationRecord
|
||||
// https://developer.apple.com/documentation/quicktime-file-format/avcdecoderconfigurationrecord
|
||||
avcc_out.reserve(11 + sps_len + pps_len);
|
||||
avcc_out.push_back(1); // configurationVersion
|
||||
avcc_out.push_back(sps[1]); // AVCProfileIndication
|
||||
avcc_out.push_back(sps[2]); // profile_compatibility
|
||||
avcc_out.push_back(sps[3]); // AVCLevelIndication
|
||||
avcc_out.push_back(0xFF); // 6 bits reserved + lengthSizeMinusOne(3 => 4 bytes)
|
||||
avcc_out.push_back(0xE1); // 3 bits reserved + numOfSequenceParameterSets(1)
|
||||
avcc_out.push_back(static_cast<uint8_t>((sps_len >> 8) & 0xFF));
|
||||
avcc_out.push_back(static_cast<uint8_t>((sps_len) & 0xFF));
|
||||
avcc_out.insert(avcc_out.end(), sps, sps + sps_len);
|
||||
avcc_out.push_back(1); // numOfPictureParameterSets
|
||||
avcc_out.push_back(static_cast<uint8_t>((pps_len >> 8) & 0xFF));
|
||||
avcc_out.push_back(static_cast<uint8_t>((pps_len) & 0xFF));
|
||||
avcc_out.insert(avcc_out.end(), pps, pps + pps_len);
|
||||
return true;
|
||||
}
|
||||
|
||||
static std::vector<uint8_t> AnnexBToLengthPrefixed(const uint8_t* d, size_t n) {
|
||||
std::vector<uint8_t> out;
|
||||
auto nals = SplitAnnexBNals(d, n);
|
||||
// Each NAL becomes [len_be32][nal_bytes]
|
||||
size_t total = 0;
|
||||
for (const auto& nal : nals) total += 4 + nal.size();
|
||||
out.reserve(total);
|
||||
for (const auto& nal : nals) {
|
||||
const uint32_t len = static_cast<uint32_t>(nal.size());
|
||||
out.push_back(static_cast<uint8_t>((len >> 24) & 0xFF));
|
||||
out.push_back(static_cast<uint8_t>((len >> 16) & 0xFF));
|
||||
out.push_back(static_cast<uint8_t>((len >> 8) & 0xFF));
|
||||
out.push_back(static_cast<uint8_t>((len) & 0xFF));
|
||||
out.insert(out.end(), nal.begin(), nal.end());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool ClipAction::Init(const SimpleJson& config) {
|
||||
pre_sec_ = config.ValueOr<int>("pre_sec", 5);
|
||||
post_sec_ = config.ValueOr<int>("post_sec", 10);
|
||||
format_ = config.ValueOr<std::string>("format", "mp4");
|
||||
fps_ = config.ValueOr<int>("fps", 25);
|
||||
use_date_prefix_ = config.ValueOr<bool>("use_date_prefix", true);
|
||||
|
||||
if (const SimpleJson* upload_cfg = config.Find("upload")) {
|
||||
uploader_ = CreateUploader(*upload_cfg);
|
||||
if (!uploader_) {
|
||||
LogError("[ClipAction] failed to create uploader");
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
SimpleJson default_cfg;
|
||||
uploader_ = CreateUploader(default_cfg);
|
||||
}
|
||||
|
||||
LogInfo("[ClipAction] initialized, pre=" + std::to_string(pre_sec_) +
|
||||
"s post=" + std::to_string(post_sec_) +
|
||||
"s format=" + format_);
|
||||
return true;
|
||||
}
|
||||
|
||||
void ClipAction::Execute(AlarmEvent& event, std::shared_ptr<Frame> frame) {
|
||||
if (!packet_buffer_) {
|
||||
LogError("[ClipAction] packet buffer not set");
|
||||
return;
|
||||
}
|
||||
if (!frame) return;
|
||||
|
||||
const int64_t trigger_pts_ms = frame->pts > 0 ? static_cast<int64_t>(frame->pts / 1000ULL) : 0;
|
||||
if (trigger_pts_ms <= 0) {
|
||||
LogWarn("[ClipAction] invalid trigger pts");
|
||||
return;
|
||||
}
|
||||
|
||||
const int64_t pre_ms = static_cast<int64_t>(std::max(0, pre_sec_)) * 1000LL;
|
||||
const int64_t post_ms = static_cast<int64_t>(std::max(0, post_sec_)) * 1000LL;
|
||||
const int64_t start_pts = std::max<int64_t>(0, trigger_pts_ms - pre_ms);
|
||||
const int64_t end_pts = trigger_pts_ms + post_ms;
|
||||
|
||||
// Best-effort wait for post window to arrive.
|
||||
if (post_sec_ > 0) {
|
||||
using namespace std::chrono;
|
||||
const auto deadline = steady_clock::now() + seconds(post_sec_);
|
||||
while (steady_clock::now() < deadline) {
|
||||
if (packet_buffer_->LatestPtsMs() >= end_pts) break;
|
||||
std::this_thread::sleep_for(milliseconds(20));
|
||||
}
|
||||
}
|
||||
|
||||
auto metas = packet_buffer_->GetPacketsInRange(start_pts, end_pts);
|
||||
if (metas.empty()) {
|
||||
// Fallback: at least try the latest packet.
|
||||
metas = packet_buffer_->GetPacketsInRange(trigger_pts_ms, end_pts);
|
||||
}
|
||||
if (metas.empty()) {
|
||||
LogWarn("[ClipAction] no packets in range");
|
||||
return;
|
||||
}
|
||||
|
||||
std::string url = ProcessClipFromPackets(metas, event);
|
||||
if (!url.empty()) {
|
||||
event.clip_url = url;
|
||||
LogInfo("[ClipAction] clip uploaded: " + url);
|
||||
}
|
||||
}
|
||||
|
||||
std::string ClipAction::ProcessClipFromPackets(const std::vector<std::shared_ptr<EncodedVideoFrameMeta>>& metas,
|
||||
const AlarmEvent& event) {
|
||||
#if HAS_FFMPEG
|
||||
std::shared_ptr<EncodedVideoFrameMeta> first;
|
||||
for (const auto& m : metas) {
|
||||
if (m && m->magic == EncodedVideoFrameMeta::kMagic && m->codec && !m->pkt.data.empty()) {
|
||||
first = m;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!first || !first->codec) return "";
|
||||
|
||||
AVCodecID cid = AV_CODEC_ID_NONE;
|
||||
const auto codec = first->codec->codec;
|
||||
if (codec == VideoCodec::H264) cid = AV_CODEC_ID_H264;
|
||||
else if (codec == VideoCodec::H265) cid = AV_CODEC_ID_HEVC;
|
||||
else {
|
||||
LogError("[ClipAction] unsupported codec");
|
||||
return "";
|
||||
}
|
||||
|
||||
const int width = first->codec->width;
|
||||
const int height = first->codec->height;
|
||||
if (width <= 0 || height <= 0) {
|
||||
LogError("[ClipAction] invalid codec size");
|
||||
return "";
|
||||
}
|
||||
|
||||
std::filesystem::path tmp_dir;
|
||||
try {
|
||||
tmp_dir = std::filesystem::temp_directory_path();
|
||||
} catch (...) {
|
||||
LogWarn("[ClipAction] temp_directory_path failed; falling back to current directory");
|
||||
tmp_dir = ".";
|
||||
}
|
||||
std::filesystem::path tmp_path = CreateTempFilePath(tmp_dir, "clip_" + std::to_string(event.timestamp_ms));
|
||||
|
||||
AVFormatContext* fmt_ctx = nullptr;
|
||||
if (avformat_alloc_output_context2(&fmt_ctx, nullptr, format_.c_str(), tmp_path.string().c_str()) < 0 || !fmt_ctx) {
|
||||
LogError("[ClipAction] failed to create output context");
|
||||
return "";
|
||||
}
|
||||
|
||||
AVStream* stream = avformat_new_stream(fmt_ctx, nullptr);
|
||||
if (!stream) {
|
||||
avformat_free_context(fmt_ctx);
|
||||
return "";
|
||||
}
|
||||
|
||||
const int fps_eff = std::max(1, (first->codec->fps > 0 ? first->codec->fps : fps_));
|
||||
// Use a stable timescale to avoid rounding drift and "fast playback" caused by bad source PTS.
|
||||
stream->time_base = AVRational{1, 90000};
|
||||
stream->avg_frame_rate = AVRational{fps_eff, 1};
|
||||
stream->r_frame_rate = stream->avg_frame_rate;
|
||||
stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
|
||||
stream->codecpar->codec_id = cid;
|
||||
stream->codecpar->width = width;
|
||||
stream->codecpar->height = height;
|
||||
stream->codecpar->format = AV_PIX_FMT_YUV420P;
|
||||
|
||||
// MP4 requires AVCC/HVCC style extradata and length-prefixed samples.
|
||||
std::vector<uint8_t> mp4_extradata;
|
||||
if (!first->codec->extradata.empty()) {
|
||||
const auto& ex = first->codec->extradata;
|
||||
if (cid == AV_CODEC_ID_H264) {
|
||||
if (!ex.empty() && ex[0] == 1) {
|
||||
mp4_extradata = ex; // AVCC
|
||||
} else if (HasAnnexBStartCode(ex.data(), ex.size())) {
|
||||
if (!BuildAvccFromAnnexB(ex, mp4_extradata)) {
|
||||
LogWarn("[ClipAction] failed to build AVCC from AnnexB extradata");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (cid == AV_CODEC_ID_H264 && mp4_extradata.empty()) {
|
||||
// As a fallback, try extracting SPS/PPS from the first keyframe packet.
|
||||
for (const auto& m : metas) {
|
||||
if (!m || m->magic != EncodedVideoFrameMeta::kMagic) continue;
|
||||
if (!m->pkt.key) continue;
|
||||
if (!HasAnnexBStartCode(m->pkt.data.data(), m->pkt.data.size())) continue;
|
||||
if (BuildAvccFromAnnexB(m->pkt.data, mp4_extradata)) break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!mp4_extradata.empty()) {
|
||||
stream->codecpar->extradata_size = static_cast<int>(mp4_extradata.size());
|
||||
stream->codecpar->extradata = static_cast<uint8_t*>(av_mallocz(mp4_extradata.size() + AV_INPUT_BUFFER_PADDING_SIZE));
|
||||
std::memcpy(stream->codecpar->extradata, mp4_extradata.data(), mp4_extradata.size());
|
||||
}
|
||||
|
||||
if (!(fmt_ctx->oformat->flags & AVFMT_NOFILE)) {
|
||||
if (avio_open(&fmt_ctx->pb, tmp_path.string().c_str(), AVIO_FLAG_WRITE) < 0) {
|
||||
avformat_free_context(fmt_ctx);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
if (avformat_write_header(fmt_ctx, nullptr) < 0) {
|
||||
if (!(fmt_ctx->oformat->flags & AVFMT_NOFILE)) avio_closep(&fmt_ctx->pb);
|
||||
avformat_free_context(fmt_ctx);
|
||||
return "";
|
||||
}
|
||||
|
||||
// Start from the first keyframe inside the window to ensure decodability.
|
||||
size_t start_idx = 0;
|
||||
for (size_t i = 0; i < metas.size(); ++i) {
|
||||
if (metas[i] && metas[i]->pkt.key) {
|
||||
start_idx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const int64_t frame_dur = av_rescale_q(1, AVRational{1, fps_eff}, stream->time_base);
|
||||
int64_t frame_idx = 0;
|
||||
|
||||
for (size_t i = start_idx; i < metas.size(); ++i) {
|
||||
const auto& m = metas[i];
|
||||
if (!m || m->magic != EncodedVideoFrameMeta::kMagic) continue;
|
||||
if (!m->codec || m->pkt.data.empty()) continue;
|
||||
if (m->pkt.pts_ms <= 0) continue;
|
||||
|
||||
const int64_t pts = av_rescale_q(frame_idx, AVRational{1, fps_eff}, stream->time_base);
|
||||
|
||||
std::vector<uint8_t> sample = m->pkt.data;
|
||||
if (cid == AV_CODEC_ID_H264 && HasAnnexBStartCode(sample.data(), sample.size())) {
|
||||
sample = AnnexBToLengthPrefixed(sample.data(), sample.size());
|
||||
}
|
||||
if (sample.empty()) continue;
|
||||
|
||||
AVPacket* pkt = av_packet_alloc();
|
||||
if (!pkt) continue;
|
||||
if (av_new_packet(pkt, static_cast<int>(sample.size())) < 0) {
|
||||
av_packet_free(&pkt);
|
||||
continue;
|
||||
}
|
||||
std::memcpy(pkt->data, sample.data(), sample.size());
|
||||
pkt->stream_index = stream->index;
|
||||
pkt->pts = pts;
|
||||
pkt->dts = pts;
|
||||
pkt->duration = frame_dur;
|
||||
if (m->pkt.key) pkt->flags |= AV_PKT_FLAG_KEY;
|
||||
|
||||
(void)av_interleaved_write_frame(fmt_ctx, pkt);
|
||||
av_packet_free(&pkt);
|
||||
|
||||
++frame_idx;
|
||||
}
|
||||
|
||||
av_write_trailer(fmt_ctx);
|
||||
if (!(fmt_ctx->oformat->flags & AVFMT_NOFILE)) avio_closep(&fmt_ctx->pb);
|
||||
avformat_free_context(fmt_ctx);
|
||||
|
||||
std::ifstream file(tmp_path, std::ios::binary | std::ios::ate);
|
||||
if (!file) {
|
||||
LogError("[ClipAction] failed to read temp file");
|
||||
return "";
|
||||
}
|
||||
const size_t file_size = static_cast<size_t>(file.tellg());
|
||||
file.seekg(0);
|
||||
std::vector<uint8_t> file_data(file_size);
|
||||
file.read(reinterpret_cast<char*>(file_data.data()), static_cast<std::streamsize>(file_data.size()));
|
||||
file.close();
|
||||
|
||||
std::string key = GenerateKey(event);
|
||||
auto result = uploader_->Upload(key, file_data.data(), file_data.size(), "video/mp4");
|
||||
|
||||
std::error_code ec;
|
||||
std::filesystem::remove(tmp_path, ec);
|
||||
|
||||
if (result.success) return result.url;
|
||||
LogWarn("[ClipAction] upload failed: " + result.error);
|
||||
return "";
|
||||
#else
|
||||
(void)metas;
|
||||
(void)event;
|
||||
LogError("[ClipAction] FFmpeg not enabled");
|
||||
return "";
|
||||
#endif
|
||||
}
|
||||
|
||||
std::string ClipAction::GenerateKey(const AlarmEvent& event) {
|
||||
auto now = std::chrono::system_clock::now();
|
||||
auto time_t_now = std::chrono::system_clock::to_time_t(now);
|
||||
std::tm tm{};
|
||||
(void)SafeLocalTime(time_t_now, tm);
|
||||
|
||||
std::ostringstream oss;
|
||||
if (use_date_prefix_) {
|
||||
oss << std::put_time(&tm, "%Y%m%d") << "/";
|
||||
}
|
||||
oss << event.node_id << "_"
|
||||
<< std::put_time(&tm, "%H%M%S") << "_"
|
||||
<< event.frame_id << "." << format_;
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
} // namespace rk3588
|
||||
|
||||
@ -1,36 +1,37 @@
|
||||
#pragma once
|
||||
|
||||
#include "action_base.h"
|
||||
#include "../uploaders/uploader_base.h"
|
||||
#include "../packet_ring_buffer.h"
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
namespace rk3588 {
|
||||
|
||||
class ClipAction : public IAlarmAction {
|
||||
public:
|
||||
~ClipAction() override = default;
|
||||
bool Init(const SimpleJson& config) override;
|
||||
void Execute(AlarmEvent& event, std::shared_ptr<Frame> frame) override;
|
||||
void Drain() override {}
|
||||
void Stop() override {}
|
||||
std::string Name() const override { return "clip"; }
|
||||
|
||||
void SetPacketBuffer(std::shared_ptr<PacketRingBuffer> buffer) { packet_buffer_ = buffer; }
|
||||
|
||||
private:
|
||||
std::string ProcessClipFromPackets(const std::vector<std::shared_ptr<EncodedVideoFrameMeta>>& metas,
|
||||
const AlarmEvent& event);
|
||||
std::string GenerateKey(const AlarmEvent& event);
|
||||
|
||||
int pre_sec_ = 5;
|
||||
int post_sec_ = 10;
|
||||
std::string format_ = "mp4";
|
||||
int fps_ = 25;
|
||||
std::unique_ptr<IUploader> uploader_;
|
||||
std::shared_ptr<PacketRingBuffer> packet_buffer_;
|
||||
};
|
||||
|
||||
} // namespace rk3588
|
||||
#pragma once
|
||||
|
||||
#include "action_base.h"
|
||||
#include "../uploaders/uploader_base.h"
|
||||
#include "../packet_ring_buffer.h"
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
namespace rk3588 {
|
||||
|
||||
class ClipAction : public IAlarmAction {
|
||||
public:
|
||||
~ClipAction() override = default;
|
||||
bool Init(const SimpleJson& config) override;
|
||||
void Execute(AlarmEvent& event, std::shared_ptr<Frame> frame) override;
|
||||
void Drain() override {}
|
||||
void Stop() override {}
|
||||
std::string Name() const override { return "clip"; }
|
||||
|
||||
void SetPacketBuffer(std::shared_ptr<PacketRingBuffer> buffer) { packet_buffer_ = buffer; }
|
||||
|
||||
private:
|
||||
std::string ProcessClipFromPackets(const std::vector<std::shared_ptr<EncodedVideoFrameMeta>>& metas,
|
||||
const AlarmEvent& event);
|
||||
std::string GenerateKey(const AlarmEvent& event);
|
||||
|
||||
int pre_sec_ = 5;
|
||||
int post_sec_ = 10;
|
||||
std::string format_ = "mp4";
|
||||
int fps_ = 25;
|
||||
bool use_date_prefix_ = true;
|
||||
std::unique_ptr<IUploader> uploader_;
|
||||
std::shared_ptr<PacketRingBuffer> packet_buffer_;
|
||||
};
|
||||
|
||||
} // namespace rk3588
|
||||
|
||||
@ -1,330 +1,333 @@
|
||||
#include "snapshot_action.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
#include <mutex>
|
||||
#include <sstream>
|
||||
|
||||
#include "utils/dma_alloc.h"
|
||||
#include "utils/logger.h"
|
||||
|
||||
#if defined(RK3588_ENABLE_FFMPEG)
|
||||
extern "C" {
|
||||
#include <libavcodec/avcodec.h>
|
||||
#include <libavutil/imgutils.h>
|
||||
#include <libavutil/pixfmt.h>
|
||||
}
|
||||
#define HAS_FFMPEG 1
|
||||
#else
|
||||
#define HAS_FFMPEG 0
|
||||
#endif
|
||||
|
||||
namespace rk3588 {
|
||||
|
||||
namespace {
|
||||
|
||||
bool SafeLocalTime(std::time_t t, std::tm& out) {
|
||||
#if defined(_WIN32)
|
||||
return localtime_s(&out, &t) == 0;
|
||||
#elif defined(__unix__) || defined(__APPLE__)
|
||||
return localtime_r(&t, &out) != nullptr;
|
||||
#else
|
||||
static std::mutex mu;
|
||||
std::lock_guard<std::mutex> lock(mu);
|
||||
std::tm* p = std::localtime(&t);
|
||||
if (!p) return false;
|
||||
out = *p;
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
inline uint8_t ClipU8(int v) {
|
||||
if (v < 0) return 0;
|
||||
if (v > 255) return 255;
|
||||
return static_cast<uint8_t>(v);
|
||||
}
|
||||
|
||||
inline const uint8_t* PlanePtr(const Frame& f, int idx) {
|
||||
if (idx < 0 || idx >= f.plane_count) return nullptr;
|
||||
if (f.planes[idx].data) return f.planes[idx].data;
|
||||
if (!f.data) return nullptr;
|
||||
const int off = f.planes[idx].offset;
|
||||
if (off < 0) return nullptr;
|
||||
return f.data + static_cast<size_t>(off);
|
||||
}
|
||||
|
||||
inline int PlaneStride(const Frame& f, int idx, int fallback) {
|
||||
if (idx >= 0 && idx < f.plane_count && f.planes[idx].stride > 0) return f.planes[idx].stride;
|
||||
if (f.stride > 0) return f.stride;
|
||||
return fallback;
|
||||
}
|
||||
|
||||
#if HAS_FFMPEG
|
||||
bool FillYuv420pFromFrame(const Frame& src, AVFrame* dst) {
|
||||
if (!dst) return false;
|
||||
if (dst->format != AV_PIX_FMT_YUV420P && dst->format != AV_PIX_FMT_YUVJ420P) return false;
|
||||
if (src.width <= 0 || src.height <= 0) return false;
|
||||
|
||||
const int w = src.width;
|
||||
const int h = src.height;
|
||||
uint8_t* y = dst->data[0];
|
||||
uint8_t* u = dst->data[1];
|
||||
uint8_t* v = dst->data[2];
|
||||
const int y_stride = dst->linesize[0];
|
||||
const int u_stride = dst->linesize[1];
|
||||
const int v_stride = dst->linesize[2];
|
||||
if (!y || !u || !v || y_stride <= 0 || u_stride <= 0 || v_stride <= 0) return false;
|
||||
|
||||
if (src.format == PixelFormat::YUV420) {
|
||||
const uint8_t* sy = PlanePtr(src, 0);
|
||||
const uint8_t* su = PlanePtr(src, 1);
|
||||
const uint8_t* sv = PlanePtr(src, 2);
|
||||
if (!sy || !su || !sv) return false;
|
||||
const int sy_stride = PlaneStride(src, 0, w);
|
||||
const int su_stride = PlaneStride(src, 1, w / 2);
|
||||
const int sv_stride = PlaneStride(src, 2, w / 2);
|
||||
|
||||
for (int row = 0; row < h; ++row) {
|
||||
std::memcpy(y + row * y_stride, sy + row * sy_stride, static_cast<size_t>(w));
|
||||
}
|
||||
const int uv_h = h / 2;
|
||||
const int uv_w = w / 2;
|
||||
for (int row = 0; row < uv_h; ++row) {
|
||||
std::memcpy(u + row * u_stride, su + row * su_stride, static_cast<size_t>(uv_w));
|
||||
std::memcpy(v + row * v_stride, sv + row * sv_stride, static_cast<size_t>(uv_w));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
if (src.format == PixelFormat::NV12) {
|
||||
const uint8_t* sy = PlanePtr(src, 0);
|
||||
const uint8_t* suv = PlanePtr(src, 1);
|
||||
if (!sy) return false;
|
||||
|
||||
const int sy_stride = PlaneStride(src, 0, w);
|
||||
const int suv_stride = PlaneStride(src, 1, w);
|
||||
if (!suv) {
|
||||
// Fallback: packed NV12 layout
|
||||
if (!src.data) return false;
|
||||
suv = src.data + static_cast<size_t>(sy_stride) * static_cast<size_t>(h);
|
||||
}
|
||||
|
||||
for (int row = 0; row < h; ++row) {
|
||||
std::memcpy(y + row * y_stride, sy + row * sy_stride, static_cast<size_t>(w));
|
||||
}
|
||||
|
||||
const int uv_h = h / 2;
|
||||
const int uv_w = w / 2;
|
||||
for (int row = 0; row < uv_h; ++row) {
|
||||
const uint8_t* src_uv = suv + row * suv_stride;
|
||||
uint8_t* dst_u = u + row * u_stride;
|
||||
uint8_t* dst_v = v + row * v_stride;
|
||||
for (int col = 0; col < uv_w; ++col) {
|
||||
dst_u[col] = src_uv[col * 2 + 0];
|
||||
dst_v[col] = src_uv[col * 2 + 1];
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
if (src.format == PixelFormat::RGB || src.format == PixelFormat::BGR) {
|
||||
const bool is_bgr = (src.format == PixelFormat::BGR);
|
||||
const uint8_t* s = PlanePtr(src, 0);
|
||||
if (!s) s = src.data;
|
||||
if (!s) return false;
|
||||
const int s_stride = PlaneStride(src, 0, w * 3);
|
||||
|
||||
const int uv_w = w / 2;
|
||||
const int uv_h = h / 2;
|
||||
|
||||
for (int row = 0; row < h; row += 2) {
|
||||
const uint8_t* row0 = s + row * s_stride;
|
||||
const uint8_t* row1 = (row + 1 < h) ? (s + (row + 1) * s_stride) : row0;
|
||||
uint8_t* y0 = y + row * y_stride;
|
||||
uint8_t* y1 = (row + 1 < h) ? (y + (row + 1) * y_stride) : y0;
|
||||
const int uv_row = row / 2;
|
||||
uint8_t* uu = (uv_row < uv_h) ? (u + uv_row * u_stride) : nullptr;
|
||||
uint8_t* vv = (uv_row < uv_h) ? (v + uv_row * v_stride) : nullptr;
|
||||
|
||||
for (int col = 0; col < w; col += 2) {
|
||||
int u_sum = 0;
|
||||
int v_sum = 0;
|
||||
int samples = 0;
|
||||
|
||||
auto sample = [&](const uint8_t* p, uint8_t* ydst) {
|
||||
const int b = is_bgr ? p[0] : p[2];
|
||||
const int g = p[1];
|
||||
const int r = is_bgr ? p[2] : p[0];
|
||||
const int yy = (77 * r + 150 * g + 29 * b + 128) >> 8;
|
||||
*ydst = ClipU8(yy);
|
||||
u_sum += (-43 * r - 84 * g + 127 * b);
|
||||
v_sum += (127 * r - 106 * g - 21 * b);
|
||||
samples += 1;
|
||||
};
|
||||
|
||||
const uint8_t* p00 = row0 + col * 3;
|
||||
const uint8_t* p01 = (col + 1 < w) ? (row0 + (col + 1) * 3) : p00;
|
||||
const uint8_t* p10 = row1 + col * 3;
|
||||
const uint8_t* p11 = (col + 1 < w) ? (row1 + (col + 1) * 3) : p10;
|
||||
|
||||
sample(p00, &y0[col]);
|
||||
if (col + 1 < w) sample(p01, &y0[col + 1]);
|
||||
if (row + 1 < h) {
|
||||
sample(p10, &y1[col]);
|
||||
if (col + 1 < w) sample(p11, &y1[col + 1]);
|
||||
} else {
|
||||
// If no second row, count row0 twice for chroma.
|
||||
sample(p10, &y1[col]);
|
||||
if (col + 1 < w) sample(p11, &y1[col + 1]);
|
||||
}
|
||||
|
||||
const int denom = samples > 0 ? samples : 1;
|
||||
const int u_val = ((u_sum / denom) + 128 * 256 + 128) >> 8;
|
||||
const int v_val = ((v_sum / denom) + 128 * 256 + 128) >> 8;
|
||||
const int uv_col = col / 2;
|
||||
if (uu && vv && uv_col >= 0 && uv_col < uv_w) {
|
||||
uu[uv_col] = ClipU8(u_val);
|
||||
vv[uv_col] = ClipU8(v_val);
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace
|
||||
|
||||
bool SnapshotAction::Init(const SimpleJson& config) {
|
||||
format_ = config.ValueOr<std::string>("format", "jpg");
|
||||
quality_ = config.ValueOr<int>("quality", 85);
|
||||
|
||||
if (const SimpleJson* upload_cfg = config.Find("upload")) {
|
||||
uploader_ = CreateUploader(*upload_cfg);
|
||||
if (!uploader_) {
|
||||
LogError("[SnapshotAction] failed to create uploader");
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
// Default to local storage
|
||||
SimpleJson default_cfg;
|
||||
uploader_ = CreateUploader(default_cfg);
|
||||
}
|
||||
|
||||
LogInfo("[SnapshotAction] initialized, format=" + format_ + " quality=" + std::to_string(quality_));
|
||||
return true;
|
||||
}
|
||||
|
||||
void SnapshotAction::Execute(AlarmEvent& event, std::shared_ptr<Frame> frame) {
|
||||
if (!frame || !frame->data) {
|
||||
LogWarn("[SnapshotAction] no frame data");
|
||||
return;
|
||||
}
|
||||
|
||||
if (frame->DmaFd() >= 0) frame->SyncStart();
|
||||
auto jpeg_data = EncodeJpeg(frame);
|
||||
if (frame->DmaFd() >= 0) frame->SyncEnd();
|
||||
if (jpeg_data.empty()) {
|
||||
LogWarn("[SnapshotAction] failed to encode JPEG");
|
||||
return;
|
||||
}
|
||||
|
||||
std::string key = GenerateKey(event);
|
||||
auto result = uploader_->Upload(key, jpeg_data.data(), jpeg_data.size(), "image/jpeg");
|
||||
|
||||
if (result.success) {
|
||||
event.snapshot_url = result.url;
|
||||
LogInfo("[SnapshotAction] uploaded: " + result.url);
|
||||
} else {
|
||||
LogWarn("[SnapshotAction] upload failed: " + result.error);
|
||||
}
|
||||
}
|
||||
|
||||
std::string SnapshotAction::GenerateKey(const AlarmEvent& event) {
|
||||
auto now = std::chrono::system_clock::now();
|
||||
auto time_t_now = std::chrono::system_clock::to_time_t(now);
|
||||
std::tm tm{};
|
||||
(void)SafeLocalTime(time_t_now, tm);
|
||||
|
||||
std::ostringstream oss;
|
||||
oss << std::put_time(&tm, "%Y%m%d") << "/"
|
||||
<< event.node_id << "_"
|
||||
<< std::put_time(&tm, "%H%M%S") << "_"
|
||||
<< event.frame_id << "." << format_;
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
std::vector<uint8_t> SnapshotAction::EncodeJpeg(const std::shared_ptr<Frame>& frame) {
|
||||
std::vector<uint8_t> output;
|
||||
|
||||
#if HAS_FFMPEG
|
||||
const AVCodec* codec = avcodec_find_encoder(AV_CODEC_ID_MJPEG);
|
||||
if (!codec) {
|
||||
LogError("[SnapshotAction] MJPEG encoder not found");
|
||||
return output;
|
||||
}
|
||||
|
||||
AVCodecContext* ctx = avcodec_alloc_context3(codec);
|
||||
if (!ctx) return output;
|
||||
|
||||
ctx->width = frame->width;
|
||||
ctx->height = frame->height;
|
||||
ctx->time_base = AVRational{1, 25};
|
||||
// Some embedded FFmpeg builds expose MJPEG encoder that only accepts YUVJ*.
|
||||
ctx->pix_fmt = AV_PIX_FMT_YUVJ420P;
|
||||
ctx->color_range = AVCOL_RANGE_JPEG;
|
||||
|
||||
// Set quality (1-31, lower is better for MJPEG)
|
||||
int q = 31 - (quality_ * 30 / 100);
|
||||
ctx->qmin = q;
|
||||
ctx->qmax = q;
|
||||
|
||||
if (avcodec_open2(ctx, codec, nullptr) < 0) {
|
||||
avcodec_free_context(&ctx);
|
||||
return output;
|
||||
}
|
||||
|
||||
AVFrame* av_frame = av_frame_alloc();
|
||||
av_frame->width = frame->width;
|
||||
av_frame->height = frame->height;
|
||||
av_frame->format = AV_PIX_FMT_YUVJ420P;
|
||||
av_frame->color_range = AVCOL_RANGE_JPEG;
|
||||
|
||||
if (av_frame_get_buffer(av_frame, 32) < 0) {
|
||||
av_frame_free(&av_frame);
|
||||
avcodec_free_context(&ctx);
|
||||
return output;
|
||||
}
|
||||
|
||||
if (!FillYuv420pFromFrame(*frame, av_frame)) {
|
||||
av_frame_free(&av_frame);
|
||||
avcodec_free_context(&ctx);
|
||||
return output;
|
||||
}
|
||||
|
||||
av_frame->pts = 0;
|
||||
|
||||
AVPacket* pkt = av_packet_alloc();
|
||||
if (avcodec_send_frame(ctx, av_frame) == 0) {
|
||||
if (avcodec_receive_packet(ctx, pkt) == 0) {
|
||||
output.assign(pkt->data, pkt->data + pkt->size);
|
||||
}
|
||||
}
|
||||
|
||||
av_packet_free(&pkt);
|
||||
av_frame_free(&av_frame);
|
||||
avcodec_free_context(&ctx);
|
||||
#else
|
||||
// Stub: just create a minimal valid JPEG header for testing
|
||||
LogError("[SnapshotAction] FFmpeg not enabled, cannot encode JPEG");
|
||||
#endif
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
} // namespace rk3588
|
||||
#include "snapshot_action.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
#include <mutex>
|
||||
#include <sstream>
|
||||
|
||||
#include "utils/dma_alloc.h"
|
||||
#include "utils/logger.h"
|
||||
|
||||
#if defined(RK3588_ENABLE_FFMPEG)
|
||||
extern "C" {
|
||||
#include <libavcodec/avcodec.h>
|
||||
#include <libavutil/imgutils.h>
|
||||
#include <libavutil/pixfmt.h>
|
||||
}
|
||||
#define HAS_FFMPEG 1
|
||||
#else
|
||||
#define HAS_FFMPEG 0
|
||||
#endif
|
||||
|
||||
namespace rk3588 {
|
||||
|
||||
namespace {
|
||||
|
||||
bool SafeLocalTime(std::time_t t, std::tm& out) {
|
||||
#if defined(_WIN32)
|
||||
return localtime_s(&out, &t) == 0;
|
||||
#elif defined(__unix__) || defined(__APPLE__)
|
||||
return localtime_r(&t, &out) != nullptr;
|
||||
#else
|
||||
static std::mutex mu;
|
||||
std::lock_guard<std::mutex> lock(mu);
|
||||
std::tm* p = std::localtime(&t);
|
||||
if (!p) return false;
|
||||
out = *p;
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
inline uint8_t ClipU8(int v) {
|
||||
if (v < 0) return 0;
|
||||
if (v > 255) return 255;
|
||||
return static_cast<uint8_t>(v);
|
||||
}
|
||||
|
||||
inline const uint8_t* PlanePtr(const Frame& f, int idx) {
|
||||
if (idx < 0 || idx >= f.plane_count) return nullptr;
|
||||
if (f.planes[idx].data) return f.planes[idx].data;
|
||||
if (!f.data) return nullptr;
|
||||
const int off = f.planes[idx].offset;
|
||||
if (off < 0) return nullptr;
|
||||
return f.data + static_cast<size_t>(off);
|
||||
}
|
||||
|
||||
inline int PlaneStride(const Frame& f, int idx, int fallback) {
|
||||
if (idx >= 0 && idx < f.plane_count && f.planes[idx].stride > 0) return f.planes[idx].stride;
|
||||
if (f.stride > 0) return f.stride;
|
||||
return fallback;
|
||||
}
|
||||
|
||||
#if HAS_FFMPEG
|
||||
bool FillYuv420pFromFrame(const Frame& src, AVFrame* dst) {
|
||||
if (!dst) return false;
|
||||
if (dst->format != AV_PIX_FMT_YUV420P && dst->format != AV_PIX_FMT_YUVJ420P) return false;
|
||||
if (src.width <= 0 || src.height <= 0) return false;
|
||||
|
||||
const int w = src.width;
|
||||
const int h = src.height;
|
||||
uint8_t* y = dst->data[0];
|
||||
uint8_t* u = dst->data[1];
|
||||
uint8_t* v = dst->data[2];
|
||||
const int y_stride = dst->linesize[0];
|
||||
const int u_stride = dst->linesize[1];
|
||||
const int v_stride = dst->linesize[2];
|
||||
if (!y || !u || !v || y_stride <= 0 || u_stride <= 0 || v_stride <= 0) return false;
|
||||
|
||||
if (src.format == PixelFormat::YUV420) {
|
||||
const uint8_t* sy = PlanePtr(src, 0);
|
||||
const uint8_t* su = PlanePtr(src, 1);
|
||||
const uint8_t* sv = PlanePtr(src, 2);
|
||||
if (!sy || !su || !sv) return false;
|
||||
const int sy_stride = PlaneStride(src, 0, w);
|
||||
const int su_stride = PlaneStride(src, 1, w / 2);
|
||||
const int sv_stride = PlaneStride(src, 2, w / 2);
|
||||
|
||||
for (int row = 0; row < h; ++row) {
|
||||
std::memcpy(y + row * y_stride, sy + row * sy_stride, static_cast<size_t>(w));
|
||||
}
|
||||
const int uv_h = h / 2;
|
||||
const int uv_w = w / 2;
|
||||
for (int row = 0; row < uv_h; ++row) {
|
||||
std::memcpy(u + row * u_stride, su + row * su_stride, static_cast<size_t>(uv_w));
|
||||
std::memcpy(v + row * v_stride, sv + row * sv_stride, static_cast<size_t>(uv_w));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
if (src.format == PixelFormat::NV12) {
|
||||
const uint8_t* sy = PlanePtr(src, 0);
|
||||
const uint8_t* suv = PlanePtr(src, 1);
|
||||
if (!sy) return false;
|
||||
|
||||
const int sy_stride = PlaneStride(src, 0, w);
|
||||
const int suv_stride = PlaneStride(src, 1, w);
|
||||
if (!suv) {
|
||||
// Fallback: packed NV12 layout
|
||||
if (!src.data) return false;
|
||||
suv = src.data + static_cast<size_t>(sy_stride) * static_cast<size_t>(h);
|
||||
}
|
||||
|
||||
for (int row = 0; row < h; ++row) {
|
||||
std::memcpy(y + row * y_stride, sy + row * sy_stride, static_cast<size_t>(w));
|
||||
}
|
||||
|
||||
const int uv_h = h / 2;
|
||||
const int uv_w = w / 2;
|
||||
for (int row = 0; row < uv_h; ++row) {
|
||||
const uint8_t* src_uv = suv + row * suv_stride;
|
||||
uint8_t* dst_u = u + row * u_stride;
|
||||
uint8_t* dst_v = v + row * v_stride;
|
||||
for (int col = 0; col < uv_w; ++col) {
|
||||
dst_u[col] = src_uv[col * 2 + 0];
|
||||
dst_v[col] = src_uv[col * 2 + 1];
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
if (src.format == PixelFormat::RGB || src.format == PixelFormat::BGR) {
|
||||
const bool is_bgr = (src.format == PixelFormat::BGR);
|
||||
const uint8_t* s = PlanePtr(src, 0);
|
||||
if (!s) s = src.data;
|
||||
if (!s) return false;
|
||||
const int s_stride = PlaneStride(src, 0, w * 3);
|
||||
|
||||
const int uv_w = w / 2;
|
||||
const int uv_h = h / 2;
|
||||
|
||||
for (int row = 0; row < h; row += 2) {
|
||||
const uint8_t* row0 = s + row * s_stride;
|
||||
const uint8_t* row1 = (row + 1 < h) ? (s + (row + 1) * s_stride) : row0;
|
||||
uint8_t* y0 = y + row * y_stride;
|
||||
uint8_t* y1 = (row + 1 < h) ? (y + (row + 1) * y_stride) : y0;
|
||||
const int uv_row = row / 2;
|
||||
uint8_t* uu = (uv_row < uv_h) ? (u + uv_row * u_stride) : nullptr;
|
||||
uint8_t* vv = (uv_row < uv_h) ? (v + uv_row * v_stride) : nullptr;
|
||||
|
||||
for (int col = 0; col < w; col += 2) {
|
||||
int u_sum = 0;
|
||||
int v_sum = 0;
|
||||
int samples = 0;
|
||||
|
||||
auto sample = [&](const uint8_t* p, uint8_t* ydst) {
|
||||
const int b = is_bgr ? p[0] : p[2];
|
||||
const int g = p[1];
|
||||
const int r = is_bgr ? p[2] : p[0];
|
||||
const int yy = (77 * r + 150 * g + 29 * b + 128) >> 8;
|
||||
*ydst = ClipU8(yy);
|
||||
u_sum += (-43 * r - 84 * g + 127 * b);
|
||||
v_sum += (127 * r - 106 * g - 21 * b);
|
||||
samples += 1;
|
||||
};
|
||||
|
||||
const uint8_t* p00 = row0 + col * 3;
|
||||
const uint8_t* p01 = (col + 1 < w) ? (row0 + (col + 1) * 3) : p00;
|
||||
const uint8_t* p10 = row1 + col * 3;
|
||||
const uint8_t* p11 = (col + 1 < w) ? (row1 + (col + 1) * 3) : p10;
|
||||
|
||||
sample(p00, &y0[col]);
|
||||
if (col + 1 < w) sample(p01, &y0[col + 1]);
|
||||
if (row + 1 < h) {
|
||||
sample(p10, &y1[col]);
|
||||
if (col + 1 < w) sample(p11, &y1[col + 1]);
|
||||
} else {
|
||||
// If no second row, count row0 twice for chroma.
|
||||
sample(p10, &y1[col]);
|
||||
if (col + 1 < w) sample(p11, &y1[col + 1]);
|
||||
}
|
||||
|
||||
const int denom = samples > 0 ? samples : 1;
|
||||
const int u_val = ((u_sum / denom) + 128 * 256 + 128) >> 8;
|
||||
const int v_val = ((v_sum / denom) + 128 * 256 + 128) >> 8;
|
||||
const int uv_col = col / 2;
|
||||
if (uu && vv && uv_col >= 0 && uv_col < uv_w) {
|
||||
uu[uv_col] = ClipU8(u_val);
|
||||
vv[uv_col] = ClipU8(v_val);
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace
|
||||
|
||||
bool SnapshotAction::Init(const SimpleJson& config) {
|
||||
format_ = config.ValueOr<std::string>("format", "jpg");
|
||||
quality_ = config.ValueOr<int>("quality", 85);
|
||||
use_date_prefix_ = config.ValueOr<bool>("use_date_prefix", true);
|
||||
|
||||
if (const SimpleJson* upload_cfg = config.Find("upload")) {
|
||||
uploader_ = CreateUploader(*upload_cfg);
|
||||
if (!uploader_) {
|
||||
LogError("[SnapshotAction] failed to create uploader");
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
// Default to local storage
|
||||
SimpleJson default_cfg;
|
||||
uploader_ = CreateUploader(default_cfg);
|
||||
}
|
||||
|
||||
LogInfo("[SnapshotAction] initialized, format=" + format_ + " quality=" + std::to_string(quality_));
|
||||
return true;
|
||||
}
|
||||
|
||||
void SnapshotAction::Execute(AlarmEvent& event, std::shared_ptr<Frame> frame) {
|
||||
if (!frame || !frame->data) {
|
||||
LogWarn("[SnapshotAction] no frame data");
|
||||
return;
|
||||
}
|
||||
|
||||
if (frame->DmaFd() >= 0) frame->SyncStart();
|
||||
auto jpeg_data = EncodeJpeg(frame);
|
||||
if (frame->DmaFd() >= 0) frame->SyncEnd();
|
||||
if (jpeg_data.empty()) {
|
||||
LogWarn("[SnapshotAction] failed to encode JPEG");
|
||||
return;
|
||||
}
|
||||
|
||||
std::string key = GenerateKey(event);
|
||||
auto result = uploader_->Upload(key, jpeg_data.data(), jpeg_data.size(), "image/jpeg");
|
||||
|
||||
if (result.success) {
|
||||
event.snapshot_url = result.url;
|
||||
LogInfo("[SnapshotAction] uploaded: " + result.url);
|
||||
} else {
|
||||
LogWarn("[SnapshotAction] upload failed: " + result.error);
|
||||
}
|
||||
}
|
||||
|
||||
std::string SnapshotAction::GenerateKey(const AlarmEvent& event) {
|
||||
auto now = std::chrono::system_clock::now();
|
||||
auto time_t_now = std::chrono::system_clock::to_time_t(now);
|
||||
std::tm tm{};
|
||||
(void)SafeLocalTime(time_t_now, tm);
|
||||
|
||||
std::ostringstream oss;
|
||||
if (use_date_prefix_) {
|
||||
oss << std::put_time(&tm, "%Y%m%d") << "/";
|
||||
}
|
||||
oss << event.node_id << "_"
|
||||
<< std::put_time(&tm, "%H%M%S") << "_"
|
||||
<< event.frame_id << "." << format_;
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
std::vector<uint8_t> SnapshotAction::EncodeJpeg(const std::shared_ptr<Frame>& frame) {
|
||||
std::vector<uint8_t> output;
|
||||
|
||||
#if HAS_FFMPEG
|
||||
const AVCodec* codec = avcodec_find_encoder(AV_CODEC_ID_MJPEG);
|
||||
if (!codec) {
|
||||
LogError("[SnapshotAction] MJPEG encoder not found");
|
||||
return output;
|
||||
}
|
||||
|
||||
AVCodecContext* ctx = avcodec_alloc_context3(codec);
|
||||
if (!ctx) return output;
|
||||
|
||||
ctx->width = frame->width;
|
||||
ctx->height = frame->height;
|
||||
ctx->time_base = AVRational{1, 25};
|
||||
// Some embedded FFmpeg builds expose MJPEG encoder that only accepts YUVJ*.
|
||||
ctx->pix_fmt = AV_PIX_FMT_YUVJ420P;
|
||||
ctx->color_range = AVCOL_RANGE_JPEG;
|
||||
|
||||
// Set quality (1-31, lower is better for MJPEG)
|
||||
int q = 31 - (quality_ * 30 / 100);
|
||||
ctx->qmin = q;
|
||||
ctx->qmax = q;
|
||||
|
||||
if (avcodec_open2(ctx, codec, nullptr) < 0) {
|
||||
avcodec_free_context(&ctx);
|
||||
return output;
|
||||
}
|
||||
|
||||
AVFrame* av_frame = av_frame_alloc();
|
||||
av_frame->width = frame->width;
|
||||
av_frame->height = frame->height;
|
||||
av_frame->format = AV_PIX_FMT_YUVJ420P;
|
||||
av_frame->color_range = AVCOL_RANGE_JPEG;
|
||||
|
||||
if (av_frame_get_buffer(av_frame, 32) < 0) {
|
||||
av_frame_free(&av_frame);
|
||||
avcodec_free_context(&ctx);
|
||||
return output;
|
||||
}
|
||||
|
||||
if (!FillYuv420pFromFrame(*frame, av_frame)) {
|
||||
av_frame_free(&av_frame);
|
||||
avcodec_free_context(&ctx);
|
||||
return output;
|
||||
}
|
||||
|
||||
av_frame->pts = 0;
|
||||
|
||||
AVPacket* pkt = av_packet_alloc();
|
||||
if (avcodec_send_frame(ctx, av_frame) == 0) {
|
||||
if (avcodec_receive_packet(ctx, pkt) == 0) {
|
||||
output.assign(pkt->data, pkt->data + pkt->size);
|
||||
}
|
||||
}
|
||||
|
||||
av_packet_free(&pkt);
|
||||
av_frame_free(&av_frame);
|
||||
avcodec_free_context(&ctx);
|
||||
#else
|
||||
// Stub: just create a minimal valid JPEG header for testing
|
||||
LogError("[SnapshotAction] FFmpeg not enabled, cannot encode JPEG");
|
||||
#endif
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
} // namespace rk3588
|
||||
|
||||
@ -1,25 +1,26 @@
|
||||
#pragma once
|
||||
|
||||
#include "action_base.h"
|
||||
#include "../uploaders/uploader_base.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace rk3588 {
|
||||
|
||||
class SnapshotAction : public IAlarmAction {
|
||||
public:
|
||||
bool Init(const SimpleJson& config) override;
|
||||
void Execute(AlarmEvent& event, std::shared_ptr<Frame> frame) override;
|
||||
std::string Name() const override { return "snapshot"; }
|
||||
|
||||
private:
|
||||
std::vector<uint8_t> EncodeJpeg(const std::shared_ptr<Frame>& frame);
|
||||
std::string GenerateKey(const AlarmEvent& event);
|
||||
|
||||
std::string format_ = "jpg";
|
||||
int quality_ = 85;
|
||||
std::unique_ptr<IUploader> uploader_;
|
||||
};
|
||||
|
||||
} // namespace rk3588
|
||||
#pragma once
|
||||
|
||||
#include "action_base.h"
|
||||
#include "../uploaders/uploader_base.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace rk3588 {
|
||||
|
||||
class SnapshotAction : public IAlarmAction {
|
||||
public:
|
||||
bool Init(const SimpleJson& config) override;
|
||||
void Execute(AlarmEvent& event, std::shared_ptr<Frame> frame) override;
|
||||
std::string Name() const override { return "snapshot"; }
|
||||
|
||||
private:
|
||||
std::vector<uint8_t> EncodeJpeg(const std::shared_ptr<Frame>& frame);
|
||||
std::string GenerateKey(const AlarmEvent& event);
|
||||
|
||||
std::string format_ = "jpg";
|
||||
int quality_ = 85;
|
||||
bool use_date_prefix_ = true;
|
||||
std::unique_ptr<IUploader> uploader_;
|
||||
};
|
||||
|
||||
} // namespace rk3588
|
||||
|
||||
@ -329,7 +329,7 @@ public:
|
||||
NodeStatus Process(FramePtr frame) override {
|
||||
static int process_count = 0;
|
||||
if (id_.find("post") != std::string::npos && process_count++ % 30 == 0) {
|
||||
LogInfo("[preprocess] post_cam1 Process called, frame=" + std::to_string(frame ? 1 : 0));
|
||||
|
||||
}
|
||||
if (!frame) return NodeStatus::DROP;
|
||||
if (!image_processor_) return NodeStatus::ERROR;
|
||||
|
||||
@ -806,10 +806,7 @@ public:
|
||||
}
|
||||
|
||||
NodeStatus Process(FramePtr frame) override {
|
||||
static int process_count = 0;
|
||||
if (process_count++ % 30 == 0) {
|
||||
LogInfo("[publish] Process called, frame=" + std::to_string(frame ? 1 : 0));
|
||||
}
|
||||
|
||||
if (!frame) {
|
||||
LogWarn("[publish] received null frame");
|
||||
return NodeStatus::DROP;
|
||||
@ -829,33 +826,13 @@ public:
|
||||
#endif
|
||||
}
|
||||
|
||||
static int frame_count = 0;
|
||||
if (frame_count++ % 30 == 0) {
|
||||
int no_boots = 0;
|
||||
if (frame->det) {
|
||||
for (const auto& d : frame->det->items) {
|
||||
if (d.cls_id == 10) no_boots++;
|
||||
}
|
||||
}
|
||||
LogInfo("[publish] pushing frame to downstream, dets=" +
|
||||
std::to_string(frame->det ? frame->det->items.size() : 0) +
|
||||
" no_boots=" + std::to_string(no_boots) +
|
||||
" queues=" + std::to_string(output_queues_.size()));
|
||||
}
|
||||
|
||||
PushToDownstream(frame);
|
||||
return NodeStatus::OK;
|
||||
}
|
||||
|
||||
private:
|
||||
void PushToDownstream(const FramePtr& frame) {
|
||||
if (output_queues_.empty()) {
|
||||
static int log_count = 0;
|
||||
if (log_count++ % 100 == 0) {
|
||||
LogInfo("[publish] no output queues");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (output_queues_.empty()) return;
|
||||
for (auto& q : output_queues_) {
|
||||
if (q) q->Push(frame);
|
||||
}
|
||||
|
||||
@ -19,6 +19,12 @@ void PrintVersion() {
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
// Force line buffering for real-time log output
|
||||
setvbuf(stdout, nullptr, _IOLBF, 0);
|
||||
setvbuf(stderr, nullptr, _IOLBF, 0);
|
||||
setenv("PYTHONUNBUFFERED", "1", 1);
|
||||
setenv("STL_BUFFER_MODE", "line", 1);
|
||||
|
||||
std::string config_path = "configs/sample_cam1.json";
|
||||
bool show_version = false;
|
||||
|
||||
|
||||
BIN
transform/RetinaFace_mobile320.onnx
Normal file
BIN
transform/RetinaFace_mobile320.onnx
Normal file
Binary file not shown.
BIN
transform/calib/calib.jpg
Normal file
BIN
transform/calib/calib.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
BIN
transform/calib/calibration.jpg
Normal file
BIN
transform/calib/calibration.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
294
transform/calibrate_camera.py
Normal file
294
transform/calibrate_camera.py
Normal file
@ -0,0 +1,294 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
相机标定工具 - 为车间人脸识别系统生成ROI和三分区参数
|
||||
|
||||
Usage:
|
||||
python calibrate_camera.py --height 5.0 --pitch 45 --report
|
||||
python calibrate_camera.py --height 4.5 --pitch 40 --focal-estimate 2000 -o cam.json
|
||||
|
||||
Output:
|
||||
生成可直接复制到配置文件的params参数
|
||||
"""
|
||||
|
||||
import json
|
||||
import argparse
|
||||
import numpy as np
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Tuple, Dict
|
||||
import sys
|
||||
|
||||
|
||||
@dataclass
|
||||
class CameraParams:
|
||||
"""相机参数"""
|
||||
height: float # 安装高度(m)
|
||||
pitch_deg: float # 俯仰角(°)
|
||||
focal_px: float # 像素焦距(px)
|
||||
img_w: int # 图像宽度
|
||||
img_h: int # 图像高度
|
||||
|
||||
|
||||
class CameraCalibrator:
|
||||
"""相机标定器 - 计算ROI和三分区参数"""
|
||||
|
||||
def __init__(self, p: CameraParams):
|
||||
self.p = p
|
||||
self.cy = p.img_h // 2
|
||||
self.cx = p.img_w // 2
|
||||
self.theta = np.radians(p.pitch_deg)
|
||||
self._build_lut()
|
||||
|
||||
def _build_lut(self):
|
||||
"""预计算距离查找表 LUT[y] = distance(m)"""
|
||||
self.lut = np.zeros(self.p.img_h, dtype=np.float32)
|
||||
for y in range(self.p.img_h):
|
||||
dy = y - self.cy
|
||||
angle = self.theta + np.arctan2(dy, self.p.focal_px)
|
||||
if abs(angle) > 1e-6 and np.tan(angle) > 0:
|
||||
self.lut[y] = self.p.height / np.tan(angle)
|
||||
else:
|
||||
self.lut[y] = np.inf
|
||||
|
||||
def pixel_to_distance(self, y: int) -> float:
|
||||
"""像素y坐标 -> 距离(m)"""
|
||||
if 0 <= y < self.p.img_h:
|
||||
return float(self.lut[y])
|
||||
return np.inf
|
||||
|
||||
def distance_to_pixel(self, d: float) -> int:
|
||||
"""距离(m) -> 像素y坐标"""
|
||||
if d <= 0:
|
||||
return self.cy
|
||||
angle = np.arctan2(self.p.height, d)
|
||||
offset = self.p.focal_px * np.tan(angle - self.theta)
|
||||
return int(np.clip(self.cy + offset, 0, self.p.img_h - 1))
|
||||
|
||||
def calculate_roi(self, min_d: float = 3.0, max_d: float = 9.0,
|
||||
margin: int = 20) -> Dict:
|
||||
"""计算ROI裁剪区域
|
||||
|
||||
Args:
|
||||
min_d: 最近检测距离（米）
|
||||
max_d: 最远检测距离（米）
|
||||
margin: 边界余量（像素）
|
||||
"""
|
||||
# 注意：距离越远，y坐标越大（画面下方）
|
||||
y_min = self.distance_to_pixel(max_d) # 远距在下
|
||||
y_max = self.distance_to_pixel(min_d) # 近距在上
|
||||
|
||||
# 添加边界余量
|
||||
y_min = max(0, y_min - margin)
|
||||
y_max = min(self.p.img_h, y_max + margin)
|
||||
|
||||
saving = 1 - (y_max - y_min) / self.p.img_h
|
||||
|
||||
return {
|
||||
"crop": {
|
||||
"x": 0,
|
||||
"y": int(y_min),
|
||||
"w": self.p.img_w,
|
||||
"h": int(y_max - y_min)
|
||||
},
|
||||
"saving_percent": round(saving * 100, 1),
|
||||
"y_min": int(y_min),
|
||||
"y_max": int(y_max)
|
||||
}
|
||||
|
||||
def get_zone_boundaries(self, boundaries_m: List[float]) -> List[int]:
|
||||
"""获取分区边界像素坐标
|
||||
|
||||
Args:
|
||||
boundaries_m: 分界线距离列表，如[5.0, 7.0]
|
||||
"""
|
||||
return [self.distance_to_pixel(d) for d in boundaries_m]
|
||||
|
||||
def estimate_face_size(self, distance: float, real_width: float = 0.16) -> float:
|
||||
"""估算给定距离的人脸像素大小"""
|
||||
if distance <= 0:
|
||||
return 0
|
||||
return self.p.focal_px * real_width / distance
|
||||
|
||||
def generate_config(self, min_d: float = 3.0, max_d: float = 9.0,
|
||||
boundaries_m: List[float] = None) -> Dict:
|
||||
"""生成配置文件
|
||||
|
||||
Returns:
|
||||
包含params和meta的配置字典
|
||||
"""
|
||||
if boundaries_m is None:
|
||||
boundaries_m = [5.0, 7.0] # 默认5米和7米分界线
|
||||
|
||||
roi = self.calculate_roi(min_d, max_d)
|
||||
boundaries_y = self.get_zone_boundaries(boundaries_m)
|
||||
|
||||
# 生成可直接复制到instances params的配置
|
||||
config = {
|
||||
"params": {
|
||||
"camera_pitch": self.p.pitch_deg,
|
||||
"roi_enabled": True,
|
||||
"roi_y": roi["crop"]["y"],
|
||||
"roi_h": roi["crop"]["h"],
|
||||
"zones_enabled": True,
|
||||
"zone_boundary_5m": boundaries_y[0],
|
||||
"zone_boundary_7m": boundaries_y[1],
|
||||
# 320模型缩放因子
|
||||
"zone_scale_near": 1.0,
|
||||
"zone_scale_mid": 1.3,
|
||||
"zone_scale_far": 1.8
|
||||
},
|
||||
"meta": {
|
||||
"height": self.p.height,
|
||||
"pitch": self.p.pitch_deg,
|
||||
"focal_px": self.p.focal_px,
|
||||
"image_size": [self.p.img_w, self.p.img_h],
|
||||
"roi_saving": roi["saving_percent"],
|
||||
"zone_boundaries_m": boundaries_m,
|
||||
"zone_boundaries_y": boundaries_y,
|
||||
"detection_range": [min_d, max_d]
|
||||
}
|
||||
}
|
||||
|
||||
return config
|
||||
|
||||
def print_report(self, min_d: float = 3.0, max_d: float = 9.0,
|
||||
boundaries_m: List[float] = None):
|
||||
"""打印标定报告"""
|
||||
if boundaries_m is None:
|
||||
boundaries_m = [5.0, 7.0]
|
||||
|
||||
roi = self.calculate_roi(min_d, max_d)
|
||||
boundaries_y = self.get_zone_boundaries(boundaries_m)
|
||||
|
||||
print("=" * 70)
|
||||
print("相机标定报告")
|
||||
print("=" * 70)
|
||||
print(f"\n【相机参数】")
|
||||
print(f" 安装高度 H: {self.p.height}m")
|
||||
print(f" 俯仰角 θ: {self.p.pitch_deg}°")
|
||||
print(f" 像素焦距 f: {self.p.focal_px}px")
|
||||
print(f" 图像尺寸: {self.p.img_w}x{self.p.img_h}")
|
||||
|
||||
print(f"\n【ROI配置】(检测范围 {min_d}-{max_d}m)")
|
||||
print(f" 裁剪区域: y={roi['crop']['y']}, h={roi['crop']['h']}")
|
||||
print(f" 算力节省: {roi['saving_percent']}%")
|
||||
|
||||
print(f"\n【三分区配置】(RetinaFace_320)")
|
||||
print(f" 分界线y坐标: 5m={boundaries_y[0]}px, 7m={boundaries_y[1]}px")
|
||||
# y坐标：画面上方（y小）= 远距离，画面下方（y大）= 近距离
|
||||
by5, by7 = boundaries_y[0], boundaries_y[1]
|
||||
if by5 > by7:
|
||||
# 5米分界线在下方（y大），7米在上方（y小）
|
||||
print(f" 画面分布: 上方(y小)=远距离, 下方(y大)=近距离")
|
||||
print(f" 近区(3-5m): y>{by5}, scale=1.0x (画面下方)")
|
||||
print(f" 中区(5-7m): {by7}<y≤{by5}, scale=1.3x (画面中部)")
|
||||
print(f" 远区(7-9m): y≤{by7}, scale=1.8x (画面上方)")
|
||||
else:
|
||||
print(f" 近区(3-5m): y<{by5}, scale=1.0x")
|
||||
print(f" 中区(5-7m): {by5}≤y<{by7}, scale=1.3x")
|
||||
print(f" 远区(7-9m): y≥{by7}, scale=1.8x")
|
||||
|
||||
print(f"\n【距离-像素-人脸大小映射】")
|
||||
print(f" {'距离':>6} | {'像素y':>6} | {'原始人脸':>10} | {'320输入':>9} | {'区域':>6}")
|
||||
print(f" {'-'*60}")
|
||||
|
||||
for d in [3, 4, 5, 6, 7, 8, 9]:
|
||||
y = self.distance_to_pixel(d)
|
||||
face_orig = self.estimate_face_size(d)
|
||||
|
||||
# 确定区域和320输入大小
|
||||
zone_idx = 0
|
||||
if d >= boundaries_m[1]:
|
||||
zone_idx = 2
|
||||
elif d >= boundaries_m[0]:
|
||||
zone_idx = 1
|
||||
|
||||
scales = [1.0, 1.3, 1.8]
|
||||
face_320 = face_orig * scales[zone_idx]
|
||||
zone_name = ["近", "中", "远"][zone_idx]
|
||||
|
||||
print(f" {d:>6.0f}m | {y:>6} | {face_orig:>9.0f}px | {face_320:>9.0f}px | {zone_name:>6}")
|
||||
|
||||
print(f"\n【320模型说明】")
|
||||
print(f" - 320最佳检测范围：25-35px（占320输入的8-11%）")
|
||||
print(f" - 8-9米检测率可能降至70%，如不满足可升级到640模型")
|
||||
print(f" - 升级只需改：model_path + scales=[0.7,1.0,1.4]")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='相机标定工具 - 生成ROI和三分区参数',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
示例:
|
||||
# 基础标定（推荐）
|
||||
python calibrate_camera.py --height 5.0 --pitch 45 --report
|
||||
|
||||
# 自定义参数并保存
|
||||
python calibrate_camera.py --height 4.5 --pitch 40 --focal-estimate 2000 -o cam.json --report
|
||||
|
||||
# 调整分区边界
|
||||
python calibrate_camera.py --height 5.0 --zones 4.5 6.5 --report
|
||||
"""
|
||||
)
|
||||
parser.add_argument('--height', type=float, required=True,
|
||||
help='相机安装高度（米），如 5.0')
|
||||
parser.add_argument('--pitch', type=float, default=45,
|
||||
help='俯仰角（度），默认45')
|
||||
parser.add_argument('--focal-estimate', type=float, default=2200,
|
||||
help='像素焦距估算值，默认2200（2.5K相机4-6mm镜头）')
|
||||
parser.add_argument('--image-size', type=int, nargs=2,
|
||||
default=[2560, 1440],
|
||||
help='图像尺寸 宽 高，默认 2560 1440')
|
||||
parser.add_argument('--range', type=float, nargs=2,
|
||||
default=[3.0, 9.0],
|
||||
help='检测范围 最小距离 最大距离，默认 3.0 9.0')
|
||||
parser.add_argument('--zones', type=float, nargs=2,
|
||||
default=[5.0, 7.0],
|
||||
help='分区边界距离（米），默认 5.0 7.0（形成3-5,5-7,7-9三区）')
|
||||
parser.add_argument('-o', '--output', default='camera_calib.json',
|
||||
help='输出文件路径，默认 camera_calib.json')
|
||||
parser.add_argument('--report', action='store_true',
|
||||
help='打印详细报告')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 创建标定器
|
||||
params = CameraParams(
|
||||
height=args.height,
|
||||
pitch_deg=args.pitch,
|
||||
focal_px=args.focal_estimate,
|
||||
img_w=args.image_size[0],
|
||||
img_h=args.image_size[1]
|
||||
)
|
||||
|
||||
calib = CameraCalibrator(params)
|
||||
config = calib.generate_config(
|
||||
args.range[0],
|
||||
args.range[1],
|
||||
list(args.zones)
|
||||
)
|
||||
|
||||
# 保存配置
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
json.dump(config, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"[OK] 配置已保存: {args.output}")
|
||||
|
||||
# 打印报告
|
||||
if args.report:
|
||||
calib.print_report(args.range[0], args.range[1], list(args.zones))
|
||||
print(f"\n【可直接复制到配置文件的 params】")
|
||||
print(json.dumps(config["params"], indent=2, ensure_ascii=False))
|
||||
else:
|
||||
# 简洁输出关键参数
|
||||
print(f"\n关键参数:")
|
||||
print(f" roi_y: {config['params']['roi_y']}")
|
||||
print(f" roi_h: {config['params']['roi_h']}")
|
||||
print(f" zone_boundary_5m: {config['params']['zone_boundary_5m']}")
|
||||
print(f" zone_boundary_7m: {config['params']['zone_boundary_7m']}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
76
transform/convert_retinaface.py
Normal file
76
transform/convert_retinaface.py
Normal file
@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Convert RetinaFace ONNX to RKNN for RK3588"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Try to import rknn
|
||||
try:
|
||||
from rknn.api import RKNN
|
||||
except ImportError:
|
||||
print("Error: rknn-toolkit2 not installed")
|
||||
print("Install with: pip install rknn-toolkit2")
|
||||
sys.exit(1)
|
||||
|
||||
ONNX_MODEL = 'RetinaFace_mobile320.onnx'
|
||||
RKNN_MODEL = 'RetinaFace_mobile320.rknn'
|
||||
|
||||
def convert():
|
||||
print(f"Converting {ONNX_MODEL} to {RKNN_MODEL}...")
|
||||
|
||||
# Create RKNN object
|
||||
rknn = RKNN(verbose=True)
|
||||
|
||||
# Pre-process config
|
||||
print("Configuring model...")
|
||||
rknn.config(
|
||||
target_platform='rk3588',
|
||||
mean_values=[[0, 0, 0]], # No mean subtraction
|
||||
std_values=[[255, 255, 255]], # Normalize to 0-1
|
||||
quantized_dtype='w8a8',
|
||||
optimization_level=2
|
||||
)
|
||||
|
||||
# Load ONNX model (auto-detect inputs/outputs)
|
||||
print("Loading ONNX model...")
|
||||
ret = rknn.load_onnx(model=ONNX_MODEL)
|
||||
if ret != 0:
|
||||
print("Failed to load ONNX model")
|
||||
return False
|
||||
|
||||
# Build RKNN model
|
||||
print("Building RKNN model (this may take a while)...")
|
||||
ret = rknn.build(
|
||||
do_quantization=True,
|
||||
dataset='./dataset.txt' # Need calibration dataset
|
||||
)
|
||||
if ret != 0:
|
||||
print("Failed to build RKNN model")
|
||||
return False
|
||||
|
||||
# Export RKNN model
|
||||
print(f"Exporting to {RKNN_MODEL}...")
|
||||
ret = rknn.export_rknn(RKNN_MODEL)
|
||||
if ret != 0:
|
||||
print("Failed to export RKNN model")
|
||||
return False
|
||||
|
||||
print("Conversion successful!")
|
||||
rknn.release()
|
||||
return True
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Create a dummy dataset file for calibration
|
||||
with open('dataset.txt', 'w') as f:
|
||||
f.write('calibration.jpg\n')
|
||||
|
||||
if not os.path.exists('calibration.jpg'):
|
||||
print("Warning: calibration.jpg not found, creating dummy...")
|
||||
# Create a dummy image for calibration
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
dummy = np.random.randint(0, 255, (320, 320, 3), dtype=np.uint8)
|
||||
Image.fromarray(dummy).save('calibration.jpg')
|
||||
|
||||
success = convert()
|
||||
sys.exit(0 if success else 1)
|
||||
87
transform/convert_scrfd.sh
Executable file
87
transform/convert_scrfd.sh
Executable file
@ -0,0 +1,87 @@
|
||||
#!/bin/bash
|
||||
# SCRFD PTH -> ONNX -> RKNN 转换脚本
|
||||
|
||||
cd /home/orangepi/apps/OrangePi3588Media/utils
|
||||
|
||||
echo "=== 步骤1: 导出 ONNX ==="
|
||||
python3 << 'PYEOF'
|
||||
import torch
|
||||
import sys
|
||||
sys.path.insert(0, '/home/orangepi/apps/OrangePi3588Media')
|
||||
|
||||
# 加载 SCRFD 模型
|
||||
print("加载 SCRFD 模型...")
|
||||
from insightface.model_zoo.scrfd import SCRFD
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
# 创建模型实例
|
||||
model = SCRFD(model_file='scrfd_2.5g_kps.pth')
|
||||
model.eval()
|
||||
|
||||
# 检查是否有 CUDA
|
||||
if torch.cuda.is_available():
|
||||
model = model.cuda()
|
||||
device = 'cuda'
|
||||
else:
|
||||
device = 'cpu'
|
||||
|
||||
print(f"使用设备: {device}")
|
||||
|
||||
# 创建 dummy input (640x640)
|
||||
dummy_input = torch.randn(1, 3, 640, 640)
|
||||
if device == 'cuda':
|
||||
dummy_input = dummy_input.cuda()
|
||||
|
||||
# 导出 ONNX
|
||||
print("导出 ONNX...")
|
||||
torch.onnx.export(
|
||||
model,
|
||||
dummy_input,
|
||||
'scrfd_2.5g_kps.onnx',
|
||||
input_names=['input'],
|
||||
output_names=['score_8', 'score_16', 'score_32', 'bbox_8', 'bbox_16', 'bbox_32', 'kps_8', 'kps_16', 'kps_32'],
|
||||
dynamic_axes={'input': {0: 'batch', 2: 'height', 3: 'width'}},
|
||||
opset_version=11,
|
||||
do_constant_folding=True
|
||||
)
|
||||
print('ONNX 导出成功!')
|
||||
PYEOF
|
||||
|
||||
echo ""
|
||||
echo "=== 步骤2: 简化 ONNX ==="
|
||||
python3 -m onnxsim scrfd_2.5g_kps.onnx scrfd_2.5g_kps_sim.onnx
|
||||
|
||||
echo ""
|
||||
echo "=== 步骤3: 转换为 RKNN ==="
|
||||
cd /home/orangepi/apps/OrangePi3588Media/models
|
||||
|
||||
python3 << 'PYEOF'
|
||||
from rknn.api import RKNN
|
||||
|
||||
print("初始化 RKNN...")
|
||||
rknn = RKNN(verbose=False)
|
||||
|
||||
# 配置 - SCRFD 使用标准预处理
|
||||
rknn.config(
|
||||
target_platform='rk3588',
|
||||
mean_values=[[127.5, 127.5, 127.5]], # 归一化到 -1~1
|
||||
std_values=[[128.0, 128.0, 128.0]],
|
||||
quantized_dtype='w8a8',
|
||||
)
|
||||
|
||||
print("加载简化后的 ONNX...")
|
||||
rknn.load_onnx(model='../utils/scrfd_2.5g_kps_sim.onnx')
|
||||
|
||||
print("构建 RKNN 模型...")
|
||||
rknn.build(do_quantization=True, dataset='./dataset.txt')
|
||||
|
||||
print("导出 RKNN...")
|
||||
rknn.export_rknn('scrfd_2.5g_kps.rknn')
|
||||
print("SCRFD 640 RKNN 模型转换完成!")
|
||||
rknn.release()
|
||||
PYEOF
|
||||
|
||||
echo ""
|
||||
echo "=== 完成 ==="
|
||||
ls -lh /home/orangepi/apps/OrangePi3588Media/models/scrfd_2.5g_kps.rknn
|
||||
30
transform/convert_simple.py
Normal file
30
transform/convert_simple.py
Normal file
@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env python3
|
||||
"""ONNX to RKNN converter for RetinaFace"""
|
||||
|
||||
from rknn.api import RKNN
|
||||
|
||||
# Create RKNN
|
||||
rknn = RKNN(verbose=False)
|
||||
|
||||
# Config
|
||||
rknn.config(
|
||||
target_platform='rk3588',
|
||||
mean_values=[[0, 0, 0]], # No mean subtraction
|
||||
std_values=[[1, 1, 1]], # No normalization (already done in code)
|
||||
quantized_dtype='w8a8', # INT8 quantization
|
||||
)
|
||||
|
||||
# Load ONNX
|
||||
print("Loading ONNX...")
|
||||
rknn.load_onnx(model='RetinaFace_mobile320.onnx')
|
||||
|
||||
# Build
|
||||
print("Building RKNN...")
|
||||
rknn.build(do_quantization=True)
|
||||
|
||||
# Export
|
||||
print("Exporting...")
|
||||
rknn.export_rknn('RetinaFace_mobile320_new.rknn')
|
||||
print("Done! Output: RetinaFace_mobile320_new.rknn")
|
||||
|
||||
rknn.release()
|
||||
@ -1,10 +1 @@
|
||||
models/calib/img_0.npy
|
||||
models/calib/img_1.npy
|
||||
models/calib/img_2.npy
|
||||
models/calib/img_3.npy
|
||||
models/calib/img_4.npy
|
||||
models/calib/img_5.npy
|
||||
models/calib/img_6.npy
|
||||
models/calib/img_7.npy
|
||||
models/calib/img_8.npy
|
||||
models/calib/img_9.npy
|
||||
calib.jpg
|
||||
|
||||
BIN
transform/scrfd_500m_640.onnx
Normal file
BIN
transform/scrfd_500m_640.onnx
Normal file
Binary file not shown.
BIN
transform/test_face.jpg
Normal file
BIN
transform/test_face.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 3.3 KiB |
BIN
transform/test_input.jpg
Normal file
BIN
transform/test_input.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 53 KiB |
43
transform/test_model.py
Normal file
43
transform/test_model.py
Normal file
@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Test RetinaFace RKNN model directly"""
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from rknn.api import RKNN
|
||||
|
||||
# Create dummy test image (simulating a face region)
|
||||
test_img = np.random.randint(0, 255, (320, 320, 3), dtype=np.uint8)
|
||||
# Add a rectangle to simulate face
|
||||
test_img[100:220, 100:220] = 200 # lighter region
|
||||
|
||||
# Save test image
|
||||
Image.fromarray(test_img).save('test_input.jpg')
|
||||
print("Created test image")
|
||||
|
||||
# Load and test RKNN model
|
||||
rknn = RKNN(verbose=False)
|
||||
print("Loading RKNN model...")
|
||||
ret = rknn.load_rknn('RetinaFace_mobile320.rknn')
|
||||
if ret != 0:
|
||||
print("Failed to load model")
|
||||
exit(1)
|
||||
|
||||
print("Initializing runtime...")
|
||||
ret = rknn.init_runtime(target='rk3588')
|
||||
if ret != 0:
|
||||
print("Failed to init runtime")
|
||||
exit(1)
|
||||
|
||||
# Prepare input (NHWC -> NCHW)
|
||||
input_data = np.expand_dims(test_img, 0) # Add batch dimension
|
||||
print(f"Input shape: {input_data.shape}, dtype: {input_data.dtype}")
|
||||
|
||||
# Inference
|
||||
print("Running inference...")
|
||||
outputs = rknn.inference(inputs=[input_data])
|
||||
print(f"Output count: {len(outputs)}")
|
||||
for i, out in enumerate(outputs):
|
||||
print(f"Output {i}: shape={out.shape}, min={out.min():.4f}, max={out.max():.4f}")
|
||||
|
||||
rknn.release()
|
||||
print("Done!")
|
||||
@ -58,6 +58,7 @@
|
||||
|
||||
<script>
|
||||
const streams = [
|
||||
{ name: 'SCRFD Face Detection', url: '/hls/scrfd/index.m3u8' },
|
||||
{ name: 'Cam 1', url: '/hls/cam1/index.m3u8' },
|
||||
{ name: 'Cam 2', url: '/hls/cam2/index.m3u8' },
|
||||
{ name: 'Cam 3', url: '/hls/cam3/index.m3u8' },
|
||||
|
||||
Loading…
Reference in New Issue
Block a user