Doni/OrangePi3588Media
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复yolov8的FP16转换错误，增加压测脚本

Browse Source
This commit is contained in:
haotian 2026-02-26 20:28:33 +08:00
parent f8baebf6ea
commit 870f0ac624
22 changed files with 1366 additions and 1112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
best.onnx → best.onnx.data
View File

Binary file not shown.

BIN
best_cxn.onnx
View File

Binary file not shown.

BIN
check0_base_optimize.onnx
View File

Binary file not shown.

BIN
check1_fold_constant.onnx
View File

Binary file not shown.

BIN
check2_correct_ops.onnx
View File

Binary file not shown.

BIN
check3_fuse_ops.onnx
View File

Binary file not shown.

1
configs/sample_cam2.json.last_good.json
View File

File diff suppressed because one or more lines are too long

1
configs/sample_cam3.json.last_good.json
View File

File diff suppressed because one or more lines are too long

0
configs/sample_cam3.json → configs/sample_cam3_v5.json
View File

370
configs/sample_cam4_best.json Normal file
View File

@ -0,0 +1,370 @@
{
"queue": {
"size": 8,
"strategy": "drop_oldest"
},
"graphs": [
{
"name": "cam1_sample_full_pipeline",
"nodes": [
{
"id": "in_cam1",
"type": "input_rtsp",
"role": "source",
"enable": true,
"url": "rtsp://10.0.0.49:8554/cam",
"fps": 30,
"width": 1280,
"height": 720,
"use_mpp": true,
"use_ffmpeg": false,
"force_tcp": true,
"reconnect_sec": 5,
"reconnect_backoff_max_sec": 30
},
{
"id": "pre_cam1",
"type": "preprocess",
"role": "filter",
"enable": true,
"dst_w": 640,
"dst_h": 640,
"dst_format": "rgb",
"dst_packed": true,
"keep_ratio": false,
"rga_gate": "cam1_sample_full_pipeline",
"use_rga": true
},
{
"id": "yolo_cam1",
"type": "ai_yolo",
"role": "filter",
"enable": true,
"infer_fps": 10,
"model_path": "./models/best-640.rknn",
"model_version": "v8",
"num_classes": 3,
"conf": 0.35,
"nms": 0.45,
"class_filter": []
},
{
"id": "face_det_cam1",
"type": "ai_face_det",
"role": "filter",
"enable": true,
"model_path": "./models/RetinaFace_mobile320.rknn",
"conf": 0.7,
"nms": 0.4,
"max_faces": 10,
"output_landmarks": true,
"input_format": "rgb"
},
{
"id": "face_recog_cam1",
"type": "ai_face_recog",
"role": "filter",
"enable": true,
"model_path": "./models/mobilefacenet_arcface.rknn",
"align": true,
"emit_embedding": false,
"max_faces": 10,
"input_format": "rgb",
"input_dtype": "uint8",
"threshold": {
"accept": 0.45,
"margin": 0.05
},
"gallery": {
"backend": "sqlite",
"path": "./models/face_gallery.db",
"load_on_start": true,
"expected_dim": 512,
"dtype": "auto"
}
},
{
"id": "trk_cam1",
"type": "tracker",
"role": "filter",
"enable": true,
"mode": "bytetrack_lite",
"per_class": true,
"state_key": "cam1_sample_full_pipeline",
"track_classes": [
0,
1,
2
],
"ignore_classes": [],
"allowed_models": [
"yolov5",
"yolov8"
],
"high_th": 0.5,
"low_th": 0.1,
"iou_th": 0.3,
"max_age_ms": 1500,
"min_hits": 2,
"max_tracks": 128,
"debug": {
"stats": false,
"stats_interval": 200
}
},
{
"id": "pre_face_cam1",
"type": "preprocess",
"role": "filter",
"enable": true,
"dst_w": 0,
"dst_h": 0,
"dst_format": "rgb",
"dst_packed": true,
"keep_ratio": false,
"rga_gate": "cam1_sample_full_pipeline",
"use_rga": true
},
{
"id": "osd_cam1",
"type": "osd",
"role": "filter",
"enable": true,
"draw_bbox": true,
"draw_text": true,
"draw_face_det": false,
"draw_face_bbox": false,
"line_width": 2,
"font_scale": 1,
"use_rga_bbox": false,
"labels": []
},
{
"id": "post_cam1",
"type": "preprocess",
"role": "filter",
"enable": true,
"dst_w": 1280,
"dst_h": 720,
"dst_format": "nv12",
"keep_ratio": false,
"rga_gate": "cam1_sample_full_pipeline",
"use_rga": true
},
{
"id": "pub_cam1",
"type": "publish",
"role": "filter",
"enable": true,
"codec": "h264",
"fps": 30,
"gop": 60,
"bitrate_kbps": 2000,
"use_mpp": true,
"use_ffmpeg_mux": true,
"outputs": [
{
"proto": "hls",
"path": "./web/hls/cam1/index.m3u8",
"segment_sec": 2
}
]
},
{
"id": "alarm_cam1",
"type": "alarm",
"role": "sink",
"enable": true,
"eval_fps": 10,
"labels": [],
"rules": [
{
"name": "object_in_view",
"class_ids": [
0,
1,
2
],
"roi": {
"x": 0.0,
"y": 0.0,
"w": 1.0,
"h": 1.0
},
"min_score": 0.4,
"min_box_area_ratio": 0.02,
"require_track_id": true,
"min_duration_ms": 1500,
"min_hits": 3,
"hit_window_ms": 1500,
"cooldown_ms": 5000,
"per_track_cooldown_ms": 5000
}
],
"face_rules": [],
"actions": {
"log": {
"enable": true,
"level": "info"
},
"snapshot": {
"enable": true,
"format": "jpg",
"quality": 85,
"upload": {
"type": "minio",
"endpoint": "http://10.0.0.49:9000",
"bucket": "myminio",
"region": "us-east-1",
"access_key": "minioadmin",
"secret_key": "minioadmin"
}
},
"clip": {
"enable": true,
"pre_sec": 5,
"post_sec": 10,
"format": "mp4",
"fps": 30,
"upload": {
"type": "minio",
"endpoint": "http://10.0.0.49:9000",
"bucket": "myminio",
"region": "us-east-1",
"access_key": "minioadmin",
"secret_key": "minioadmin"
}
},
"http": {
"enable": false,
"url": "http://127.0.0.1:8080/api/alarm",
"timeout_ms": 3000,
"include_media_url": true,
"method": "POST"
},
"external_api": {
"enable": true,
"getTokenUrl": "http://10.0.0.49:8080/api/getToken",
"putMessageUrl": "http://10.0.0.49:8080/api/putMessage",
"tenantCode": "32",
"channelNo": "${vod_channelNo}",
"timeout_ms": 3000,
"include_media_url": true,
"token_header": "X-Access-Token",
"token_json_path": "responseBody.token",
"token_cache_sec": 1200
}
}
},
{
"id": "alarm_face_cam1",
"type": "alarm",
"role": "sink",
"enable": true,
"eval_fps": 5,
"labels": [],
"rules": [],
"face_rules": [
{
"name": "unknown_face",
"type": "unknown",
"cooldown_ms": 7000,
"min_sim": 0.35,
"min_hits": 2,
"hit_window_ms": 1500,
"min_face_area_ratio": 0.01,
"min_face_aspect": 0.6,
"max_face_aspect": 1.6
},
{
"name": "known_person",
"type": "person",
"cooldown_ms": 7000,
"min_sim": 0.6,
"min_hits": 2,
"hit_window_ms": 1500,
"min_face_area_ratio": 0.01,
"min_face_aspect": 0.6,
"max_face_aspect": 1.6
}
],
"actions": {
"log": {
"enable": false,
"level": "info"
},
"snapshot": {
"enable": true,
"format": "jpg",
"quality": 85,
"upload": {
"type": "minio",
"endpoint": "http://10.0.0.49:9000",
"bucket": "myminio",
"region": "us-east-1",
"access_key": "minioadmin",
"secret_key": "minioadmin"
}
},
"clip": {
"enable": false
},
"http": {
"enable": true,
"url": "http://127.0.0.1:8080/api/alarm",
"timeout_ms": 3000,
"include_media_url": true,
"method": "POST"
}
}
}
],
"edges": [
[
"in_cam1",
"pre_cam1"
],
[
"in_cam1",
"pre_face_cam1"
],
[
"pre_cam1",
"yolo_cam1"
],
[
"yolo_cam1",
"trk_cam1"
],
[
"trk_cam1",
"osd_cam1"
],
[
"osd_cam1",
"post_cam1"
],
[
"post_cam1",
"pub_cam1"
],
[
"pub_cam1",
"alarm_cam1"
],
[
"pre_face_cam1",
"face_det_cam1"
],
[
"face_det_cam1",
"face_recog_cam1"
],
[
"face_recog_cam1",
"alarm_face_cam1"
]
]
}
]
}

281
configs/sample_cam5_v8.json Normal file
View File

@ -0,0 +1,281 @@
{
"queue": { "size": 8, "strategy": "drop_oldest" },
"graphs": [
{
"name": "cam1_sample_full_pipeline",
"nodes": [
{
"id": "in_cam1",
"type": "input_rtsp",
"role": "source",
"enable": true,
"url": "rtsp://10.0.0.49:8554/cam",
"fps": 30,
"width": 1280,
"height": 720,
"use_mpp": true,
"use_ffmpeg": false,
"force_tcp": true,
"reconnect_sec": 5,
"reconnect_backoff_max_sec": 30
},
{
"id": "pre_cam1",
"type": "preprocess",
"role": "filter",
"enable": true,
"dst_w": 640,
"dst_h": 640,
"dst_format": "rgb",
"dst_packed": true,
"keep_ratio": false,
"rga_gate": "cam1_sample_full_pipeline",
"use_rga": true
},
{
"id": "yolo_cam1",
"type": "ai_yolo",
"role": "filter",
"enable": true,
"infer_fps": 10,
"model_path": "./models/yolov8n-640.rknn",
"model_version": "v8",
"num_classes": 80,
"conf": 0.25,
"nms": 0.45,
"class_filter": []
},
{
"id": "face_det_cam1",
"type": "ai_face_det",
"role": "filter",
"enable": true,
"model_path": "./models/RetinaFace_mobile320.rknn",
"conf": 0.7,
"nms": 0.4,
"max_faces": 10,
"output_landmarks": true,
"input_format": "rgb"
},
{
"id": "face_recog_cam1",
"type": "ai_face_recog",
"role": "filter",
"enable": true,
"model_path": "./models/mobilefacenet_arcface.rknn",
"align": true,
"emit_embedding": false,
"max_faces": 10,
"input_format": "rgb",
"input_dtype": "uint8",
"threshold": { "accept": 0.45, "margin": 0.05 },
"gallery": {
"backend": "sqlite",
"path": "./models/face_gallery.db",
"load_on_start": true,
"expected_dim": 512,
"dtype": "auto"
}
},
{
"id": "trk_cam1",
"type": "tracker",
"role": "filter",
"enable": true,
"mode": "bytetrack_lite",
"per_class": true,
"state_key": "cam1_sample_full_pipeline",
"track_classes": [0],
"ignore_classes": [],
"allowed_models": ["yolov5", "yolov8"],
"high_th": 0.5,
"low_th": 0.1,
"iou_th": 0.3,
"max_age_ms": 1500,
"min_hits": 2,
"max_tracks": 128,
"debug": { "stats": false, "stats_interval": 200 }
},
{
"id": "pre_face_cam1",
"type": "preprocess",
"role": "filter",
"enable": true,
"dst_w": 0,
"dst_h": 0,
"dst_format": "rgb",
"dst_packed": true,
"keep_ratio": false,
"rga_gate": "cam1_sample_full_pipeline",
"use_rga": true
},
{
"id": "osd_cam1",
"type": "osd",
"role": "filter",
"enable": true,
"draw_bbox": true,
"draw_text": true,
"draw_face_det": false,
"draw_face_bbox": false,
"line_width": 2,
"font_scale": 1,
"use_rga_bbox": false,
"labels": []
},
{
"id": "post_cam1",
"type": "preprocess",
"role": "filter",
"enable": true,
"dst_w": 1280,
"dst_h": 720,
"dst_format": "nv12",
"keep_ratio": false,
"rga_gate": "cam1_sample_full_pipeline",
"use_rga": true
},
{
"id": "pub_cam1",
"type": "publish",
"role": "filter",
"enable": true,
"codec": "h264",
"fps": 30,
"gop": 60,
"bitrate_kbps": 2000,
"use_mpp": true,
"use_ffmpeg_mux": true,
"outputs": [
{ "proto": "rtsp_server", "port": 8555, "path": "/live/cam1" },
{ "proto": "hls", "path": "./web/hls/cam1/index.m3u8", "segment_sec": 2 }
]
},
{
"id": "alarm_cam1",
"type": "alarm",
"role": "sink",
"enable": true,
"eval_fps": 10,
"labels": [],
"rules": [
{
"name": "person_in_view_v8",
"class_ids": [0],
"roi": { "x": 0.0, "y": 0.0, "w": 1.0, "h": 1.0 },
"min_score": 0.3,
"min_box_area_ratio": 0.02,
"require_track_id": true,
"min_duration_ms": 1500,
"min_hits": 3,
"hit_window_ms": 1500,
"cooldown_ms": 5000,
"per_track_cooldown_ms": 5000
}
],
"face_rules": [],
"actions": {
"log": { "enable": true, "level": "debug" },
"snapshot": {
"enable": true,
"format": "jpg",
"quality": 85,
"upload": {
"type": "minio",
"endpoint": "http://10.0.0.49:9000",
"bucket": "myminio",
"region": "us-east-1",
"access_key": "minioadmin",
"secret_key": "minioadmin"
}
},
"clip": {
"enable": true,
"pre_sec": 5,
"post_sec": 10,
"format": "mp4",
"fps": 30,
"upload": {
"type": "minio",
"endpoint": "http://10.0.0.49:9000",
"bucket": "myminio",
"region": "us-east-1",
"access_key": "minioadmin",
"secret_key": "minioadmin"
}
},
"http": {
"enable": false,
"url": "http://127.0.0.1:8080/api/alarm",
"timeout_ms": 3000,
"include_media_url": true,
"method": "POST"
},
"external_api": {
"enable": true,
"getTokenUrl": "http://10.0.0.49:8080/api/getToken",
"putMessageUrl": "http://10.0.0.49:8080/api/putMessage",
"tenantCode": "32",
"channelNo": "${vod_channelNo}",
"timeout_ms": 3000,
"include_media_url": true,
"token_header": "X-Access-Token",
"token_json_path": "responseBody.token",
"token_cache_sec": 1200
}
}
},
{
"id": "alarm_face_cam1",
"type": "alarm",
"role": "sink",
"enable": true,
"eval_fps": 5,
"labels": [],
"rules": [],
"face_rules": [
{ "name": "unknown_face", "type": "unknown", "cooldown_ms": 7000, "min_sim": 0.35, "min_hits": 2, "hit_window_ms": 1500, "min_face_area_ratio": 0.01, "min_face_aspect": 0.6, "max_face_aspect": 1.6 },
{ "name": "known_person", "type": "person", "cooldown_ms": 7000, "min_sim": 0.6, "min_hits": 2, "hit_window_ms": 1500, "min_face_area_ratio": 0.01, "min_face_aspect": 0.6, "max_face_aspect": 1.6 }
],
"actions": {
"log": { "enable": false, "level": "info" },
"snapshot": {
"enable": true,
"format": "jpg",
"quality": 85,
"upload": {
"type": "minio",
"endpoint": "http://10.0.0.49:9000",
"bucket": "myminio",
"region": "us-east-1",
"access_key": "minioadmin",
"secret_key": "minioadmin"
}
},
"clip": { "enable": false },
"http": {
"enable": true,
"url": "http://127.0.0.1:8080/api/alarm",
"timeout_ms": 3000,
"include_media_url": true,
"method": "POST"
}
}
}
],
"edges": [
["in_cam1", "pre_cam1"],
["in_cam1", "pre_face_cam1"],
["pre_cam1", "yolo_cam1"],
["yolo_cam1", "trk_cam1"],
["trk_cam1", "osd_cam1"],
["osd_cam1", "post_cam1"],
["post_cam1", "pub_cam1"],
["pub_cam1", "alarm_cam1"],
["pre_face_cam1", "face_det_cam1"],
["face_det_cam1", "face_recog_cam1"],
["face_recog_cam1", "alarm_face_cam1"]
]
}
]
}

353
configs/stress_5ch_cam4_template.json Normal file
View File

@ -0,0 +1,353 @@
{
"global": {
"metrics_port": 9000,
"web_root": "web"
},
"queue": {
"size": 4,
"strategy": "drop_oldest"
},
"templates": {
"strict_minio_alarm_pipeline": {
"nodes": [
{
"id": "in",
"type": "input_rtsp",
"role": "source",
"enable": true,
"url": "${url}",
"fps": "${fps}",
"width": "${src_w}",
"height": "${src_h}",
"use_mpp": true,
"use_ffmpeg": false,
"force_tcp": true,
"reconnect_sec": 5,
"reconnect_backoff_max_sec": 30
},
{
"id": "pre",
"type": "preprocess",
"role": "filter",
"enable": true,
"dst_w": 640,
"dst_h": 640,
"dst_format": "rgb",
"dst_packed": true,
"keep_ratio": false,
"rga_gate": "${name}",
"use_rga": true
},
{
"id": "ai",
"type": "ai_yolo",
"role": "filter",
"enable": true,
"infer_fps": 10,
"model_path": "./models/best-640.rknn",
"model_version": "v8",
"num_classes": 3,
"conf": 0.35,
"nms": 0.45,
"class_filter": []
},
{
"id": "trk",
"type": "tracker",
"role": "filter",
"enable": true,
"mode": "bytetrack_lite",
"per_class": true,
"state_key": "${name}",
"track_classes": [
0,
1,
2
],
"ignore_classes": [],
"allowed_models": [
"yolov5",
"yolov8"
],
"high_th": 0.5,
"low_th": 0.1,
"iou_th": 0.3,
"max_age_ms": 1500,
"min_hits": 2,
"max_tracks": 128,
"debug": {
"stats": false,
"stats_interval": 200
}
},
{
"id": "osd",
"type": "osd",
"role": "filter",
"enable": true,
"draw_bbox": true,
"draw_text": true,
"line_width": 2,
"font_scale": 1,
"labels": [],
"use_rga_bbox": false,
"draw_face_det": false,
"draw_face_bbox": false
},
{
"id": "post",
"type": "preprocess",
"role": "filter",
"enable": true,
"queue": {
"size": 2,
"policy": "drop_oldest"
},
"dst_w": "${src_w}",
"dst_h": "${src_h}",
"dst_format": "nv12",
"keep_ratio": false,
"rga_gate": "${name}",
"use_rga": true
},
{
"id": "alarm",
"type": "alarm",
"role": "sink",
"enable": true,
"eval_fps": 10,
"labels": [],
"rules": [
{
"name": "object_in_view",
"class_ids": [
0,
1,
2
],
"roi": {
"x": 0.0,
"y": 0.0,
"w": 1.0,
"h": 1.0
},
"min_score": 0.4,
"min_box_area_ratio": 0.02,
"require_track_id": true,
"min_duration_ms": 1500,
"min_hits": 3,
"hit_window_ms": 1500,
"cooldown_ms": 5000,
"per_track_cooldown_ms": 5000
}
],
"actions": {
"log": {
"enable": true,
"level": "info"
},
"snapshot": {
"enable": true,
"min_interval_ms": 15000,
"format": "jpg",
"quality": 85,
"upload": {
"type": "minio",
"endpoint": "${minio_endpoint}",
"bucket": "${minio_bucket}",
"region": "us-east-1",
"access_key": "${minio_ak}",
"secret_key": "${minio_sk}"
}
},
"clip": {
"enable": true,
"min_interval_ms": 15000,
"pre_sec": 5,
"post_sec": 10,
"format": "mp4",
"fps": "${fps}",
"upload": {
"type": "minio",
"endpoint": "${minio_endpoint}",
"bucket": "${minio_bucket}",
"region": "us-east-1",
"access_key": "${minio_ak}",
"secret_key": "${minio_sk}"
}
},
"http": {
"enable": false,
"url": "http://127.0.0.1:8080/api/alarm",
"timeout_ms": 3000,
"include_media_url": true,
"method": "POST"
},
"external_api": {
"enable": true,
"getTokenUrl": "http://10.0.0.49:8080/api/getToken",
"putMessageUrl": "http://10.0.0.49:8080/api/putMessage",
"tenantCode": "32",
"channelNo": "${name}",
"timeout_ms": 3000,
"include_media_url": true,
"token_header": "X-Access-Token",
"token_json_path": "responseBody.token",
"token_cache_sec": 1200
}
}
},
{
"id": "pub",
"type": "publish",
"role": "filter",
"enable": true,
"queue": {
"size": 2,
"policy": "drop_oldest"
},
"codec": "h264",
"fps": "${fps}",
"gop": "${gop}",
"bitrate_kbps": "${bitrate_kbps}",
"use_mpp": true,
"use_ffmpeg_mux": true,
"outputs": [
{
"proto": "rtsp_server",
"port": 8555,
"path": "/live/${name}"
},
{
"proto": "hls",
"path": "./web/hls/${name}/index.m3u8",
"segment_sec": 2
}
]
}
],
"edges": [
[
"in",
"pre"
],
[
"pre",
"ai"
],
[
"ai",
"trk"
],
[
"trk",
"osd"
],
[
"osd",
"post"
],
[
"post",
"pub"
],
[
"pub",
"alarm"
]
]
}
},
"instances": [
{
"name": "cam1",
"template": "strict_minio_alarm_pipeline",
"params": {
"name": "cam1",
"url": "rtsp://10.0.0.49:8554/cam",
"src_w": 1280,
"src_h": 720,
"fps": 30,
"gop": 60,
"bitrate_kbps": 2000,
"model_path": "./models/best-640.rknn",
"minio_endpoint": "http://10.0.0.49:9000",
"minio_bucket": "myminio",
"minio_ak": "minioadmin",
"minio_sk": "minioadmin"
}
},
{
"name": "cam2",
"template": "strict_minio_alarm_pipeline",
"params": {
"name": "cam2",
"url": "rtsp://10.0.0.49:8554/cam",
"src_w": 1280,
"src_h": 720,
"fps": 30,
"gop": 60,
"bitrate_kbps": 2000,
"model_path": "./models/best-640.rknn",
"minio_endpoint": "http://10.0.0.49:9000",
"minio_bucket": "myminio",
"minio_ak": "minioadmin",
"minio_sk": "minioadmin"
}
},
{
"name": "cam3",
"template": "strict_minio_alarm_pipeline",
"params": {
"name": "cam3",
"url": "rtsp://10.0.0.49:8554/cam",
"src_w": 1280,
"src_h": 720,
"fps": 30,
"gop": 60,
"bitrate_kbps": 2000,
"model_path": "./models/best-640.rknn",
"minio_endpoint": "http://10.0.0.49:9000",
"minio_bucket": "myminio",
"minio_ak": "minioadmin",
"minio_sk": "minioadmin"
}
},
{
"name": "cam4",
"template": "strict_minio_alarm_pipeline",
"params": {
"name": "cam4",
"url": "rtsp://10.0.0.49:8554/cam",
"src_w": 1280,
"src_h": 720,
"fps": 30,
"gop": 60,
"bitrate_kbps": 2000,
"model_path": "./models/best-640.rknn",
"minio_endpoint": "http://10.0.0.49:9000",
"minio_bucket": "myminio",
"minio_ak": "minioadmin",
"minio_sk": "minioadmin"
}
},
{
"name": "cam5",
"template": "strict_minio_alarm_pipeline",
"params": {
"name": "cam5",
"url": "rtsp://10.0.0.49:8554/cam",
"src_w": 1280,
"src_h": 720,
"fps": 30,
"gop": 60,
"bitrate_kbps": 2000,
"model_path": "./models/best-640.rknn",
"minio_endpoint": "http://10.0.0.49:9000",
"minio_bucket": "myminio",
"minio_ak": "minioadmin",
"minio_sk": "minioadmin"
}
}
]
}

1105
configs/stress_test_4ch_shared_source.json
View File

File diff suppressed because it is too large Load Diff

1
configs/stress_test_4ch_shared_source.json.last_good.json
View File

File diff suppressed because one or more lines are too long

128
docs/architecture/zlmediakit_multi_stream.md Normal file
View File

@ -0,0 +1,128 @@
# ZLMediaKit 多路流输出机制
## 发现背景
在开发 4-5 路并发压测配置时，发现 ZLMediaKit 并非通过多端口监听来实现多路输出，而是通过**单端口 + 多路径**的方式实现。
## 机制说明
### 传统理解（错误）
以为每个 publish 节点需要独立的端口：
```
cam1 -> port 8555 -> /live/cam1
cam2 -> port 8556 -> /live/cam2
cam3 -> port 8557 -> /live/cam3
cam4 -> port 8558 -> /live/cam4
```
### 实际机制（正确）
ZLMediaKit 采用 **HTTP-like 虚拟主机机制**：
```
Single TCP Server: port 8555
├─ path: /live/cam1 -> stream cam1
├─ path: /live/cam2 -> stream cam2
├─ path: /live/cam3 -> stream cam3
├─ path: /live/cam4 -> stream cam4
└─ path: /live/cam5 -> stream cam5
```
## 日志验证
启动 5 路并发时观察到的日志：
```
# 只启动一次 TCP Server
TcpServer.cpp:200 start_l | TCP server listening on [::]: 8555
# 5 路流共享同一个端口，不同 path
zlm rtsp server ready: rtsp://0.0.0.0:8555/live/cam1
zlm rtsp server ready: rtsp://0.0.0.0:8555/live/cam2
zlm rtsp server ready: rtsp://0.0.0.0:8555/live/cam3
zlm rtsp server ready: rtsp://0.0.0.0:8555/live/cam4
zlm rtsp server ready: rtsp://0.0.0.0:8555/live/cam5
```
## 配置示例
### 错误的配置（端口冲突）
```json
"outputs": [
{ "proto": "rtsp_server", "port": 8555, "path": "/live/cam1" }
]
```
5 个实例分别配置 8555/8556/8557/8558/8559 会导致资源浪费。
### 正确的配置（共享端口）
```json
"outputs": [
{ "proto": "rtsp_server", "port": 8555, "path": "/live/${name}" }
]
```
所有实例使用相同端口，通过 `${name}` 变量区分不同路径。
## 播放地址
基于共享端口机制，客户端播放地址为：
| 通道 | RTSP 地址 | HLS 地址 |
|-----|-----------|----------|
| cam1 | `rtsp://<ip>:8555/live/cam1` | `http://<ip>:9000/hls/cam1/index.m3u8` |
| cam2 | `rtsp://<ip>:8555/live/cam2` | `http://<ip>:9000/hls/cam2/index.m3u8` |
| cam3 | `rtsp://<ip>:8555/live/cam3` | `http://<ip>:9000/hls/cam3/index.m3u8` |
| cam4 | `rtsp://<ip>:8555/live/cam4` | `http://<ip>:9000/hls/cam4/index.m3u8` |
| cam5 | `rtsp://<ip>:8555/live/cam5` | `http://<ip>:9000/hls/cam5/index.m3u8` |
## 优势
1. **端口资源节省** - 无论多少路流，只需 1 个端口
2. **防火墙友好** - 只需开放 1 个端口
3. **管理简便** - 统一入口，通过路径区分
4. **扩展性好** - 理论上支持无限路流（受限于系统资源）
## 限制
1. **单进程限制** - ZLMediaKit 实例是单例的，必须在同一个 media-server 进程内
2. **路径冲突** - 不同流的路径必须唯一
3. **端口独占** - 8555 被占用后，其他进程无法使用
## 相关配置
多路并发压测配置示例：`configs/stress_5ch_cam4_template.json`
关键配置片段：
```json
{
"templates": {
"strict_minio_alarm_pipeline": {
"nodes": [
{
"id": "pub",
"type": "publish",
"outputs": [
{ "proto": "rtsp_server", "port": 8555, "path": "/live/${name}" },
{ "proto": "hls", "path": "./web/hls/${name}/index.m3u8" }
]
}
]
}
},
"instances": [
{ "name": "cam1", "template": "strict_minio_alarm_pipeline" },
{ "name": "cam2", "template": "strict_minio_alarm_pipeline" },
{ "name": "cam3", "template": "strict_minio_alarm_pipeline" },
{ "name": "cam4", "template": "strict_minio_alarm_pipeline" },
{ "name": "cam5", "template": "strict_minio_alarm_pipeline" }
]
}
```
## 参考
- ZLMediaKit 文档：https://github.com/ZLMediaKit/ZLMediaKit
- RTSP 协议虚拟主机机制

163
docs/bugfix_yolov8_fp16.md Normal file
View File

@ -0,0 +1,163 @@
# YOLOv8 FP16 数据解析错误修复
## 问题描述
**现象：** 使用 YOLOv8 RKNN 模型（如 `yolov8n-640.rknn`、`best-640.rknn`）时，无法检测到目标，跟踪器显示 `tracks=0`，模型输出为垃圾值。
**错误日志特征：**
```
[ai_yolo] First box: x=125969024.000000, y=38730874880.000000, w=0.000000, h=0.000000
[ai_yolo] ProcessOutputV8 result: valid_count=8400 out of 8400 boxes
```
- 所有 8400 个 anchor 都通过了阈值检测
- 坐标值为超大异常数字
- 检测分数为 `score=5561747627709562880.000000`（溢出值）
## 根因分析
### 1. 数据类型不匹配
RKNN 模型的输出数据类型与代码解析方式不匹配：
| 模型 | RKNN 输出类型 | 代码原处理方式 | 结果 |
|------|--------------|---------------|------|
| YOLOv5 | INT8 (量化) | `int8_t*` + 反量化 | ✅ 正常 |
| YOLOv8 | FP16 (半精度) | `reinterpret_cast<float*>` | ❌ 错误 |
**问题代码：**
```cpp
// ai_yolo_node.cpp 第 591-592 行（修复前）
if (outputs[0].type == RKNN_TENSOR_FLOAT32 ||
outputs[0].type == RKNN_TENSOR_FLOAT16) {
// 两者都按 float32 解析，导致 FP16 数据被错误解析
valid_count = ProcessOutputV8(reinterpret_cast<float*>(...), ...);
}
```
### 2. 为什么 FP16 不能直接当 FP32 解析
- **FP16** (半精度浮点): 16位 = 1位符号 + 5位指数 + 10位尾数
- **FP32** (单精度浮点): 32位 = 1位符号 + 8位指数 + 23位尾数
直接内存解释为 FP32 时，两个 FP16 数值会被错误地合并成一个 FP32，导致数据完全错乱。
### 3. 为什么 YOLOv5 正常
YOLOv5 RKNN 模型默认使用 INT8 量化，代码本来就有反量化逻辑：
```cpp
DequantizeAffineToF32(int8_t qnt, int32_t zp, float scale)
```
而 YOLOv8 RKNN 模型默认使用 FP16，代码缺乏 FP16→FP32 转换。
## 解决方法
### 1. 添加 FP16 到 FP32 转换函数
```cpp
// ai_yolo_node.cpp
// FP16 (half) to FP32 conversion
// IEEE 754 half-precision: 1 sign bit, 5 exponent bits, 10 mantissa bits
inline float Fp16ToFp32(uint16_t h) {
uint32_t sign = (h >> 15) & 0x1;
uint32_t exp = (h >> 10) & 0x1F;
uint32_t mant = h & 0x3FF;
uint32_t f;
if (exp == 0) {
// Zero or subnormal
if (mant == 0) {
f = (sign << 31); // Signed zero
} else {
// Subnormal: convert to normal
exp = 1;
while ((mant & 0x400) == 0) {
mant <<= 1;
exp--;
}
mant &= 0x3FF;
f = (sign << 31) | ((exp + 112) << 23) | (mant << 13);
}
} else if (exp == 0x1F) {
// Infinity or NaN
f = (sign << 31) | (0xFF << 23) | (mant << 13);
} else {
// Normal number
f = (sign << 31) | ((exp + 112) << 23) | (mant << 13);
}
float result;
memcpy(&result, &f, sizeof(float));
return result;
}
```
### 2. 单独处理 FP16 分支
```cpp
// ai_yolo_node.cpp PostProcessBorrowed() 函数
if (outputs[0].type == RKNN_TENSOR_FLOAT32) {
// FP32 直接解析
valid_count = ProcessOutputV8(reinterpret_cast<float*>(...), ...);
} else if (outputs[0].type == RKNN_TENSOR_FLOAT16) {
// FP16 先转换到 FP32 缓冲区
size_t num_elements = outputs[0].size / sizeof(uint16_t);
fp32_buffer_.resize(num_elements);
const uint16_t* fp16_data = reinterpret_cast<const uint16_t*>(outputs[0].data);
for (size_t i = 0; i < num_elements; ++i) {
fp32_buffer_[i] = Fp16ToFp32(fp16_data[i]);
}
valid_count = ProcessOutputV8(fp32_buffer_.data(), ...);
} else {
// INT8 反量化
valid_count = ProcessOutputV8Int8(reinterpret_cast<int8_t*>(...), ...);
}
```
### 3. 添加 FP32 缓冲区成员变量
```cpp
class AiYoloNode : public INode {
// ...
#if defined(RK3588_ENABLE_RKNN)
ModelHandle model_handle_ = kInvalidModelHandle;
uint32_t n_output_ = 0;
std::vector<uint8_t> rgb_tmp_;
std::vector<float> fp32_buffer_; // For FP16 to FP32 conversion
#endif
};
```
## 修复后的验证
**正常日志：**
```
[ai_yolo] First box: x=4.632812, y=6.921875, w=11.078125, h=14.296875
[ai_yolo] ProcessOutputV8 result: valid_count=23 out of 8400 boxes
[tracker] id=trk_cam1 tracks=1 created=1 removed=0 matched=153 unmatch_det=1
```
- 坐标值在正常范围 (0-640)
- 检测数量合理 (23/8400)
- 跟踪器正常工作 (tracks=1)
- 检测分数正常 (0.67)
## 相关文件
- `plugins/ai_yolo/ai_yolo_node.cpp`
## 影响范围
- ✅ YOLOv5 (INT8): 不受影响，继续正常工作
- ✅ YOLOv8 (FP16): 修复后正常工作
- ✅ YOLOv8 (FP32): 不受影响
- ✅ YOLOv8 (INT8): 不受影响
## 参考
- RKNN API 数据类型定义: `rknn_tensor_type` in `rknn_api.h`
- `RKNN_TENSOR_FLOAT32 = 0`
- `RKNN_TENSOR_FLOAT16 = 1`
- `RKNN_TENSOR_INT8 = 2`
- IEEE 754 半精度浮点标准

17
docs/requirements/guide.md
View File

@ -27,4 +27,19 @@ ffplay rtsp://localhost:8555/live/cam1_face_det
rtsp://3588的IP:8555/live/cam1_face_det
- 编译agent
go build -o rk3588-agent_linux_arm64 ./cmd/rk3588-agent
go build -o rk3588-agent_linux_arm64 ./cmd/rk3588-agent
- 运行模拟告警服务
python .\mock_alarm_server.py
然后在 RK3588 上测试 token 接口：
curl -X POST http://10.0.0.49:8080/api/getToken
- best-640.rknn模型只检测3类
类别映射：
0 = person（人）
1 = shoe（鞋子）✅
2 = phone（手机）

BIN
models/best-640.rknn
View File

Binary file not shown.

BIN
models/yolov8n-640.rknn
View File

Binary file not shown.

53
plugins/ai_yolo/ai_yolo_node.cpp
View File

@ -64,6 +64,41 @@ inline float DequantizeAffineToF32(int8_t qnt, int32_t zp, float scale) {
return (static_cast<float>(qnt) - static_cast<float>(zp)) * scale;
}
// FP16 (half) to FP32 conversion
// IEEE 754 half-precision: 1 sign bit, 5 exponent bits, 10 mantissa bits
inline float Fp16ToFp32(uint16_t h) {
uint32_t sign = (h >> 15) & 0x1;
uint32_t exp = (h >> 10) & 0x1F;
uint32_t mant = h & 0x3FF;
uint32_t f;
if (exp == 0) {
// Zero or subnormal
if (mant == 0) {
f = (sign << 31); // Signed zero
} else {
// Subnormal: convert to normal
exp = 1;
while ((mant & 0x400) == 0) {
mant <<= 1;
exp--;
}
mant &= 0x3FF;
f = (sign << 31) | ((exp + 112) << 23) | (mant << 13);
}
} else if (exp == 0x1F) {
// Infinity or NaN
f = (sign << 31) | (0xFF << 23) | (mant << 13);
} else {
// Normal number
f = (sign << 31) | ((exp + 112) << 23) | (mant << 13);
}
float result;
memcpy(&result, &f, sizeof(float));
return result;
}
float CalculateIoU(float x1_min, float y1_min, float x1_max, float y1_max,
float x2_min, float y2_min, float x2_max, float y2_max) {
float w = std::fmax(0.f, std::fmin(x1_max, x2_max) - std::fmax(x1_min, x2_min) + 1.0f);
@ -504,7 +539,7 @@ private:
outputs[2].zp, outputs[2].scale);
valid_count = cnt0 + cnt1 + cnt2;
} else {
if (outputs.empty()) return;
if (outputs.empty()) return;
if (!outputs[0].data || outputs[0].size == 0) return;
int num_boxes = 0;
@ -522,12 +557,23 @@ private:
num_boxes = static_cast<int>(outputs[0].size) / num_channels;
}
if (outputs[0].type == RKNN_TENSOR_FLOAT32 ||
outputs[0].type == RKNN_TENSOR_FLOAT16) {
if (outputs[0].type == RKNN_TENSOR_FLOAT32) {
valid_count = ProcessOutputV8(reinterpret_cast<float*>(const_cast<uint8_t*>(outputs[0].data)),
num_boxes, num_classes_,
model_input_h_, model_input_w_,
boxes, obj_probs, class_ids, conf_thresh_);
} else if (outputs[0].type == RKNN_TENSOR_FLOAT16) {
// Convert FP16 to FP32
size_t num_elements = outputs[0].size / sizeof(uint16_t);
fp32_buffer_.resize(num_elements);
const uint16_t* fp16_data = reinterpret_cast<const uint16_t*>(outputs[0].data);
for (size_t i = 0; i < num_elements; ++i) {
fp32_buffer_[i] = Fp16ToFp32(fp16_data[i]);
}
valid_count = ProcessOutputV8(fp32_buffer_.data(),
num_boxes, num_classes_,
model_input_h_, model_input_w_,
boxes, obj_probs, class_ids, conf_thresh_);
} else {
valid_count = ProcessOutputV8Int8(reinterpret_cast<int8_t*>(const_cast<uint8_t*>(outputs[0].data)),
num_boxes, num_classes_,
@ -740,6 +786,7 @@ private:
ModelHandle model_handle_ = kInvalidModelHandle;
uint32_t n_output_ = 0;
std::vector<uint8_t> rgb_tmp_;
std::vector<float> fp32_buffer_; // For FP16 to FP32 conversion
#endif
};

5
scripts/pt2rknn_yolov8.py
View File

@ -14,7 +14,12 @@ def run_cmd(cmd):
def export_pt_to_onnx(pt_path: Path, onnx_path: Path, imgsz: int, opset: int):
try:
import torch
from ultralytics import YOLO # type: ignore
from ultralytics.nn.tasks import DetectionModel, SegmentationModel, ClassificationModel, PoseModel
# Fix for PyTorch 2.6+ weights_only default change
torch.serialization.add_safe_globals([DetectionModel, SegmentationModel, ClassificationModel, PoseModel])
print("[INFO] exporting via ultralytics python API")
model = YOLO(str(pt_path))

BIN
yolov8n_cxn.onnx
View File

Binary file not shown.