1. Yolo11 Camera Demo
这里基于V4L2 Camera Demo的基础上， 添加yolo11推理处理； 也就是使用RKNN-Model-Zoo里的yolo11来做实时推理。
1.1 yolo11 模型转换
这里使用Yolo11 Demo已经转换好的模型， 拷贝到rknn_model_zoo/examples/yolo11/model目录里， 如下：
neardi@LPA3588:~/rknn/rknn_model_zoo/examples/yolo11/model$ ls
bus.jpg coco_80_labels_list.txt dataset.txt download_model.sh yolo11n.onnx yolo11.rknn
1.2 C++代码
需要在V4l2-camera的例程里添加yolo11推理代码， 如下：
/*-------------------------------------------
Main Function
-------------------------------------------*/
int main(int argc, char **argv)
{
if (argc != 3)
{
printf("%s <cam_id> <model_path>\n", argv[0]);
return -1;
}
int cam_id = std::stoi(argv[1]);
const char* model_path = argv[2];
std::signal(SIGINT, signalHandler);
int ret;
rknn_app_context_t rknn_app_ctx;
memset(&rknn_app_ctx, 0, sizeof(rknn_app_context_t));
init_post_process();
ret = init_yolo11_model(model_path, &rknn_app_ctx);
if (ret != 0)
{
printf("init_yolo11_model fail! ret=%d model_path=%s\n", ret, model_path);
deinit_post_process();
return -1;
}
try {
MemoryPool pool(4, 1, width, height); // 4 preallocated NV12 dma buffers
// create separate DMA buffer for RGA destination (BGR)
DMABuffer dstBuf;
size_t dst_size = size_t(width) * height * 3; // BGR888
if (!dstBuf.alloc(DMA_HEAP_DMA32_UNCACHED_PATH, dst_size)) {
cerr << "alloc dst dma failed\n";
return -1;
}
std::string cam_dev = "/dev/video" + std::to_string(cam_id);
// start camera thread
thread cam_thread(camera_thread_func, std::ref(pool), cam_dev.c_str(), width, height);
auto dataQ = pool.getDataQueue(0);
auto idleQ = pool.getIdleQueue(0);
image_buffer_t src_image;
object_detect_result_list od_results;
string window_name = "RKNN Demo Window";
namedWindow(window_name);
while (!g_exit.load(std::memory_order_relaxed)) {
shared_ptr<NV12FrameDMABuf> frame;
if (!dataQ->pop(frame)) {
this_thread::sleep_for(chrono::milliseconds(1));
continue;
}
// Run RGA: src = frame->buf->fd (NV12), dst = dstBuf.fd (BGR)
int src_fd = frame->buf->fd;
int src_size = frame->buf->size;
int dst_fd = dstBuf.fd;
int dst_size_int = int(dstBuf.size);
int ret = rga_nv12_fd_to_bgr_fd(src_fd, src_size, dst_fd, dst_size_int, width, height);
if (ret != 0) {
cerr << "rga convert failed\n";
idleQ->push(frame);
continue;
}
// ensure device->cpu sync before reading dstBuf.va
dma_sync_device_to_cpu(dst_fd);
cv::Mat bgr(height, width, CV_8UC3, dstBuf.va);
memset(&src_image, 0, sizeof(image_buffer_t));
if (!mat_to_image_buffer(bgr, src_image, IMAGE_FORMAT_RGB888)) {
printf("mat to image buffer failed\n");
idleQ->push(frame);
continue;
}
ret = inference_yolo11_model(&rknn_app_ctx, &src_image, &od_results);
if (0 != ret) {
idleQ->push(frame);
continue;
}
// 画框和概率
char text[256];
for (int i = 0; i < od_results.count; i++)
{
object_detect_result *det_result = &(od_results.results[i]);
printf("%s @ (%d %d %d %d) %.3f\n", coco_cls_to_name(det_result->cls_id),
det_result->box.left, det_result->box.top,
det_result->box.right, det_result->box.bottom,
det_result->prop);
int x1 = det_result->box.left;
int y1 = det_result->box.top;
int x2 = det_result->box.right;
int y2 = det_result->box.bottom;
draw_rectangle(&src_image, x1, y1, x2 - x1, y2 - y1, COLOR_BLUE, 3);
sprintf(text, "%s %.1f%%", coco_cls_to_name(det_result->cls_id), det_result->prop * 100);
draw_text(&src_image, text, x1, y1 - 20, COLOR_RED, 10);
}
imshow("BGR", bgr);
imshow(window_name, bgr);
if (waitKey(1) == 27) {
cout << "exit\n";
break;
}
// return frame to idle pool
idleQ->push(frame);
}
cam_thread.detach();
} catch (const exception &e) {
cerr << "fatal: " << e.what() << endl;
}
deinit_post_process();
release_yolo11_model(&rknn_app_ctx);
return 0;
}
从下此处下载完整的代码
1.3 编译
更改CMakefile.txt文件， 修改的内如如下：
--- a/examples/yolo11/cpp/CMakeLists.txt
+++ b/examples/yolo11/cpp/CMakeLists.txt
@@ -28,10 +28,26 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../../../utils/ utils.out)
set(CMAKE_INSTALL_RPATH "$ORIGIN/../lib")
+find_package(OpenCV REQUIRED)
+include_directories(${OpenCV_INCLUDE_DIRS})
+
+# GStreamer
+set(GSTREAMER_INCLUDE_DIR "/usr/include/gstreamer-1.0")
+set(GSTREAMER_LIBRARY_DIR "/usr/lib/aarch64-linux-gnu/gstreamer-1.0")
+
+# Ensure that pkg-config can find GStreamer
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(GSTREAMER REQUIRED gstreamer-1.0)
+
+# Add the correct GStreamer library path
+link_directories(${GSTREAMER_LIBRARY_DIR})
+link_directories(/usr/local/lib)
+
file(GLOB SRCS ${CMAKE_CURRENT_SOURCE_DIR}/*.cc)
add_executable(${PROJECT_NAME}
main.cc
+ dma_alloc.cpp
postprocess.cc
${rknpu_yolo11_file}
)
@@ -39,8 +55,11 @@ add_executable(${PROJECT_NAME}
target_link_libraries(${PROJECT_NAME}
imageutils
fileutils
- imagedrawing
+ imagedrawing
+ ${OpenCV_LIBS}
+ ${GSTREAMER_LIBRARIES}
${LIBRKNNRT}
+ -lrga
dl
)
@@ -68,6 +87,7 @@ if (NOT (TARGET_SOC STREQUAL "rv1106" OR TARGET_SOC STREQUAL "rv1103" OR TARGET_
OR TARGET_SOC STREQUAL "rv1109" OR TARGET_SOC STREQUAL "rv1126" OR TARGET_SOC STREQUAL "rv1103b"))
add_executable(${PROJECT_NAME}_zero_copy
main.cc
+ dma_alloc.cpp
postprocess.cc
rknpu2/yolo11_zero_copy.cc
)
@@ -77,8 +97,11 @@ if (NOT (TARGET_SOC STREQUAL "rv1106" OR TARGET_SOC STREQUAL "rv1103" OR TARGET_
target_link_libraries(${PROJECT_NAME}_zero_copy
imageutils
fileutils
- imagedrawing
+ imagedrawing
+ ${OpenCV_LIBS}
+ ${GSTREAMER_LIBRARIES}
${LIBRKNNRT}
+ -lrga
dl
)
编译命令如下:
neardi@LPA3588:~/rknn/rknn_model_zoo$ ./build-linux.sh -t rk3588 -a aarch64 -d yolo11 -b Release
./build-linux.sh -t rk3588 -a aarch64 -d yolo11 -b Release
aarch64-linux-gnu
===================================
BUILD_DEMO_NAME=yolo11
BUILD_DEMO_PATH=examples/yolo11/cpp
TARGET_SOC=rk3588
TARGET_ARCH=aarch64
BUILD_TYPE=Release
ENABLE_ASAN=OFF
DISABLE_RGA=OFF
DISABLE_LIBJPEG=OFF
INSTALL_DIR=/home/neardi/rknn/rknn_model_zoo/install/rk3588_linux_aarch64/rknn_yolo11_demo
BUILD_DIR=/home/neardi/rknn/rknn_model_zoo/build/build_rknn_yolo11_demo_rk3588_linux_aarch64_Release
CC=aarch64-linux-gnu-gcc
CXX=aarch64-linux-gnu-g++
===================================
-- !!!!!!!!!!!CMAKE_SYSTEM_NAME: Linux
-- Configuring done
-- Generating done
-- Build files have been written to: /home/neardi/rknn/rknn_model_zoo/build/build_rknn_yolo11_demo_rk3588_linux_aarch64_Release
Consolidate compiler generated dependencies of target imagedrawing
Consolidate compiler generated dependencies of target fileutils
Consolidate compiler generated dependencies of target imageutils
Consolidate compiler generated dependencies of target audioutils
[ 11%] Built target fileutils
[ 22%] Built target imagedrawing
[ 33%] Built target imageutils
[ 44%] Built target audioutils
Consolidate compiler generated dependencies of target rknn_yolo11_demo_zero_copy
Consolidate compiler generated dependencies of target rknn_yolo11_demo
[100%] Built target rknn_yolo11_demo_zero_copy
[100%] Built target rknn_yolo11_demo
[ 11%] Built target imageutils
[ 22%] Built target fileutils
[ 33%] Built target imagedrawing
[ 61%] Built target rknn_yolo11_demo
[ 88%] Built target rknn_yolo11_demo_zero_copy
[100%] Built target audioutils
Install the project...
-- Install configuration: "Release"
-- Installing: /home/neardi/rknn/rknn_model_zoo/install/rk3588_linux_aarch64/rknn_yolo11_demo/./rknn_yolo11_demo_zero_copy
-- Set runtime path of "/home/neardi/rknn/rknn_model_zoo/install/rk3588_linux_aarch64/rknn_yolo11_demo/./rknn_yolo11_demo_zero_copy" to "$ORIGIN/../lib"
-- Installing: /home/neardi/rknn/rknn_model_zoo/install/rk3588_linux_aarch64/rknn_yolo11_demo/./rknn_yolo11_demo
-- Set runtime path of "/home/neardi/rknn/rknn_model_zoo/install/rk3588_linux_aarch64/rknn_yolo11_demo/./rknn_yolo11_demo" to "$ORIGIN/../lib"
-- Installing: /home/neardi/rknn/rknn_model_zoo/install/rk3588_linux_aarch64/rknn_yolo11_demo/model/bus.jpg
-- Installing: /home/neardi/rknn/rknn_model_zoo/install/rk3588_linux_aarch64/rknn_yolo11_demo/model/coco_80_labels_list.txt
-- Installing: /home/neardi/rknn/rknn_model_zoo/install/rk3588_linux_aarch64/rknn_yolo11_demo/model/yolo11.rknn
-- Installing: /home/neardi/rknn/rknn_model_zoo/install/rk3588_linux_aarch64/rknn_yolo11_demo/lib/librknnrt.so
-- Installing: /home/neardi/rknn/rknn_model_zoo/install/rk3588_linux_aarch64/rknn_yolo11_demo/lib/librga.so
1.4 运行及测试
编译成功后， 生成的目标文件如下:
neardi@LPA3588:~/rknn/rknn_model_zoo/install/rk3588_linux_aarch64/rknn_yolo11_demo$ tree
.
├── lib
│ ├── librga.so
│ └── librknnrt.so
├── model
│ ├── bus.jpg
│ ├── coco_80_labels_list.txt
│ └── yolo11.rknn
├── rknn_yolo11_demo
└── rknn_yolo11_demo_zero_copy
如下运行:
neardi@LPA3588:~/rknn/rknn_model_zoo/install/rk3588_linux_aarch64/rknn_yolo11_demo$ ./rknn_yolo11_demo 11 ./model/yolo11.rknn
load lable ./model/coco_80_labels_list.txt
model input num: 1, output num: 9
input tensors:
index=0, name=images, n_dims=4, dims=[1, 640, 640, 3], n_elems=1228800, size=1228800, fmt=NHWC, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003922
output tensors:
index=0, name=462, n_dims=4, dims=[1, 64, 80, 80], n_elems=409600, size=409600, fmt=NCHW, type=INT8, qnt_type=AFFINE, zp=-44, scale=0.135466
index=1, name=onnx::ReduceSum_476, n_dims=4, dims=[1, 80, 80, 80], n_elems=512000, size=512000, fmt=NCHW, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003027
index=2, name=480, n_dims=4, dims=[1, 1, 80, 80], n_elems=6400, size=6400, fmt=NCHW, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003036
index=3, name=487, n_dims=4, dims=[1, 64, 40, 40], n_elems=102400, size=102400, fmt=NCHW, type=INT8, qnt_type=AFFINE, zp=-38, scale=0.093536
index=4, name=onnx::ReduceSum_501, n_dims=4, dims=[1, 80, 40, 40], n_elems=128000, size=128000, fmt=NCHW, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003476
index=5, name=505, n_dims=4, dims=[1, 1, 40, 40], n_elems=1600, size=1600, fmt=NCHW, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003922
index=6, name=512, n_dims=4, dims=[1, 64, 20, 20], n_elems=25600, size=25600, fmt=NCHW, type=INT8, qnt_type=AFFINE, zp=-42, scale=0.084287
index=7, name=onnx::ReduceSum_526, n_dims=4, dims=[1, 80, 20, 20], n_elems=32000, size=32000, fmt=NCHW, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003903
index=8, name=530, n_dims=4, dims=[1, 1, 20, 20], n_elems=400, size=400, fmt=NCHW, type=INT8, qnt_type=AFFINE, zp=-128, scale=0.003922
model is NHWC input fmt
model input height=640, width=640, channel=3
camera_thread started
skip unstalbe packets, skip count = 9
skip unstalbe packets, skip count = 8
skip unstalbe packets, skip count = 7
skip unstalbe packets, skip count = 6
skip unstalbe packets, skip count = 5
skip unstalbe packets, skip count = 4
skip unstalbe packets, skip count = 3
skip unstalbe packets, skip count = 2
skip unstalbe packets, skip count = 1
skip unstalbe packets, skip count = 0
rga_api version 1.10.1_[4]
[
您的浏览器不支持 video 标签。
](/videos/yolo11.mp4)