目标检测: yolov5算法在RK3588上部署

 YOLOv5是一种单阶段目标检测算法，该算法在YOLOv4的基础上添加了一些新的改进思路，使其速度与精度都得到了极大的性能提升。主要的改进思路如下所示：

1、输入端：在模型训练阶段，提出了一些改进思路，主要包括Mosaic数据增强、自适应锚框计算、自适应图片缩放；
2、基准网络：融合其它检测算法中的一些新思路，主要包括：Focus结构与CSP结构；
3、Neck网络：目标检测网络在BackBone与最后的Head输出层之间往往会插入一些层，Yolov5中添加了FPN+PAN结构；
4、Head输出层：输出层的锚框机制与YOLOv4相同，主要改进的是训练时的损失函数GIOU_Loss，以及预测框筛选的DIOU_nms。
yolov5的网络结构图可以看我之前的博客。

yolov5s-6.0网络模型结构图_yolov5s的网络结构图是自己使用工具绘制的还是通过运行实验导出的-CSDN博客https://zhanghuiyolodet.blog.csdn.net/article/details/122840458

瑞芯微:

在当今智能化浪潮中，强大的算力是驱动各类设备进化的核心引擎。瑞芯微电子推出的RK3588，正是这样一颗集高性能、高集成度和先进AI能力于一身的旗舰级系统级芯片（SoC），被誉为国产芯片在高端应用领域的标杆之作。

首先，在核心性能上，RK3588展现了卓越的硬件实力。 它采用了先进的8nm制程工艺，这在有效控制功耗和发热的同时，为极致性能提供了基础。其CPU采用了创新的“四大核加四小核”架构，即4个高性能的ARM Cortex-A76核心与4个高能效的ARM Cortex-A55核心，可以根据任务负载智能调度，兼顾了峰值性能与持久续航。在图形处理方面，集成的ARM Mali-G610 MP4 GPU，不仅支持4K/8K超高清视频的编解码，更提供了强大的图形渲染能力，足以流畅运行复杂的图形界面和3D游戏。

其次，RK3588的AI算力是其最亮眼的标签之一。 芯片内置了高达6 TOPS（万亿次运算/秒）算力的NPU（神经网络处理单元），专为机器学习和AI推理任务优化。这使得RK3588能够轻松应对人脸识别、物体检测、行为分析等复杂的AI应用场景，为智能安防、服务机器人、自动驾驶辅助系统等提供了强大的边缘端计算大脑，减少了对云端算力的依赖。

再者，其超凡的多媒体与接口扩展能力令人印象深刻。 RK3588支持丰富的显示输出，可同时驱动多个4K或一个8K显示屏，并支持多种显示接口如HDMI 2.1、eDP等。在视频处理上，它支持包括AV1在内的多种主流格式的8K@60fps视频解码，以及4K多路视频编码。此外，它集成了丰富的接口，如PCIe 3.0、SATA 3.0、双Type-C/USB3.1等，为连接高速存储、千兆网络及各类外设提供了极大的灵活性。

综上所述，瑞芯微RK3588凭借其8nm工艺、八核CPU、强大GPU和6TOPS NPU，构成了一个全能的计算平台。 它被广泛应用于高端平板、边缘计算盒子、智能NVR、AR/VR设备、云终端及工业控制等多个领域，正以其全面的性能，推动着人工智能与物联网技术在千行百业中的深度融合与创新。

下面将yolov5转为RKNN进行端侧部署：

onnx2rknn.py

import cv2
import numpy as npfrom rknn.api import RKNN
import osif __name__ == '__main__':isQUANT = True  # True platform = 'rk3588'NAME = "yolov5s"ONNX_DIR = "./"OUT_DIR = "./"Width = 640Height = 640MODEL_PATH =  ONNX_DIR + NAME + '.onnx'RKNN_NAME = NAMEif isQUANT:RKNN_NAME = NAME + '_quant'else:RKNN_NAME = NAME + '_FP16'NEED_BUILD_MODEL = TrueDATASET = './dataset.txt'im_file = '/home/zdl/rknn_project/rtdetrv2/d1/t_15_00083.bmp'# Create RKNN objectrknn = RKNN(verbose=False,verbose_file="./outputlog.txt")RKNN_MODEL_PATH = './{}/{}.rknn'.format(OUT_DIR,RKNN_NAME)if NEED_BUILD_MODEL:# rknn.config(mean_values=[[0,0,0]], std_values=[[1,1,1]], target_platform=platform,disable_rules=['fuse_two_gather'] )rknn.config(mean_values=[[0,0,0]], std_values=[[255.0,255.0,255.0]],target_platform=platform)# rknn.config(mean_values=[[0]], std_values=[[255]],target_platform=platform)# Load modelprint('--> Loading model')ret = rknn.load_onnx(MODEL_PATH,outputs=["onnx::Reshape_329","onnx::Reshape_368","onnx::Reshape_407"])# ret = rknn.load_onnx(MODEL_PATH)if ret != 0:print('load model failed!')exit(ret)print('done')# Build modelprint('--> Building model')ret = rknn.build(do_quantization=isQUANT, dataset=DATASET)# rknn.accuracy_analysis(inputs=["./errors/2.jpg","./errors/3.jpg"],#                             output_dir='snapshot')# ret = rknn.accuracy_analysis(inputs=[im_file],output_dir='snapshot')if ret != 0:print('build model failed.')exit(ret)print('done')# Export rknn modelif not os.path.exists(OUT_DIR):os.mkdir(OUT_DIR)print('--> Export RKNN model: {}'.format(RKNN_MODEL_PATH))ret = rknn.export_rknn(RKNN_MODEL_PATH)if ret != 0:print('Export rknn model failed.')exit(ret)print('done')else:ret = rknn.load_rknn(RKNN_MODEL_PATH)rknn.release()

模型部署：

#include "RKNNModel.h"static unsigned char *load_data(FILE *fp, size_t ofst, size_t sz)
{unsigned char *data;int ret;data = NULL;if (NULL == fp){return NULL;}ret = fseek(fp, ofst, SEEK_SET);if (ret != 0){printf("blob seek failure.\n");return NULL;}data = (unsigned char *)malloc(sz);if (data == NULL){printf("buffer malloc failure.\n");return NULL;}ret = fread(data, 1, sz, fp);return data;
}static unsigned char *load_model(const char *filename, int *model_size)
{FILE *fp;unsigned char *data;fp = fopen(filename, "rb");if (NULL == fp){printf("Open file %s failed.\n", filename);return NULL;}fseek(fp, 0, SEEK_END);int size = ftell(fp);data = load_data(fp, 0, size);fclose(fp);*model_size = size;return data;
}static void dump_tensor_attr(rknn_tensor_attr* attr)
{printf("  index=%d, name=%s, n_dims=%d, dims=[%d, %d, %d, %d], n_elems=%d, size=%d, fmt=%s, type=%s, qnt_type=%s, ""zp=%d, scale=%f\n",attr->index, attr->name, attr->n_dims, attr->dims[0], attr->dims[1], attr->dims[2], attr->dims[3],attr->n_elems, attr->size, get_format_string(attr->fmt), get_type_string(attr->type),get_qnt_type_string(attr->qnt_type), attr->zp, attr->scale);
}RKNNModel::RKNNModel()
{}RKNNModel::~RKNNModel() {this->ret = rknn_destroy(this->ctx);
}void RKNNModel::deinit() {}int RKNNModel::loadRKNN(std::string modelpath ,int core_id)
{this->modelselect = modelselect;int model_data_size = 0;unsigned char *model_data = load_model(modelpath.c_str(), &model_data_size);this->ret  = rknn_init(&this->ctx, model_data, model_data_size, 0, NULL); // 当加载模型路径的时候，size被置为0if (this->ret < 0) {printf("rknn_init error ret=%d\n", ret); // 模型加载失败return -1;}if(core_id == 0){this->ret = rknn_set_core_mask(this->ctx,RKNN_NPU_CORE_0); // 使用1核}else if(core_id == 1){this->ret = rknn_set_core_mask(this->ctx,RKNN_NPU_CORE_1); // 使用2核}else if(core_id == 2){this->ret = rknn_set_core_mask(this->ctx,RKNN_NPU_CORE_2); // 使用3核}else if(core_id == 3){this->ret = rknn_set_core_mask(this->ctx,RKNN_NPU_CORE_0_1_2); // 使用0 1 2核}else if(core_id == 4){this->ret = rknn_set_core_mask(this->ctx,RKNN_NPU_CORE_0_1); // 使用0 1核}rknn_sdk_version version;this->ret = rknn_query(this->ctx,RKNN_QUERY_SDK_VERSION,&version,sizeof(rknn_sdk_version));if(this->ret < 0){printf("获取版本失败！！");return -2;}printf("sdk version: %s driver version: %s\n", version.api_version, version.drv_version);// ================ 获得输出输出个数 ===================//this->ret = rknn_query(this->ctx, RKNN_QUERY_IN_OUT_NUM, &this->io_num, sizeof(this->io_num));if (this->ret < 0) {printf("无法获取输出个数!!! ret=%d\n", ret);return -3;}// ================ 获得输出输出个数 ===================//this->ret = rknn_query(ctx, RKNN_QUERY_IN_OUT_NUM, &this->io_num, sizeof(this->io_num));if (this->ret < 0) {printf("rknn_init error ret=%d\n", ret);return -4;}// =========== 模型输入tensor==========//rknn_tensor_attr input_attrs[this->io_num.n_input];memset(input_attrs, 0, sizeof(input_attrs));for (int i = 0; i < this->io_num.n_input; i++) {input_attrs[i].index = i;this->ret                  = rknn_query(ctx, RKNN_QUERY_INPUT_ATTR, &(input_attrs[i]), sizeof(rknn_tensor_attr));if (this->ret < 0) {printf("rknn_init error ret=%d\n", ret);return -5;}dump_tensor_attr(&(input_attrs[i]));}// ========== 判断是否NCWH ==========//if (input_attrs[0].fmt == RKNN_TENSOR_NCHW) {printf("model is NCHW input fmt\n");this->channel = input_attrs[0].dims[1];this->height  = input_attrs[0].dims[2];this->width   = input_attrs[0].dims[3];} else {printf("model is NHWC input fmt\n");this->height  = input_attrs[0].dims[1];this->width   = input_attrs[0].dims[2];this->channel = input_attrs[0].dims[3];}// ============= 模型输出 tensor===========//this->output_attrs.resize(this->io_num.n_output);for (int i = 0; i < this->io_num.n_output; i++) {this->output_attrs[i].index = i;this->ret                   = rknn_query(ctx, RKNN_QUERY_OUTPUT_ATTR, &(this->output_attrs[i]), sizeof(rknn_tensor_attr));dump_tensor_attr(&(output_attrs[i]));}// ============== 将图像数据加载到inputs ==============//this->inputs.resize(this->io_num.n_input);this->inputs[0].index        = 0;this->inputs[0].type         = RKNN_TENSOR_UINT8; //RKNN_TENSOR_UINT8;this->inputs[0].size         = width * height * channel;this->inputs[0].fmt          = RKNN_TENSOR_NHWC;this->inputs[0].pass_through = 0;// ============ 模型输出 ===========//this->outputs.resize(this->io_num.n_output);for (int i = 0; i < this->io_num.n_output; i++) {this->outputs[i].want_float = 0;}// 获取输出的scale 以及 zp值for (int i=0;i < this->io_num.n_output;i++) {this->out_scales.push_back(this->output_attrs[i].scale);this->out_zps.push_back(this->output_attrs[i].zp);}return 0;
}int RKNNModel::runRKNN(cv::Mat &image,std::vector<DetBox> &OBJBOX)
{// ------------ 图像预处理 ---------------//int x_pad ,y_pad;float scale;cv::Mat images = V5::letterboxYolo(image,x_pad,y_pad,scale,this->channel);if(image.data == NULL){std::cout << "image enpty" << std::endl;}this->inputs[0].buf = images.data; // 设置输入int rets = 0;// =========== 设置模型的输入数据，其中输入数据为rknn_input结构体对象 ==============//rets = rknn_inputs_set(this->ctx, this->io_num.n_input, &this->inputs[0]);rets = rknn_run(this->ctx, NULL);if(rets < 0){printf("模型: 推理出错！！！！ ret=%d\n", ret);return -1;}rets = rknn_outputs_get(ctx, this->io_num.n_output, &this->outputs[0], NULL);if(rets < 0){printf("模型: 获取输出出错！！！！ ret=%d\n", rets);return -2;}std::vector<cv::Rect> boxes;std::vector<float> confidences;std::vector<int>classids;std::vector<int> anchf;//rets = V5::proprecessV5(this->outputs,out_scales,out_zps,boxes,confidences,classids,anchf,0.05);std::vector<int> keepIndices;cv::dnn::NMSBoxes(boxes,confidences,0.25,0.45,keepIndices);for (int i = 0; i < keepIndices.size();i++){int id = keepIndices[i];//cv::Rect box = V5::get_rect(image,boxes[id]);cv::Rect box = {int((boxes[id].x - x_pad) / scale),int((boxes[id].y - y_pad) / scale),int(boxes[id].width / scale),int(boxes[id].height / scale)};DetBox DB;DB.bbox = box;DB.confidence = confidences[id];DB.classIdx = classids[id];DB.checkanchor = anchf[id];OBJBOX.push_back(DB);}ret = rknn_outputs_release(ctx, this->io_num.n_output, &(this->outputs[0]));return 0;
}int RKNNModel::quick_sort_indice_inverse(std::vector<float> &input, int left, int right, std::vector<int> &indices) {float key;int key_index;int low = left;int high = right;if (left < right) {key_index = indices[left];key = input[left];while (low < high) {while (low < high && input[high] <= key) {high--;}input[low] = input[high];indices[low] = indices[high];while (low < high && input[low] >= key) {low++;}input[high] = input[low];indices[high] = indices[low];}input[low] = key;indices[low] = key_index;quick_sort_indice_inverse(input, left, low - 1, indices);quick_sort_indice_inverse(input, low + 1, right, indices);}return low;
}

.h

#ifndef RKNNMODEL_H
#define RKNNMODEL_H
#include <iostream>
#include <fstream>
#include <dlfcn.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <vector>
#include <memory>
#include <unordered_map>
#include "opencv2/opencv.hpp"
#include <Eigen/Core>
#include <Eigen/Dense>
#include "rknn_api.h"
#include "YoloV5.h"typedef struct DetBox
{cv::Rect bbox; // 坐标框float confidence; // 置信度int classIdx; // 目标的IDint checkanchor;
}DetBox;class RKNNModel{
public:RKNNModel();~RKNNModel();int loadRKNN(std::string modelpath,int core_id);int runRKNN(cv::Mat &image,std::vector<DetBox> &OBJBOX);void deinit();
private:int            ret; // 返回值判断rknn_context   ctx;   // context 上下文rknn_input_output_num io_num; // 输入输出std::vector<rknn_tensor_attr> output_attrs;std::vector<rknn_output> outputs;std::vector<rknn_input> inputs;std::vector<float>    out_scales;std::vector<int32_t>  out_zps;// ======= 图像通道、宽、高==============//int channel = 3;int width   = 0;int height  = 0;int modelselect;// OBB的后处理以及NMS函数int quick_sort_indice_inverse(std::vector<float> &input, int left, int right, std::vector<int> &indices);void rbbox_to_corners(const float *in_rbbox, int *out_rbbox);
};
#endif

运行命令:

./rknn_yolov5_demo ../yolov5s_quant.rknn ../dog.jpg 10

结果展示: