爱芯元智芯片推理cn-clip

主要参考：​​​​​​GitHub - AXERA-TECH/CLIP-ONNX-AX650-CPP

1.clip模型转onnx

 pip install onnxmltools   

pip install protobuf==4.25.5
 

export CUDA_VISIBLE_DEVICES=0

export PYTHONPATH=${PYTHONPATH}:`pwd`/cn_clip

DATAPATH="/data/LLM/clip-data/"

mkdir -p ${DATAPATH}/deploy/  

checkpoint_path=${DATAPATH}/pretrained_weights/epoch_latest_event0521.pt
cd /data/LLM/Chinese-CLIP       

执行以下命令可以转换onnx:

python cn_clip/deploy/pytorch_to_onnx.py        --model-arch ViT-B-16        --pytorch-ckpt-path ${checkpoint_path}        --save-onnx-path ${DATAPATH}/deploy/vit-b-16        --convert-text --convert-vision

 2.onnx转axmodel

       cnclip_config_npu3_U8.json文件

{"model_type": "ONNX","npu_mode": "NPU3","quant": {"input_configs": [{"tensor_name": "image","calibration_dataset": "./val.tar","calibration_size": 224,"calibration_mean": [122.77,116.746,104.093],"calibration_std": [68.5,66.632,70.323]}],"calibration_method": "MinMax","precision_analysis": false},"input_processors": [{"tensor_name": "image","tensor_format": "RGB","src_format": "RGB","src_dtype": "U8","src_layout": "NHWC","csc_mode": "NoCSC"}],"compiler": {"check": 0}
}

执行：

pulsar2 build --input vit-b-16.img.fp32.onnx --output_name  vit-b-16.img.fp32.axmodel --config cnclip_config_npu3_U8.json

可以得到：

3.代码编译

git clone https://github.com/AXERA-TECH/CLIP-ONNX-AX650-CPP/tree/main.git

1.下载依赖的opencv和onnxruntime

2.cmakelist修改

打开cmakelist.txt,在开头增加

set(CMAKE_C_COMPILER "aarch64-linux-gnu-gcc")
set(CMAKE_CXX_COMPILER "aarch64-linux-gnu-g++")

add_compile_options(-g3 -Wall)

 增加包含文件：

增加依赖库：

 mkdir build 

cd build 

执行：

cmake -DONNXRUNTIME_DIR=${onnxruntime_dir} -DOpenCV_DIR=${opencv_cmake_file_dir} -DBSP_MSP_DIR=${msp_out_dir} -DBUILD_WITH_AX650=ON ..
make -j4

4运行程序

下载feature_matmul.onnx

# feature matmul model
wget https://github.com/ZHEQIUSHUI/CLIP-ONNX-AX650-CPP/releases/download/3models/feature_matmul.onnx

执行代码： 

./main -l 1 -v cn_vocab.txt -t cn_text.txt  -i images/ --ienc cn_clip_vitb16.axmodel --tenc vitb16.txt.fp32.onnx -d feature_matmul.onnx