WGAI项目图像视频语音识别功能

1、图像识别

<template><a-card class="contablelist" :bordered="false"><!-- 查询区域 --><div class="table-page-search-wrapper"><a-form layout="inline" @keyup.enter.native="searchQuery"></a-form></div> <!-- 查询区域-END --><div class="contable"><!-- 操作按钮区域 --><div class="table-operator"><a-button @click="handleAdd" type="primary" class="xz" icon="plus">新增</a-button><a-button type="primary" icon="download" class="dc" @click="handleExportXls('模型绑定')">导出</a-button><a-upload name="file" :showUploadList="false" :multiple="false" :headers="tokenHeader" :action="importExcelUrl" @change="handleImportExcel"><a-button type="primary"class="dr" icon="import">导入</a-button></a-upload><!-- 高级查询区域 --><a-dropdown v-if="selectedRowKeys.length > 0"><a-menu slot="overlay"><a-menu-item key="1" @click="batchDel"><a-icon type="delete"/>删除</a-menu-item></a-menu><a-button style="margin-left: 8px"> 批量操作 <a-icon type="down" /></a-button></a-dropdown></div><!-- table区域-begin --><div><div class="ant-alert ant-alert-info" style="margin-bottom: 16px;"><i class="anticon anticon-info-circle ant-alert-icon"></i> 已选择 <astyle="font-weight: 600">{{ selectedRowKeys.length }}</a>项<a style="margin-left: 24px" @click="onClearSelected">清空</a></div><a-table ref="table" size="middle" :scroll="{x:true}" bordered rowKey="id" :columns="columns":dataSource="dataSource" :pagination="ipagination" :loading="loading":rowSelection="{selectedRowKeys: selectedRowKeys, onChange: onSelectChange}" class="j-table-force-nowrap"@change="handleTableChange"><template slot="htmlSlot" slot-scope="text"><div v-html="text"></div></template><template slot="imgSlot" slot-scope="text,record"><span v-if="!text" style="font-size: 12px;font-style: italic;">无图片</span><img v-else :src="getImgView(text)" :preview="record.id" height="25px" alt=""style="max-width:80px;font-size: 12px;font-style: italic;" /></template><template slot="fileSlot" slot-scope="text"><span v-if="!text" style="font-size: 12px;font-style: italic;">无文件</span><a-button v-else :ghost="true" type="primary" icon="download" size="small" @click="downloadFile(text)">下载</a-button></template><span slot="action" slot-scope="text, record"><a @click="handleEdit(record)">编辑</a><a-divider type="vertical" /><a @click="handleIdentify(record)">AI识别</a><!--  <a-divider type="vertical" v-if="record.spaceOne==='1'" /><a v-if="record.spaceOne==='1'" @click="handleIdentifyClose(record)">视频识别结束</a><a-divider type="vertical" v-if="record.spaceOne==='1'" /><a  v-if="record.spaceOne==='1'" @click="handleOpenVideo(record)">视频区域报警配置</a>--><a-divider type="vertical" /> <a-dropdown><a class="ant-dropdown-link">更多<a-icon type="down" /></a><a-menu slot="overlay"><a-menu-item><a  v-if="record.spaceOne==='1'" @click="handleOpenVideo(record)">视频区域报警配置</a></a-menu-item><a-menu-item><a v-if="record.spaceOne==='1'" @click="handleIdentifyClose(record)">视频识别结束</a></a-menu-item><a-menu-item><a @click="handleDetail(record)">详情</a></a-menu-item><a-menu-item><a-popconfirm title="确定删除吗?" @confirm="() => handleDelete(record.id)"><a>删除</a></a-popconfirm></a-menu-item></a-menu></a-dropdown></span></a-table></div><tab-ai-model-bund-modal class="contc" :width="1200" ref="modalForm" @ok="modalFormOk"></tab-ai-model-bund-modal></div></a-card>
</template><script>import {httpAction,getAction} from '@/api/manage'import '@/assets/less/TableExpand.less'import {mixinDevice} from '@/utils/mixin'import {JeecgListMixin} from '@/mixins/JeecgListMixin'import TabAiModelBundModal from './modules/TabAiModelBundModal'
import { filterObj } from '@/utils/util';export default {name: 'TabAiModelBundList',mixins: [JeecgListMixin, mixinDevice],components: {TabAiModelBundModal},data() {return {description: '模型绑定管理页面',// 表头columns: [{title: '#',dataIndex: '',key: 'rowIndex',width: 60,align: "center",customRender: function(t, r, index) {return parseInt(index) + 1;}},{title: '识别名称',align: "center",dataIndex: 'spaceTwo'},{title: '模型名称',align: "center",dataIndex: 'modelName_dictText'}, {title: '识别类型',align: "center",dataIndex: 'spaceOne_dictText'},{title: '图片|视频地址',align: "center",dataIndex: 'sendUrl'},{title: '图片显示',align: "center",dataIndex: 'saveUrl',scopedSlots: {customRender: 'imgSlot'}},{title:'是否推送',align:"center",dataIndex: 'isPush'}, {title:'是否播报',align:"center",dataIndex: 'isAudio'}, {title:'推送地址',align:"center",dataIndex: 'pushId_dictText'}, {title:'播报地址',align:"center",dataIndex: 'audioId_dictText'},{title: '备注',align: "center",dataIndex: 'remake'},{title: '操作',dataIndex: 'action',align: "center",fixed: "right",width: 200,scopedSlots: {customRender: 'action'}}],url: {list: "/tab/tabAiModelBund/list",delete: "/tab/tabAiModelBund/delete",deleteBatch: "/tab/tabAiModelBund/deleteBatch",exportXlsUrl: "/tab/tabAiModelBund/exportXls",importExcelUrl: "tab/tabAiModelBund/importExcel",identifyUrl: "/tab/tabAiHistory/addIdentify",identifyCloseUrl: "/tab/tabAiHistory/addIdentifyClose"},dictOptions: {},superFieldList: [],}},created() {this.getSuperFieldList();},computed: {importExcelUrl: function() {return `${window._CONFIG['domianURL']}/${this.url.importExcelUrl}`;},},methods: {initDictConfig() {},getSuperFieldList() {let fieldList = [];fieldList.push({type: 'string',value: 'modelName',text: '模型名称',dictCode: ''})fieldList.push({type: 'string',value: 'sendUrl',text: '输入图片地址',dictCode: ''})fieldList.push({type: 'string',value: 'saveUrl',text: '保存图片地址',dictCode: ''})fieldList.push({type: 'string',value: 'remake',text: '备注',dictCode: ''})this.superFieldList = fieldList},handleIdentify(info) {console.log("info", this.url);let that = this;this.$confirm({title: "确认识别吗",content: "手动触发图片识别只会生成一次结果! 但视频会识别到结束",onOk: function() {let httpurl = '';let method = '';//  debugger;httpurl += that.url.identifyUrl;method = 'post';httpAction(httpurl, info, method).then((res) => {if (res.success) {that.$message.success(res.message);that.$emit('ok');} else {that.$message.warning(res.message);}}).finally(() => {that.confirmLoading = false;})}});},handleIdentifyClose(info) {let that = this;this.$confirm({title: "确认结束视频结束吗",content: "结束视频识别结果输出!",onOk: function() {let httpurl = '';let method = '';//  debugger;httpurl += that.url.identifyCloseUrl;method = 'post';httpAction(httpurl, info, method).then((res) => {if (res.success) {that.$message.success(res.message);that.$emit('ok');} else {that.$message.warning(res.message);}}).finally(() => {that.confirmLoading = false;})}});},handleOpenVideo(info){this.$router.push('livecanvas/AddressList');}}}
</script>
<style src="@assets/zwyStyle/css/main.css"></style>
<style scoped>@import '~@assets/less/common.less';/deep/ .ant-table-scroll{height: calc(100vh - 337px);}
</style>

前端就一个点击事件，主要是查看后端识别图片信息的功能。

   @Overridepublic Result<String>  startAi(TabAiModelBund tabAiModelBund, String path, String userId) {TabAiModel aiModel=modelMapper.selectById(tabAiModelBund.getModelName());if(aiModel!=null){switch (aiModel.getSpareOne()){case "1": //v3{log.info("【进入V3开始识别内容】{}",tabAiModelBund.getSpaceTwo());if(tabAiModelBund.getSpaceOne().equals("0")){ //当前为图片int a=this.saveIdentify(tabAiModelBund,path);if(a==0){return Result.OK("识别图片成功！");}}else{// 输出视频// tabAiHistoryService.saveIdentifyVideo(tabAiModelBund,uploadpath);// 输出坐标 延迟3-5s//tabAiHistoryService.saveIdentifyLocalVideo(tabAiModelBund,uploadpath,sysUser.getId());//多线程输出坐标this.saveIdentifyLocalVideoThread(tabAiModelBund,path,userId);return Result.OK("视频识别开始");}break;}case "2":{log.info("【进入V5开始识别内容】{}",tabAiModelBund.getSpaceTwo());if(tabAiModelBund.getSpaceOne().equals("0")){ //当前为图片if(tabAiModelBund.getSpaceTwo().indexOf("车牌")>-1||tabAiModelBund.getSpaceTwo().indexOf("区域")>-1){log.info("【进入车牌识别内容】{}",tabAiModelBund.getSpaceTwo());int a=this.saveCarIdentifyV5(tabAiModelBund,path);if(a==0){return Result.OK("识别图片成功！");}}else{int a=this.saveIdentifyYolov5(tabAiModelBund,path);if(a==0){return Result.OK("识别图片成功！");}}}else{// 输出视频// tabAiHistoryService.saveIdentifyVideo(tabAiModelBund,uploadpath);// 输出坐标 延迟3-5s//tabAiHistoryService.saveIdentifyLocalVideo(tabAiModelBund,uploadpath,sysUser.getId());//多线程输出坐标this.saveIdentifyLocalVideoThreadV5(tabAiModelBund,path,userId);return Result.OK("视频识别开始");}break;}//v5case "3":{  log.info("【进入V8开始识别内容】{}",tabAiModelBund.getSpaceTwo());log.info("【进入V8开始识别内容】{}",tabAiModelBund.getSpaceTwo());if(tabAiModelBund.getSpaceOne().equals("0")){ //当前为图片int a=this.saveIdentifyYolov8(tabAiModelBund,path);if(a==0){return Result.OK("识别图片成功！");}}else{// 输出视频// tabAiHistoryService.saveIdentifyVideo(tabAiModelBund,uploadpath);// 输出坐标 延迟3-5s//tabAiHistoryService.saveIdentifyLocalVideo(tabAiModelBund,uploadpath,sysUser.getId());//多线程输出坐标this.saveIdentifyLocalVideoThread(tabAiModelBund,path,userId);return Result.OK("视频识别开始");}}//v8case "4": {    break;}//jsoncase "5": {    break;}//othercase "6": { //cvlog.info("【进入cv开始识别内容】{}",tabAiModelBund.getSpaceTwo());if(tabAiModelBund.getSpaceTwo().equals("车牌识别")){int a=this.saveCarIdentify(tabAiModelBund,path);if(a==0){return Result.OK("识别图片成功！");}}break;}//jsoncase "7":{log.info("【进入OCR文字识别内容】{}",tabAiModelBund.getSpaceTwo());int a=this.saveStr(tabAiModelBund,path);if(a==0){return Result.OK("识别图片成功！");}break;}case "9":{log.info("【进入音频内容】{}",tabAiModelBund.getSpaceTwo());int a=this.saveAudioStr(tabAiModelBund,path);if(a==0){return Result.OK("识别图片成功！");}break;}}}return Result.error("识别失败未发现识别内容");}

图像识别的方法：

    /**** AI模型嵌套模型* 需要绝对路径* 输入图片*/public static String SendPicYoloV3(String weight, String cfg, String names, String picUrl, String saveName, String uploadpath) throws Exception {log.info(uploadpath);Long a=System.currentTimeMillis();// 加载类别名称List<String> classNames = Files.readAllLines(Paths.get(uploadpath+ File.separator +names));// 加载YOLOv3模型log.info("cfg地址{}",uploadpath+ File.separator +cfg);log.info("weight地址{}",uploadpath+ File.separator +weight);Net net = Dnn.readNetFromDarknet(uploadpath+ File.separator +cfg, uploadpath+ File.separator +weight);
//        net.setPreferableBackend(Dnn.DNN_BACKEND_CUDA);
//        net.setPreferableBackend(Dnn.DNN_TARGET_CUDA);net.setPreferableBackend(Dnn.DNN_BACKEND_OPENCV);net.setPreferableTarget(Dnn.DNN_TARGET_CPU);// 读取输入图像log.info("图片地址{}",uploadpath+ File.separator +picUrl);Mat image = Imgcodecs.imread(uploadpath+ File.separator +picUrl);// 将图像传递给模型进行目标检测Mat blob = Dnn.blobFromImage(image, 1.0 / 255, new Size(416, 416), new Scalar(0), true, false);net.setInput(blob);// 将图像传递给模型进行目标检测List<Mat> result = new ArrayList<>();List<String> outBlobNames = net.getUnconnectedOutLayersNames();net.forward(result, outBlobNames);// 处理检测结果float confThreshold = 0.5f;List<Rect2d> boundingBoxes = new ArrayList<>();List<Float> confidences = new ArrayList<>();List<Integer> classIds = new ArrayList<>();for (Mat level : result) {for (int i = 0; i < level.rows(); ++i) {Mat row = level.row(i);Mat scores = level.row(i).colRange(5, level.cols());Core.MinMaxLocResult minMaxLocResult = Core.minMaxLoc(scores);Point classIdPoint = minMaxLocResult.maxLoc;double confidence = row.get(0, 4)[0];if (confidence > confThreshold) {//    log.info("classIdPoint"+ classIdPoint);//    log.info("classIdPointx"+ classIdPoint.x);classIds.add((int) classIdPoint.x); //记录标签下标double centerX = row.get(0, 0)[0] * image.cols();double centerY = row.get(0, 1)[0] * image.rows();double width = row.get(0, 2)[0] * image.cols();double height = row.get(0, 3)[0] * image.rows();double left = centerX - width / 2;double top = centerY - height / 2;// 绘制边界框Rect2d rect = new Rect2d(left, top, width, height);boundingBoxes.add(rect);confidences.add((float)confidence);}}}// 执行非最大抑制，消除重复的边界框MatOfRect2d boxes = new MatOfRect2d(boundingBoxes.toArray(new Rect2d[0]));MatOfFloat confidencesMat = new MatOfFloat();confidencesMat.fromList(confidences);MatOfInt indices = new MatOfInt();Dnn.NMSBoxes(boxes, confidencesMat, confThreshold, 0.4f, indices);if(indices.empty()){log.info("类别下标啊"+"未识别到内容");return "error";}int[]   indicesArray= indices.toArray();// 获取保留的边界框log.info(confidences.size()+"类别下标啊"+indicesArray.length);// 在图像上绘制保留的边界框int c=0;for (int idx : indicesArray) {Rect2d box = boundingBoxes.get(idx);System.out.println("绘制111111"+"x:"+box.x+"y:"+ box.y+"");System.out.println("绘制11111111"+"width:"+box.width+"y:"+ box.height+"");Imgproc.rectangle(image, new Point(box.x, box.y), new Point(box.x + box.width, box.y + box.height),CommonColors(c), 2);// 添加类别标签log.info("当前有多少"+confidences.get(idx));Integer ab=classIds.get(idx);log.info("类别下标"+ab);//  AIModelYolo3.addChineseText(image, caption,new Point(box.x, box.y - 5));Imgproc.putText(image, classNames.get(ab), new Point(box.x, box.y - 5), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, CommonColors(c), 1);c++;}String savepath=uploadpath + File.separator + "temp" + File.separator;if(StringUtils.isNotBlank(saveName)){savepath+=saveName+".jpg";}else{saveName=System.currentTimeMillis()+"";savepath+=saveName+".jpg";}log.info(savepath);Imgcodecs.imwrite(savepath, image);Long b=System.currentTimeMillis();log.info("消耗时间："+(b-a));return saveName+".jpg";}

主要调用opencv中的图像方法来识别图片的内容结果。

2、视频识别

待完善。。。。。。