1.页面效果
Web 采用 flask+vue 开发，效果图如下
2.后端
import sys
import subprocess
import os
from PIL import Image
from datetime import datetime
from ASR_metrics import utils as metrics
from werkzeug.wrappers import Request, Response
from flask import Flask, render_template, request, jsonify
sys.path.append('/home/nvidia/7th_CV')
sys.path.append('/home/nvidia/7th_ASR')
# ASR 路径
pathASR = "/home/nvidia/7th_ASR"
# 项目路径
pathSky = '/home/nvidia'
app = Flask(__name__, static_folder='')
# 上传路径
uploadPath = 'uploads/'
try_model_1 = None
# 主页
@app.route('/')
def index():
return render_template('sky7.html', template_folder='templates')
# ------------------ASR------------------
# ASR 模型加载
@app.route('/asr/load')
def asrLoad():
global try_model_1
if try_model_1 == None:
import nemo.collections.asr as nemo_asr
print('Loading Nemo')
# 加载模型
try_model_1 = nemo_asr.models.EncDecCTCModel.restore_from("/home/nvidia/7th_ASR/7th_asr_model.nemo")
print('Done loading Nemo')
return 'ok'
# POST 请求上传音频
@app.route('/asr/upload', methods=['POST'])
def asrUpload():
if request.method == 'POST':
f = request.files['file']
if(f.headers.get('Content-Type') != 'audio/wav'):
return '音频格式有错误', 400
else:
fileName = f'{uploadPath}audio.wav'
f.save(fileName)
dt = datetime.now()
ts = str(int(datetime.timestamp(dt)))
return jsonify(f'/{uploadPath}audio.wav?t={ts}')
# 识别上传的音频
@app.route('/asr/identify', methods=['GET', 'POST'])
def asrIdentify():
global try_model_1
if try_model_1 == None:
return '模型无效，请重新加载', 500
try:
asr_result = try_model_1.transcribe(paths2audio_files=["uploads/audio.wav"])
s1 = request.form.get('defaultText')
s2 = " ".join(asr_result)#识别结果
result = {
"asr_result": asr_result,
"word_error_rate": metrics.calculate_cer(s1,s2),
"word_accuracy_rate":1-metrics.calculate_cer(s1,s2)
}
return jsonify(result)
except Exception as e:
return '无法识别', 400
# ------------------CV------------------
# POST 请求上传图片
@app.route("/cv/upload", methods=['POST'])
def cvUpload():
if request.method == 'POST':
f = request.files['file']
print('image', f, f.filename)
if not 'image' in f.headers.get('Content-Type'):
return '图片有错误', 400
original = f'{uploadPath}original.jpg'
try:
# Convert image to jpeg
im = Image.open(f)
rgb_im = im.convert('RGB')
rgb_im.save(original)
# Add timestamp
dt = datetime.now()
ts = str(int(datetime.timestamp(dt)))
return jsonify(original+'?t='+ts)
except Exception as e:
return '有错误', 400
# 检测图片
@app.route("/api/detect/image")
def detectImage():
cv_results = subprocess.Popen('python3 /home/nvidia/7th_CV/detection_image.py /home/nvidia/uploads/original.jpg', shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
print('code', cv_results.returncode)
cv_results = str(cv_results.stdout.read()).split('\\n')[-2]
dt = datetime.now()
ts = str(int(datetime.timestamp(dt)))
result = {
"detection_result_image_path": f'/uploads/result.jpg?t={ts}'
}
return jsonify(result)
# 获取 FPS，以 Json 格式返回前端
@app.route("/api/detect/fps")
def detectFPS():
# Code here
fps_results = subprocess.Popen('python3 /home/nvidia/7th_CV/cv_fps.py', shell = True, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
fps_results = str(fps_results.stdout.read()).split('\\n')[-2]
fps_results = fps_results.split(" ")[-1]
result = {
"detection_FPS": fps_results,
}
return jsonify(result)
# 获取 mAP
@app.route("/api/detect/map")
def detectMAP():
# Code here
map_results = subprocess.Popen('python3 /home/nvidia/7th_CV/cv_map.py', shell = True, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
map_results = str(map_results.stdout.read())
bytes(map_results, encoding="utf-8").decode()
map_results = map_results[-9:-3]
result = {
"detection_mAP": map_results,
}
return jsonify(result)
if __name__ == "__main__":
app.run(debug=True)
3.前端
3.1 html
<html>
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<head>
<script src="https://unpkg.com/vue@3"></script>
<script src="https://unpkg.com/axios/dist/axios.min.js"></script>
<link rel="stylesheet" href="//unpkg.com/layui@2.6.8/dist/css/layui.css">
<script src="//unpkg.com/layui@2.6.8/dist/layui.js"></script>
<link rel="stylesheet" type="text/css" href="/style.css">
</head>
<body id="app">
<!-- 模型加载中 -->
<div class="loading" v-if="loading!=''">
<div class="pad">%%loading%%</div>
</div>
<h1>7th Sky Hackathon</h1>
<h5>team：早八睡不醒</h5>
<div class="content">
<div class="layui-tab layui-tab-brief" lay-filter="docDemoTabBrief">
<ul class="layui-tab-title">
<li class="layui-this"><i class="layui-icon panel-title layui-icon"> ASR</i></li>
<li><i class="layui-icon panel-title layui-icon"> CV</i></li>
</ul>
<div class="layui-tab-content">
<!-- ASR 开始-->
<div class="layui-tab-item layui-show">
<div class="layui-anim layui-anim-up">
<fieldset class="asr">
<legend><span class="panel-title">ASR</span></legend>
<div class="layui-container">
<!-- 1 开始 -->
<div class="layui-row">
<div class="layui-col-md4">
1.请加载语音识别模型
</div>
<div class="layui-col-md4">
<button class="layui-btn" @click="loadModel()" v-if="!modelLoaded"><i
class="layui-icon"> 加载</i></button>
<div v-if="modelLoaded" class="modelLoaded">模型加载成功</div>
</div>
</div>
<!-- 1 结束 -->
<!-- 2、3 开始 -->
<div class="field file">
<div class="newFileUpload">
<!-- 2 开始 -->
<div class="layui-row">
<div class="layui-col-md4">
<label for="file">2.请选择音频文件</label>
<div class="note"> 仅支持 .wav 和单声道格式</div>
</div>
<div class="layui-col-md4">
<div class="userdefined-file">
<input type="text" name="userdefinedFile"
id="userdefinedFileAudio" value="未选择任何文件" />
<button type="button">选择</button>
</div>
<input type="file" name="file" id="fileAudio"
@change="handleFileUploadAudio($event)" />
</div>
</div>
<!-- 2 结束 -->
</div>
<!-- 3 开始 -->
<div class="layui-row">
<div class="layui-col-md4"> 3.请上传音频文件</div>
<div class="layui-col-md4">
<button class="layui-btn" @click="submitFile('asr')"><i
class="layui-icon"> 上传</i></button>
</div>
</div>
<!-- 3 结束 -->
</div>
<!-- 2、3 结束 -->
<!-- 4 开始 -->
<div class="layui-row">
<div class="field">
<div class="layui-col-md4">
<label>4.请试听上传语音并输入正确答案</label>
</div>
<div class="layui-col-md4">
<input id="answer" type="text" name="defaultText"
v-model="defaultText" />
</div>
</div>
</div>
<!-- 4 结束 -->
<div class="field" v-if="asrStatus=='uploaded' || asrStatus=='identified'">
<!-- 试听 开始 -->
<div class="layui-row">
<div class="audio">
<div class="layui-col-md4"> 试听 </div>
<div class="layui-col-md4">
<audio controls :src="audioOriginal"></audio>
</div>
</div>
</div>
<!-- 试听 结束 -->
<!-- 5 开始 -->
<div class="layui-row">
<div class="layui-col-md4">5.识别语音</div>
<div class="layui-col-md4">
<div class="action">
<button class="layui-btn" @click="identifyAudio()"><i
class="layui-icon"> 识别</i></button>
</div>
</div>
</div>
<!-- 5 结束 -->
</div>
<!-- 6 开始 -->
<div class="layui-row">
<div class="field result asr" v-if="asrStatus=='identified'">
<div class="layui-col-md4">6.指标</div>
<div class="layui-col-md4">
<ul>
<li v-for="(value, key) in asrResult">%%key%%: %%value%%</li>
</ul>
</div>
</div>
</div>
<!-- 6 结束 -->
</div>
</fieldset>
</div>
</div>
<!-- ASR 结束 -->
<!-- CV 开始 -->
<div class="layui-tab-item">
<div class="layui-anim layui-anim-up">
<fieldset class="cv">
<legend><span class="panel-title">CV</span></legend>
<div class="layui-container">
<!-- 1 开始 -->
<div class="field">
<div class="layui-row">
<div class="layui-col-md4">
<p>1. 获取 FPS</p>
<div class="item result"> FPS: %%cvFps%%</div>
</div>
<div class="layui-col-md4">
<div class="item action">
<button @click="getFps()" class="inline layui-btn"><i
class="layui-icon"> 获取</i></button>
</div>
</div>
</div>
</div>
<!-- 1 结束 -->
<!-- 2 开始 -->
<div class="field">
<div class="layui-row">
<div class="layui-col-md4">
<p>2. 获取 mAP</p>
<div class="item result"> mAP: %%cvMap%%</div>
</div>
<div class="layui-col-md4">
<div class="item">
<button @click="getMap()" class="inline layui-btn"><i
class="layui-icon"> 获取</i></button>
</div>
</div>
</div>
</div>
<!-- 2 结束 -->
<!-- 3、4 开始 -->
<div class="field file">
<!-- 3 开始 -->
<div class="newFileUpload">
<div class="layui-row">
<div class="layui-col-md4">
<label for="file">3.请选择图像文件 </label>
<div class="note"> </div>
</div>
<div class="layui-col-md4">
<div class="userdefined-file">
<input type="text" name="userdefinedFile"
id="userdefinedFileImage" value="未选择任何文件" />
<button type="button">选择</button>
</div>
<input type="file" name="file" ref="file" id="fileImage"
@change="handleFileUploadImage($event)" />
</div>
</div>
</div>
<!-- 3 结束 -->
<!-- 4 开始 -->
<div class="layui-row">
<div class="layui-col-md4">4.请上传图像文件 </div>
<div class="layui-col-md4">
<button @click="submitFile('cv')" class="layui-btn"><i
class="layui-icon"> 上传</i></button>
</div>
</div>
<!-- 4 结束 -->
</div>
<!-- 3、4 结束 -->
<div class="action" v-if="imageOriginal!=''">
<div class="layui-row">
<div class="layui-col-md4">5.识别图片</div>
<div class="layui-col-md4">
<button class="layui-btn" @click="identifyImage()"><i
class="layui-icon"> 识别</i></button><br>
</div>
</div>
</div>
<div class="field">
<div class="layui-row">
<div class="image original" v-if="imageOriginal!=''">
<div class="layui-col-md1">
<div class="label">原图</div>
</div>
<div class="layui-col-md3">
<image :src="imageOriginal" />
</div>
</div>
<div class="image result cv" v-if="imageResult!=''">
<div class="layui-col-md1">
<div class="label">结果图</div>
</div>
<div class="layui-col-md3">
<image :src="imageResult" />
</div>
</div>
</div>
</div>
</div>
</fieldset>
</div>
</div>
<!-- CV 结束 -->
</div>
</div>
</div>
</body>
<script>
const {
createApp
} = Vue
createApp({
data() {
return {
file: '',
defaultText: '请检测出纸箱、瓶子和果皮',
modelLoaded: false,
imageOriginal: '',
imageResult: '',
audioOriginal: '',
error: '',
asrResult: {},
cvMap: '',
cvFps: '',
loading: '',
asrStatus: 'pending',
cvStatus: 'pending'
}
},
// Avoid conflict with Flask delimiters
compilerOptions: {
delimiters: ["%%", "%%"]
},
methods: {
async loadModel() {
if (this.loading != '') return showError('在运行中，无法执行')
this.loading = '加载模型中，请耐心等待...'
this.modelLoaded = false
try {
var {
data,
status
} = await axios.get('/asr/load')
if (status == 200) {
this.modelLoaded = true
}
} catch (err) {
showError(err.response.data)
}
this.loading = ''
},
async submitFile(fileType) {
let formData = new FormData()
formData.append('file', this.file)
if (this.file == "") {
showError("请选择文件");
return false;
}
statusType = fileType + 'Status'
this.loading = '上传中...'
try {
var {
data,
status
} = await axios.post('/' + fileType + '/upload', formData, {
headers: {
'Content-Type': 'multipart/form-data'
}
})
if (status == 200) {
this[statusType] = 'uploaded'
if (fileType == 'cv') {
this.imageOriginal = data
} else {
this.audioOriginal = data
}
}
} catch (err) {
showError(err.response.data)
}
this.loading = ''
},
handleFileUploadAudio(event) {
document.getElementById("userdefinedFileAudio").value = document.getElementById("fileAudio")
.value;
this.file = event.target.files[0];
},
handleFileUploadImage(event) {
document.getElementById("userdefinedFileImage").value = document.getElementById("fileImage")
.value;
this.file = event.target.files[0];
},
async identifyAudio(event) {
// if (this.loading != '') return showError('在运行中，无法执行')
this.loading = '识别中...'
try {
let formData = new FormData()
formData.append('defaultText', this.defaultText)
console.log('t', this.defaultText)
var result = await axios.post('/asr/identify', formData)
this['asrStatus'] = 'identified'
this.asrResult = result.data
} catch (err) {
if (err.response.status == 500) this.modelLoaded = false
showError(err.response.data)
}
this.loading = ''
},
async identifyImage(event) {
if (this.loading != '') return showError('在运行中，无法执行')
this.loading = '识别中...'
this.cvStatus = 'pending'
try {
var {
data
} = await axios.get('/api/detect/image')
this.imageResult = data['detection_result_image_path']
} catch (err) {
showError(err.response.data)
}
this.loading = ''
},
async getFps(event) {
if (this.loading != '') return showError('在运行中，无法执行')
this.loading = '获取 FPS...'
try {
var {
data
} = await axios.get('/api/detect/fps')
this.cvFps = data['detection_FPS']
} catch (err) {
showError(err.response.data)
}
this.loading = ''
},
async getMap(event) {
// 接口路径: /api/detect/map
// 方式: GET
if (this.loading != '') return showError('在运行中，无法执行')
this.loading = '获取 mAP...'
try {
var {
data
} = await axios.get('/api/detect/map')
this.cvMap = data['detection_mAP']
} catch (err) {
showError(err.response.data)
}
this.loading = ''
}
}
}).mount('#app')
function showError(msg) {
layer.msg(msg || '错误')
}
</script>
</html>
3.2 CSS
body {
font-size: 20px;
margin: 0;
}
.content {
padding: 10px;
margin: 40px;
}
fieldset {
border: 1px solid #ccc;
padding: 10px;
}
.modelLoaded {
color: green;
}
.note {
color: #999;
margin: 5px 0;
font-size: 12px;
}
.field {
margin: 10px 0;
}
.action {
margin-top: 10px;
}
.loading {
position: fixed;
top: 0;
background: #E8F9D9;
text-align: center;
width: 100%;
}
.pad {
padding: 5px;
}
.inline {
display: inline-block;
}
.field .item {
margin: 5px 0;
}
.image {
display: inline-block;
margin-right: 10px;
}
.image img {
max-height: 600px;
}
/* 标题 */
h1,
h5 {
margin: 40px;
}
/* 队名 */
h5 {
margin-left: 210px;
}
/* 面板title */
.panel-title {
font-size: 25px;
}
/* 答案输入框 */
#answer {
height: 40px;
width: 200px;
font-size: 14px;
display: inline-block;
vertical-align: middle;
padding-right: 14px;
padding-left: 14px;
}
/* 音频文件选择 */
.newFileUpload {
position: relative;
height: 40px;
line-height: 40px;
}
.newFileUpload label {
display: inline-block;
}
.userdefined-file {
position: absolute;
top: 0;
/* left: 200px; */
z-index: 2;
width: 300px;
height: 40px;
line-height: 40px;
font-size: 0;
/*应对子元素为 inline-block 引起的外边距*/
}
.userdefined-file input[type="text"] {
display: inline-block;
vertical-align: middle;
padding-right: 14px;
padding-left: 14px;
width: 220px;
box-sizing: border-box;
border: 1px solid #ccc;
height: 40px;
line-height: 40px;
font-size: 14px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.userdefined-file button {
display: inline-block;
vertical-align: middle;
width: 80px;
text-align: center;
height: 40px;
line-height: 40px;
font-size: 14px;
background-color: #009688;
/* background-color: #f54; */
border: none;
color: #fff;
cursor: pointer;
}
.newFileUpload input[type="file"] {
position: absolute;
top: 0;
/* left: 200px; */
z-index: 3;
opacity: 0;
width: 300px;
height: 40px;
line-height: 40px;
cursor: pointer;
}
