hiai引擎框架-语音识别
介绍
本示例展示了使用hiai引擎框架提供的语音识别能力。
本示例展示了对一段音频流转换成文字的能力展示。
需要使用hiai引擎框架文本转语音接口@kit.CoreSpeechKit.d.ts.
效果预览
使用说明:
- 在手机的主屏幕,点击”asrDemo“,启动应用。
- 点击“CreateEngine”,进行能力初始化。
- 点击“startRecording”,开始识别。
- 点击“audioTotext”,写流进行识别,需开发者准备好音频流。 若demo中采用从音频文件中读取的方式获取音频流,优先执行执行如下命令:hdc_std file send 001.pcm /data/app/el2/100/base/com.huawei.hms.asrdemo/haps/hiaiuser/files hdc_std shell chmod 777 /data/app/el2/100/base/com.huawei.hms.asrdemo/haps/hiaiuser/files/001.pcm将PCM格式的音频信息导入本demo的沙箱路径下。 点击audioTotext按钮即可从音频文件中获取音频信息并写入。
- 点击“finish”等按钮对识别事件进行控制。
- 点击“queryLanguagesCallback/queryLanguagesPromise”,查询支持的语种和音色。
具体实现
本示例展示了在@kit.CoreSpeechKit.d.ts定义的API:
- createEngine(createEngineParams: CreateEngineParams, callback: AsyncCallback): void;
- createEngine(createEngineParams: CreateEngineParams): Promise;
- setListener(listener: RecognizerListener): void;
- queryLanguages(params: LanguageQuery, callback: AsyncCallback): void;
- queryLanguages(params: LanguageQuery): Promise;
- startListening(params: StartParams): void;
- writeAudio(sessionId: string, audio: Uint8Array): void;
- finish(sessionId: string): void;
- cancel(sessionId: string): void;
- shutdown(): void;
业务使用时,需要先进行import导入speechRecognizer。 调用writeAudio等接口,传入想要识别的音频,得到识别结果,观察日志等。参考:
import { speechRecognizer } from '@kit.CoreSpeechKit';
import { BusinessError } from '@kit.BasicServicesKit';
import { ICapturerInterface } from './ICapturerInterface';
import FileCapturer from './FileCapturer';
import { fileIo } from '@kit.CoreFileKit';
import AudioCapturer from './AudioCapturer'
const TAG: string = 'AsrDemo';
let asrEngine: speechRecognizer.SpeechRecognitionEngine;
@Entry
@Component
struct Index {
@State createCount: number = 0;
@State result: boolean = false;
@State voiceInfo: string = "";
@State sessionId: string = "123456";
private mFileCapturer: ICapturerInterface = new FileCapturer();
private mAudioCapturer: ICapturerInterface = new AudioCapturer();
build() {
Column() {
Scroll() {
Column() {
Button() {
Text("CreateEngine")
.fontColor(Color.White)
.fontSize(20)
}
.type(ButtonType.Capsule)
.backgroundColor("#0x317AE7")
.width("80%")
.height(50)
.margin(10)
.onClick(() => {
this.createCount++;
console.info(`CreateasrEngine:createCount:${this.createCount}`);
this.createByCallback();
})
Button() {
Text("CreateEngineByPromise")
.fontColor(Color.White)
.fontSize(20)
}
.type(ButtonType.Capsule)
.backgroundColor("#0x317AE7")
.width("80%")
.height(50)
.margin(10)
.onClick(() => {
this.createCount++;
console.info(`CreateasrEngine:createCount:${this.createCount}`);
this.createByPromise();
})
Button() {
Text("isBusy")
.fontColor(Color.White)
.fontSize(20)
}
.type(ButtonType.Capsule)
.backgroundColor("#0x317AE7")
.width("80%")
.height(50)
.margin(10)
.onClick(() => {
//结束识别
console.info(TAG, "isBusy click:-->");
let isBusy: boolean = asrEngine.isBusy();
console.info(TAG, "isBusy: " + isBusy);
})
Button() {
Text("startRecording")
.fontColor(Color.White)
.fontSize(20)
}
.type(ButtonType.Capsule)
.backgroundColor("#0x317AE7")
.width("80%")
.height(50)
.margin(10)
.onClick(() => {
this.startRecording();
})
Button() {
Text("audioToText")
.fontColor(Color.White)
.fontSize(20)
}
.type(ButtonType.Capsule)
.backgroundColor("#0x317AE7")
.width("80%")
.height(50)
.margin(10)
.onClick(() => {
this.writeAudio();
})
Button() {
Text("queryLanguagesCallback")
.fontColor(Color.White)
.fontSize(20)
}
.type(ButtonType.Capsule)
.backgroundColor("#0x317AE7")
.width("80%")
.height(50)
.margin(10)
.onClick(() => {
this.queryLanguagesCallback();
})
Button() {
Text("queryLanguagesPromise")
.fontColor(Color.White)
.fontSize(20)
}
.type(ButtonType.Capsule)
.backgroundColor("#0x317AE7")
.width("80%")
.height(50)
.margin(10)
.onClick(() => {
this.queryLanguagesPromise();
})
Button() {
Text("finish")
.fontColor(Color.White)
.fontSize(20)
}
.type(ButtonType.Capsule)
.backgroundColor("#0x317AE7")
.width("80%")
.height(50)
.margin(10)
.onClick(async () => {
//结束识别
console.info("finish click:-->");
await this.mFileCapturer.stop();
asrEngine.finish(this.sessionId);
})
Button() {
Text("cancel")
.fontColor(Color.White)
.fontSize(20)
}
.type(ButtonType.Capsule)
.backgroundColor("#0x317AE7")
.width("80%")
.height(50)
.margin(10)
.onClick(() => {
//取消识别
console.info("cancel click:-->");
asrEngine.cancel(this.sessionId);
})
Button() {
Text("shutdown")
.fontColor(Color.White)
.fontSize(20)
}
.type(ButtonType.Capsule)
.backgroundColor("#0x317AA7")
.width("80%")
.height(50)
.margin(10)
.onClick(() => {
//释放引擎
asrEngine.shutdown();
})
Button() {
Text("createOfErrorLanguage")
.fontColor(Color.White)
.fontSize(20)
}
.type(ButtonType.Capsule)
.backgroundColor("#0x317AE7")
.width("80%")
.height(50)
.margin(10)
.onClick(() => {
this.createCount++;
console.info(`CreateasrEngine:createCount:${this.createCount}`);
this.createOfErrorLanguage();
})
Button() {
Text("createOfErrorOnline")
.fontColor(Color.White)
.fontSize(20)
}
.type(ButtonType.Capsule)
.backgroundColor("#0x317AE7")
.width("80%")
.height(50)
.margin(10)
.onClick(() => {
this.createCount++;
console.info(`CreateasrEngine:createCount:${this.createCount}`);
this.createOfErrorOnline();
})
}
.layoutWeight(1)
}
.width('100%')
.height('100%')
}
}
//创建引擎,通过callback形式返回
private createByCallback() {
//设置创建引擎参数
let extraParam: Record<string, Object> = { "locate": "CN", "recognizerMode": "short" };
let initParamsInfo: speechRecognizer.CreateEngineParams = {
language: 'zh-CN',
online: 1,
extraParams: extraParam
};
try {
//调用createEngine方法
speechRecognizer.createEngine(initParamsInfo, (err: BusinessError, speechRecognitionEngine: speechRecognizer.SpeechRecognitionEngine) => {
if (!err) {
console.info(TAG, 'createEngine is success');
//接收创建引擎的实例
asrEngine = speechRecognitionEngine;
} else {
//无法创建引擎时返回错误码1002200008,原因:引擎正在销毁中
console.error("errCode: " + err.code + " errMessage: " + JSON.stringify(err.message));
}
});
} catch (error) {
let message = (error as BusinessError).message;
let code = (error as BusinessError).code;
console.error(`createEngine failed, error code: ${code}, message: ${message}.`)
}
}
//创建引擎,通过promise形式返回
private createByPromise() {
//设置创建引擎参数
let extraParam: Record<string, Object> = { "locate": "CN", "recognizerMode": "short" };
let initParamsInfo: speechRecognizer.CreateEngineParams = {
language: 'zh-CN',
online: 1,
extraParams: extraParam
};
//调用createEngine方法
speechRecognizer.createEngine(initParamsInfo)
.then((speechRecognitionEngine: speechRecognizer.SpeechRecognitionEngine) => {
asrEngine = speechRecognitionEngine;
console.info('result:' + JSON.stringify(speechRecognitionEngine));
})
.catch((err: BusinessError) => {
console.error('result' + JSON.stringify(err));
});
}
//查询语种信息,以callback形式返回
private queryLanguagesCallback() {
//设置查询相关参数
let languageQuery: speechRecognizer.LanguageQuery = {
sessionId: '123456'
};
//调用listLanguages方法
try {
console.info(TAG, 'listLanguages start');
asrEngine.listLanguages(languageQuery, (err: BusinessError, languages: Array<string>) => {
if (!err) {
//接收目前支持的语种信息
console.info(TAG, 'listLanguages result: ' + JSON.stringify(languages));
} else {
console.error("errCode is " + JSON.stringify(err));
}
});
} catch (error) {
let message = (error as BusinessError).message;
let code = (error as BusinessError).code;
console.error(`listLanguages failed, error code: ${code}, message: ${message}.`)
}
};
//查询语种信息,以promise返回
private queryLanguagesPromise() {
//设置查询相关的参数
let languageQuery: speechRecognizer.LanguageQuery = {
sessionId: '123456'
};
//调用listLanguages方法
asrEngine.listLanguages(languageQuery).then((res: Array<string>) => {
console.info('voiceInfo:' + JSON.stringify(res));
}).catch((err: BusinessError) => {
console.error('error' + JSON.stringify(err));
});
}
//录音转文本
private async startRecording() {
this.setListener();
//设置开始识别的相关参数
let audioParam: speechRecognizer.AudioInfo = { audioType: 'pcm', sampleRate: 16000, soundChannel: 1, sampleBit: 16 }
let extraParam: Record<string, Object> = {
"recognitionMode": 0,
"vadBegin": 2000,
"vadEnd": 3000,
"maxAudioDuration": 20000
}
let recognizerParams: speechRecognizer.StartParams = {
sessionId: this.sessionId,
audioInfo: audioParam,
extraParams: extraParam
}
//调用开始识别方法
console.info(TAG, 'startListening start');
asrEngine.startListening(recognizerParams);
//录音获取音频
let data: ArrayBuffer;
this.mFileCapturer = this.mAudioCapturer;
console.info(TAG, 'create capture success');
this.mFileCapturer.init((dataBuffer: ArrayBuffer) => {
console.info(TAG, 'start write');
console.info(TAG, 'ArrayBuffer ' + JSON.stringify(dataBuffer));
data = dataBuffer
let unit8Array: Uint8Array = new Uint8Array(data);
console.info(TAG, 'ArrayBuffer unit8Array ' + JSON.stringify(unit8Array));
//写入音频流
asrEngine.writeAudio("123456", unit8Array);
});
// await this.mFileCapturer.start();
};
//音频转文本
private async writeAudio() {
this.setListener();
//设置开始识别的相关参数
let audioParam: speechRecognizer.AudioInfo = { audioType: 'pcm', sampleRate: 16000, soundChannel: 1, sampleBit: 16 }
let recognizerParams: speechRecognizer.StartParams = {
sessionId: this.sessionId,
audioInfo: audioParam
}
//调用开始识别方法
asrEngine.startListening(recognizerParams);
//文件中获取音频
let data: ArrayBuffer;
let ctx = getContext(this);
let filenames: string[] = fileIo.listFileSync(ctx.filesDir);
if (filenames.length <= 0) {
console.error('length is null');
return;
}
let filePath: string = ctx.filesDir + '/' + filenames[0];
(this.mFileCapturer as FileCapturer).setFilePath(filePath); // 文件
this.mFileCapturer.init((dataBuffer: ArrayBuffer) => {
data = dataBuffer
let unit8Array: Uint8Array = new Uint8Array(data);
asrEngine.writeAudio("123456", unit8Array);
});
await this.mFileCapturer.start();
}
//设置回调
private setListener() {
//创建回调对象
let setListener: speechRecognizer.RecognitionListener = {
//开始识别成功回调
onStart(sessionId: string, eventMessage: string) {
console.info(TAG, "setListener onStart sessionId: " + sessionId + "eventMessage: " + eventMessage);
},
//事件回调
onEvent(sessionId: string, eventCode: number, eventMessage: string) {
console.info(TAG, "setListener onEvent sessionId: " + sessionId + "eventMessage: " + eventCode + "eventMessage: " + eventMessage);
},
//识别结果回调,包括中间结果和最终结果
onResult(sessionId: string, res: speechRecognizer.SpeechRecognitionResult) {
let isFinal: boolean = res.isFinal;
let isLast: boolean = res.isLast;
let result: string = res.result;
console.info('setListener onResult: ' + 'sessionId: ' + sessionId + ' isFinal: ' + isFinal + ' isLast: ' + isLast + ' result: ' + result);
},
//识别完成回调
onComplete(sessionId: string, eventMessage: string) {
console.info(TAG, "setListener onComplete sessionId: " + sessionId + "eventMessage: " + eventMessage);
},
//错误回调,错误码通过本方法返回
//返回错误码1002200002,开始识别失败,重复启动startListening方法时触发
//返回错误码1002200003,输入的音频超过支持的最大音频长度
//返回错误码1002200004,结束识别失败,当前没有正在识别的任务时调用finish方法触发
//返回错误码1002200005,取消识别失败,当前没有正在识别的任务时调用cancel方法触发
//返回错误码1002200006,识别引擎正忙,引擎正在识别中
//返回错误码1002200007,引擎未初始化时调用startListening、writeAudio、finish、cancel等方法时触发
onError(sessionId: string, errorCode: number, errorMessage: string) {
console.error(TAG, "setListener onError sessionId: " + sessionId + "errorCode: " + errorCode + "errorMessage: " + errorMessage);
}
}
//调用回调方法
asrEngine.setListener(setListener);
};
//使用不支持的语种创建引擎,返回错误码返回错误码1002200001,语种不支持
private createOfErrorLanguage() {
//设置创建引擎参数
let initParamsInfo: speechRecognizer.CreateEngineParams = {
//不支持的语种
language: 'zh-CNX',
online: 1
};
try {
//调用createEngine方法
speechRecognizer.createEngine(initParamsInfo, (err: BusinessError, speechRecognitionEngine: speechRecognizer.SpeechRecognitionEngine) => {
if (!err) {
console.info(TAG, 'createEngine is success');
//接受创建引擎的实例
asrEngine = speechRecognitionEngine;
} else {
//返回错误码1002200001,语种不支持,初始化失败
console.error("errCode: " + err.code + " errMessage: " + JSON.stringify(err.message));
}
});
} catch (error) {
let message = (error as BusinessError).message;
let code = (error as BusinessError).code;
console.error(`createEngine failed, error code: ${code}, message: ${message}.`)
}
}
//使用不支持的模式创建引擎,返回错误码1002200001,模式不支持初始化失败
private createOfErrorOnline() {
//设置创建引擎参数
let initParamsInfo: speechRecognizer.CreateEngineParams = {
language: 'zh-CN',
online: 2
};
try {
//调用createEngine方法
speechRecognizer.createEngine(initParamsInfo, (err: BusinessError, speechRecognitionEngine: speechRecognizer.SpeechRecognitionEngine) => {
if (!err) {
console.info(TAG, 'createEngine is success');
//接受创建引擎的实例
asrEngine = speechRecognitionEngine;
} else {
//返回错误码1002200001,模式不支持,初始化失败
console.error("errCode: " + err.code + " errMessage: " + JSON.stringify(err.message));
}
});
} catch (error) {
let message = (error as BusinessError).message;
let code = (error as BusinessError).code;
console.error(`createEngine failed, error code: ${code}, message: ${message}.`)
}
}
}