鸿蒙AI功能开发【hiai引擎框架-语音识别】 基础语音服务

hiai引擎框架-语音识别

介绍

本示例展示了使用hiai引擎框架提供的语音识别能力。

本示例展示了对一段音频流转换成文字的能力展示。

需要使用hiai引擎框架文本转语音接口@kit.CoreSpeechKit.d.ts.

效果预览

使用说明：

1. 在手机的主屏幕，点击”asrDemo“，启动应用。
2. 点击“CreateEngine”，进行能力初始化。
3. 点击“startRecording”，开始识别。
4. 点击“audioTotext”，写流进行识别，需开发者准备好音频流。 若demo中采用从音频文件中读取的方式获取音频流，优先执行执行如下命令：hdc_std file send 001.pcm /data/app/el2/100/base/com.huawei.hms.asrdemo/haps/hiaiuser/files hdc_std shell chmod 777 /data/app/el2/100/base/com.huawei.hms.asrdemo/haps/hiaiuser/files/001.pcm将PCM格式的音频信息导入本demo的沙箱路径下。 点击audioTotext按钮即可从音频文件中获取音频信息并写入。
5. 点击“finish”等按钮对识别事件进行控制。
6. 点击“queryLanguagesCallback/queryLanguagesPromise”，查询支持的语种和音色。

具体实现

本示例展示了在@kit.CoreSpeechKit.d.ts定义的API：

• createEngine(createEngineParams: CreateEngineParams, callback: AsyncCallback): void;
• createEngine(createEngineParams: CreateEngineParams): Promise;
• setListener(listener: RecognizerListener): void;
• queryLanguages(params: LanguageQuery, callback: AsyncCallback): void;
• queryLanguages(params: LanguageQuery): Promise;
• startListening(params: StartParams): void;
• writeAudio(sessionId: string, audio: Uint8Array): void;
• finish(sessionId: string): void;
• cancel(sessionId: string): void;
• shutdown(): void;

业务使用时，需要先进行import导入speechRecognizer。 调用writeAudio等接口，传入想要识别的音频，得到识别结果，观察日志等。参考：

import { speechRecognizer } from '@kit.CoreSpeechKit';
import { BusinessError } from '@kit.BasicServicesKit';
import { ICapturerInterface } from './ICapturerInterface';
import FileCapturer from './FileCapturer';
import { fileIo } from '@kit.CoreFileKit';
import AudioCapturer from './AudioCapturer'

const TAG: string = 'AsrDemo';
let asrEngine: speechRecognizer.SpeechRecognitionEngine;

@Entry
@Component
struct Index {
  @State createCount: number = 0;
  @State result: boolean = false;
  @State voiceInfo: string = "";
  @State sessionId: string = "123456";
  private mFileCapturer: ICapturerInterface = new FileCapturer();
  private mAudioCapturer: ICapturerInterface = new AudioCapturer();

  build() {
    Column() {
      Scroll() {
        Column() {
          Button() {
            Text("CreateEngine")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AE7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(() => {
            this.createCount++;
            console.info(`CreateasrEngine：createCount:${this.createCount}`);
            this.createByCallback();
          })

          Button() {
            Text("CreateEngineByPromise")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AE7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(() => {
            this.createCount++;
            console.info(`CreateasrEngine：createCount:${this.createCount}`);
            this.createByPromise();
          })

          Button() {
            Text("isBusy")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AE7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(() => {
            //结束识别
            console.info(TAG, "isBusy click:-->");
            let isBusy: boolean = asrEngine.isBusy();
            console.info(TAG, "isBusy: " + isBusy);
          })

          Button() {
            Text("startRecording")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AE7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(() => {
            this.startRecording();
          })

          Button() {
            Text("audioToText")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AE7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(() => {
            this.writeAudio();
          })

          Button() {
            Text("queryLanguagesCallback")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AE7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(() => {
            this.queryLanguagesCallback();
          })

          Button() {
            Text("queryLanguagesPromise")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AE7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(() => {
            this.queryLanguagesPromise();
          })

          Button() {
            Text("finish")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AE7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(async () => {
            //结束识别
            console.info("finish click:-->");
            await this.mFileCapturer.stop();
            asrEngine.finish(this.sessionId);
          })

          Button() {
            Text("cancel")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AE7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(() => {
            //取消识别
            console.info("cancel click:-->");
            asrEngine.cancel(this.sessionId);
          })

          Button() {
            Text("shutdown")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AA7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(() => {
            //释放引擎
            asrEngine.shutdown();
          })

          Button() {
            Text("createOfErrorLanguage")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AE7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(() => {
            this.createCount++;
            console.info(`CreateasrEngine：createCount:${this.createCount}`);
            this.createOfErrorLanguage();
          })

          Button() {
            Text("createOfErrorOnline")
              .fontColor(Color.White)
              .fontSize(20)
          }
          .type(ButtonType.Capsule)
          .backgroundColor("#0x317AE7")
          .width("80%")
          .height(50)
          .margin(10)
          .onClick(() => {
            this.createCount++;
            console.info(`CreateasrEngine：createCount:${this.createCount}`);
            this.createOfErrorOnline();
          })

        }
        .layoutWeight(1)
      }
      .width('100%')
      .height('100%')

    }
  }

  //创建引擎，通过callback形式返回
  private createByCallback() {
    //设置创建引擎参数
    let extraParam: Record<string, Object> = { "locate": "CN", "recognizerMode": "short" };
    let initParamsInfo: speechRecognizer.CreateEngineParams = {
      language: 'zh-CN',
      online: 1,
      extraParams: extraParam
    };

    try {
      //调用createEngine方法
      speechRecognizer.createEngine(initParamsInfo, (err: BusinessError, speechRecognitionEngine: speechRecognizer.SpeechRecognitionEngine) => {
        if (!err) {
          console.info(TAG, 'createEngine is success');
          //接收创建引擎的实例
          asrEngine = speechRecognitionEngine;
        } else {
          //无法创建引擎时返回错误码1002200008，原因：引擎正在销毁中
          console.error("errCode: " + err.code + " errMessage: " + JSON.stringify(err.message));
        }
      });
    } catch (error) {
      let message = (error as BusinessError).message;
      let code = (error as BusinessError).code;
      console.error(`createEngine failed, error code: ${code}, message: ${message}.`)
    }
  }

  //创建引擎，通过promise形式返回
  private createByPromise() {
    //设置创建引擎参数
    let extraParam: Record<string, Object> = { "locate": "CN", "recognizerMode": "short" };
    let initParamsInfo: speechRecognizer.CreateEngineParams = {
      language: 'zh-CN',
      online: 1,
      extraParams: extraParam
    };

    //调用createEngine方法
    speechRecognizer.createEngine(initParamsInfo)
      .then((speechRecognitionEngine: speechRecognizer.SpeechRecognitionEngine) => {
        asrEngine = speechRecognitionEngine;
        console.info('result:' + JSON.stringify(speechRecognitionEngine));
      })
      .catch((err: BusinessError) => {
        console.error('result' + JSON.stringify(err));
      });
  }

  //查询语种信息，以callback形式返回
  private queryLanguagesCallback() {
    //设置查询相关参数
    let languageQuery: speechRecognizer.LanguageQuery = {
      sessionId: '123456'
    };
    //调用listLanguages方法
    try {
      console.info(TAG, 'listLanguages  start');
      asrEngine.listLanguages(languageQuery, (err: BusinessError, languages: Array<string>) => {
        if (!err) {
          //接收目前支持的语种信息
          console.info(TAG, 'listLanguages  result: ' + JSON.stringify(languages));
        } else {
          console.error("errCode is " + JSON.stringify(err));
        }
      });
    } catch (error) {
      let message = (error as BusinessError).message;
      let code = (error as BusinessError).code;
      console.error(`listLanguages failed, error code: ${code}, message: ${message}.`)
    }

  };

  //查询语种信息，以promise返回
  private queryLanguagesPromise() {
    //设置查询相关的参数
    let languageQuery: speechRecognizer.LanguageQuery = {
      sessionId: '123456'
    };
    //调用listLanguages方法
    asrEngine.listLanguages(languageQuery).then((res: Array<string>) => {
      console.info('voiceInfo:' + JSON.stringify(res));
    }).catch((err: BusinessError) => {
      console.error('error' + JSON.stringify(err));
    });
  }

  //录音转文本
  private async startRecording() {
    this.setListener();
    //设置开始识别的相关参数
    let audioParam: speechRecognizer.AudioInfo = { audioType: 'pcm', sampleRate: 16000, soundChannel: 1, sampleBit: 16 }
    let extraParam: Record<string, Object> = {
      "recognitionMode": 0,
      "vadBegin": 2000,
      "vadEnd": 3000,
      "maxAudioDuration": 20000
    }
    let recognizerParams: speechRecognizer.StartParams = {
      sessionId: this.sessionId,
      audioInfo: audioParam,
      extraParams: extraParam
    }
    //调用开始识别方法
    console.info(TAG, 'startListening start');
    asrEngine.startListening(recognizerParams);

    //录音获取音频
    let data: ArrayBuffer;
    this.mFileCapturer = this.mAudioCapturer;
    console.info(TAG, 'create capture success');
    this.mFileCapturer.init((dataBuffer: ArrayBuffer) => {
      console.info(TAG, 'start write');
      console.info(TAG, 'ArrayBuffer ' + JSON.stringify(dataBuffer));
      data = dataBuffer
      let unit8Array: Uint8Array = new Uint8Array(data);
      console.info(TAG, 'ArrayBuffer unit8Array ' + JSON.stringify(unit8Array));
      //写入音频流
      asrEngine.writeAudio("123456", unit8Array);
    });
    // await this.mFileCapturer.start();
  };

  //音频转文本
  private async writeAudio() {
    this.setListener();
    //设置开始识别的相关参数
    let audioParam: speechRecognizer.AudioInfo = { audioType: 'pcm', sampleRate: 16000, soundChannel: 1, sampleBit: 16 }
    let recognizerParams: speechRecognizer.StartParams = {
      sessionId: this.sessionId,
      audioInfo: audioParam
    }
    //调用开始识别方法
    asrEngine.startListening(recognizerParams);

    //文件中获取音频
    let data: ArrayBuffer;
    let ctx = getContext(this);
    let filenames: string[] = fileIo.listFileSync(ctx.filesDir);
    if (filenames.length <= 0) {
      console.error('length is null');
      return;
    }
    let filePath: string = ctx.filesDir + '/' + filenames[0];
    (this.mFileCapturer as FileCapturer).setFilePath(filePath); // 文件
    this.mFileCapturer.init((dataBuffer: ArrayBuffer) => {
      data = dataBuffer
      let unit8Array: Uint8Array = new Uint8Array(data);
      asrEngine.writeAudio("123456", unit8Array);
    });
    await this.mFileCapturer.start();
  }

  //设置回调
  private setListener() {
    //创建回调对象
    let setListener: speechRecognizer.RecognitionListener = {
      //开始识别成功回调
      onStart(sessionId: string, eventMessage: string) {
        console.info(TAG, "setListener onStart sessionId: " + sessionId + "eventMessage: " + eventMessage);
      },
      //事件回调
      onEvent(sessionId: string, eventCode: number, eventMessage: string) {
        console.info(TAG, "setListener onEvent sessionId: " + sessionId + "eventMessage: " + eventCode + "eventMessage: " + eventMessage);
      },
      //识别结果回调，包括中间结果和最终结果
      onResult(sessionId: string, res: speechRecognizer.SpeechRecognitionResult) {
        let isFinal: boolean = res.isFinal;
        let isLast: boolean = res.isLast;
        let result: string = res.result;
        console.info('setListener onResult: ' + 'sessionId: ' + sessionId + ' isFinal: ' + isFinal + ' isLast: ' + isLast + ' result: ' + result);
      },
      //识别完成回调
      onComplete(sessionId: string, eventMessage: string) {
        console.info(TAG, "setListener onComplete sessionId: " + sessionId + "eventMessage: " + eventMessage);
      },
      //错误回调，错误码通过本方法返回
      //返回错误码1002200002，开始识别失败，重复启动startListening方法时触发
      //返回错误码1002200003，输入的音频超过支持的最大音频长度
      //返回错误码1002200004，结束识别失败，当前没有正在识别的任务时调用finish方法触发
      //返回错误码1002200005，取消识别失败，当前没有正在识别的任务时调用cancel方法触发
      //返回错误码1002200006，识别引擎正忙，引擎正在识别中
      //返回错误码1002200007，引擎未初始化时调用startListening、writeAudio、finish、cancel等方法时触发
      onError(sessionId: string, errorCode: number, errorMessage: string) {
        console.error(TAG, "setListener onError sessionId: " + sessionId + "errorCode: " + errorCode + "errorMessage: " + errorMessage);
      }
    }
    //调用回调方法
    asrEngine.setListener(setListener);
  };

  //使用不支持的语种创建引擎，返回错误码返回错误码1002200001，语种不支持
  private createOfErrorLanguage() {
    //设置创建引擎参数
    let initParamsInfo: speechRecognizer.CreateEngineParams = {
      //不支持的语种
      language: 'zh-CNX',
      online: 1
    };
    try {
      //调用createEngine方法
      speechRecognizer.createEngine(initParamsInfo, (err: BusinessError, speechRecognitionEngine: speechRecognizer.SpeechRecognitionEngine) => {
        if (!err) {
          console.info(TAG, 'createEngine is success');
          //接受创建引擎的实例
          asrEngine = speechRecognitionEngine;
        } else {
          //返回错误码1002200001，语种不支持，初始化失败
          console.error("errCode: " + err.code + " errMessage: " + JSON.stringify(err.message));
        }
      });
    } catch (error) {
      let message = (error as BusinessError).message;
      let code = (error as BusinessError).code;
      console.error(`createEngine failed, error code: ${code}, message: ${message}.`)
    }
  }

  //使用不支持的模式创建引擎，返回错误码1002200001，模式不支持初始化失败
  private createOfErrorOnline() {
    //设置创建引擎参数
    let initParamsInfo: speechRecognizer.CreateEngineParams = {
      language: 'zh-CN',
      online: 2
    };
    try {
      //调用createEngine方法
      speechRecognizer.createEngine(initParamsInfo, (err: BusinessError, speechRecognitionEngine: speechRecognizer.SpeechRecognitionEngine) => {
        if (!err) {
          console.info(TAG, 'createEngine is success');
          //接受创建引擎的实例
          asrEngine = speechRecognitionEngine;
        } else {
          //返回错误码1002200001，模式不支持，初始化失败
          console.error("errCode: " + err.code + " errMessage: " + JSON.stringify(err.message));
        }
      });
    } catch (error) {
      let message = (error as BusinessError).message;
      let code = (error as BusinessError).code;
      console.error(`createEngine failed, error code: ${code}, message: ${message}.`)
    }
  }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465