devbox
秋刀鱼在做梦
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

Unity对接科大讯飞实时语音转写WebAPI（Windows平台）（一）_unity webgl对接讯飞实时语音听写

作者：秋刀鱼在做梦 | 2024-07-30 18:04:28

赞

踩

unity webgl对接讯飞实时语音听写

科大讯飞官方文档：实时语音转写 API 文档 | 讯飞开放平台文档中心 (xfyun.cn)

参考文章：unity通过WebAPI连接Websocket实现讯飞语音识别与合成。_unity websocket audio-CSDN博客

        要实现语音转文字。首先我们需要从麦克风获取到语音数据，这里用到了Microphone类，Unity自带；其次，需要将语音数据发送给讯飞，这里用到的是WebSocketSharp.WebSocket，用习惯了。然后就是按照文档一步步踩坑了。

        直接贴代码了。代码主要实现握手阶段参数签名，实时通信阶段的数据传输以及结果解析。

  1. using System.Collections;
  2. using System.Collections.Generic;
  3. using UnityEngine;
  4. using System;
  5. using WebSocketSharp;
  6. using System.Text;
  7. using System.Security.Cryptography;
  8. using LitJson;
  9. using Newtonsoft.Json;
  10.  
  11. public class SpeechHelper : MonoBehaviour
  12. {
  13.     public event Action<string> 语音识别完成事件;   //语音识别回调事件
  14.     public AudioClip RecordedClip;
  15.     private string micphoneName = string.Empty;
  16.     WebSocket speechWebSocket;
  17.     private System.Action<string> resultCallback;
  18.     public void InitSpeechHelper(System.Action<string> textCallback)
  19.     {
  20.         resultCallback = textCallback;
  21.     }
  22.     public void StartSpeech()
  23.     {
  24.         if (speechWebSocket != null && speechWebSocket.ReadyState == WebSocketState.Open)
  25.         {
  26.             Debug.LogWarning("开始语音识别失败！，等待上次识别连接结束");
  27.             return;
  28.         }
  29.         if(Microphone.devices.Length <= 0)
  30.         {
  31.             Debug.LogWarning("找不到麦克风");
  32.             return;
  33.         }
  34.         messageQueue.Clear();
  35.         micphoneName = Microphone.devices[0];
  36.         Debug.Log("micphoneName:" + micphoneName);
  37.         try
  38.         {
  39.             RecordedClip = Microphone.Start(micphoneName, false, 60, 16000);
  40.             ConnectSpeechWebSocket();
  41.         }
  42.         catch(Exception ex)
  43.         {
  44.             Debug.LogError(ex.Message);
  45.         }
  46.     }
  47.  
  48.     public void StopSpeech()
  49.     {
  50.         Microphone.End(micphoneName);
  51.         Debug.Log("识别结束，停止录音");
  52.     }
  53.  
  54.     void ConnectSpeechWebSocket()
  55.     {
  56.         try
  57.         {
  58.             speechWebSocket = new WebSocket(GetWebSocketUrl());
  59.         }
  60.         catch (Exception ex)
  61.         {
  62.             UnityEngine.Debug.LogError(ex.Message);
  63.             return;
  64.         }
  65.  
  66.         speechWebSocket.OnOpen += (sender, e) =>
  67.         {
  68.             Debug.Log("OnOpen");
  69.             speechWebSocket.OnClose += OnWebSocketClose;
  70.         };
  71.         speechWebSocket.OnMessage += OnInitMessage;
  72.         speechWebSocket.OnError += OnError;
  73.         speechWebSocket.ConnectAsync();
  74.         StartCoroutine(SendVoiceData());
  75.     }
  76.     void OnWebSocketClose(object sender, CloseEventArgs e)
  77.     {
  78.         Debug.Log("OnWebSocketClose");
  79.     }
  80.     private static Queue<string> messageQueue = new Queue<string>();
  81.     void OnInitMessage(object sender, MessageEventArgs e)
  82.     {
  83.         UnityEngine.Debug.Log("qqqqqqqqqqqqqWebSocket数据返回：" + e.Data);
  84.         messageQueue.Enqueue(e.Data);
  85.     }
  86.     private void MainThreadOnMessage(string message)
  87.     {
  88.         try
  89.         {
  90.             XFResponse response = JsonConvert.DeserializeObject<XFResponse>(message);
  91.             if (0 != response.code)
  92.             {
  93.                 
  94.                 return;
  95.             }
  96.             if (response.action.Equals("result"))
  97.             {
  98.                 var result = ParseXunfeiRecognitionResult(response.data);
  99.                 if(result.IsFinal)
  100.                 {
  101.                     Debug.Log("Text最终:" + result.Text);
  102.                     resultCallback?.Invoke(result.Text);
  103.                 }else
  104.                 {
  105.                     Debug.Log("Text中间:" + result.Text);
  106.                 }
  107.             }
  108.         }
  109.         catch (Exception ex)
  110.         {
  111.             Debug.LogError(ex.Message);
  112.         }
  113.     }
  114.     void OnError(object sender, ErrorEventArgs e)
  115.     {
  116.         UnityEngine.Debug.Log("WebSoclet:发生错误:" + e.Message);
  117.  
  118.     }
  119.  
  120.     public SpeechRecognitionResult ParseXunfeiRecognitionResult(string dataJson)
  121.     {
  122.         StringBuilder builder = new StringBuilder();
  123.         SpeechRecognitionResult res = new SpeechRecognitionResult();
  124.         try
  125.         {
  126.             JsonData data = JsonMapper.ToObject(dataJson);
  127.             JsonData cn = data["cn"];
  128.             JsonData st = cn["st"];
  129.             if (st["ed"].ToString().Equals("0"))
  130.             {
  131.                 res.IsFinal = false;
  132.             }
  133.             else
  134.             {
  135.                 res.IsFinal = true;
  136.             }
  137.             JsonData rtArry = st["rt"];
  138.             foreach (JsonData rtObject in rtArry)
  139.             {
  140.                 JsonData wsArr = rtObject["ws"];
  141.                 foreach (JsonData wsObject in wsArr)
  142.                 {
  143.                     JsonData cwArr = wsObject["cw"];
  144.                     foreach (JsonData cwObject in cwArr)
  145.                     {
  146.                         builder.Append(cwObject["w"].ToString());
  147.                     }
  148.                 }
  149.             }
  150.         }catch(Exception ex)
  151.         {
  152.             Debug.LogError(ex.Message);
  153.         }
  154.         res.Text = builder.ToString();
  155.         return res;
  156.     }
  157.  
  158.     void SendData(byte[] voiceData)
  159.     {
  160.         Debug.Log("SendData:" + voiceData.Length + ",time:" + Time.realtimeSinceStartup);
  161.         if (speechWebSocket.ReadyState != WebSocketState.Open)
  162.         {
  163.             return;
  164.         }
  165.         try
  166.         {
  167.             if (speechWebSocket != null && speechWebSocket.IsAlive)
  168.             {
  169.                 speechWebSocket.SendAsync(voiceData, success =>
  170.                 {
  171.                     if (success)
  172.                     {
  173.                         UnityEngine.Debug.Log("WebSoclet:发送成功：" + voiceData.Length);
  174.                     }
  175.                     else
  176.                     {
  177.                         UnityEngine.Debug.Log("WebSoclet:发送失败：");
  178.                     }
  179.                 });
  180.             }
  181.         }
  182.         catch
  183.         {
  184.  
  185.         }
  186.     }
  187.     void SendEndMsg(System.Action callback)
  188.     {
  189.         string endMsg = "{\"end\": true}";
  190.         byte[] data = Encoding.UTF8.GetBytes(endMsg);
  191.         try
  192.         {
  193.             if (speechWebSocket != null && speechWebSocket.IsAlive)
  194.             {
  195.                 speechWebSocket.SendAsync(data, success =>
  196.                 {
  197.                     if (success)
  198.                     {
  199.                         UnityEngine.Debug.Log("WebSoclet:发送END成功：" + data.Length);
  200.                     }
  201.                     else
  202.                     {
  203.                         UnityEngine.Debug.Log("WebSoclet:发送END失败：");
  204.                     }
  205.                     callback?.Invoke();
  206.                 });
  207.             }
  208.         }
  209.         catch
  210.         {
  211.  
  212.         }
  213.     }
  214.  
  215.     IEnumerator SendVoiceData()
  216.     {
  217.         yield return new WaitUntil(()=> (speechWebSocket.ReadyState == WebSocketState.Open));
  218.         yield return new WaitWhile(() => Microphone.GetPosition(micphoneName) <= 0);
  219.         float t = 0;
  220.         int position = Microphone.GetPosition(micphoneName);
  221.         const float waitTime = 0.04f;//每隔40ms发送音频
  222.         int lastPosition = 0;
  223.         const int Maxlength = 640;//最大发送长度
  224.         //Debug.Log("position:" + position + ",samples:" + RecordedClip.samples);
  225.         while (position < RecordedClip.samples && speechWebSocket.ReadyState == WebSocketState.Open)
  226.         {
  227.             t += waitTime;
  228.             yield return new WaitForSecondsRealtime(waitTime);
  229.             if (Microphone.IsRecording(micphoneName)) position = Microphone.GetPosition(micphoneName);
  230.             //Debug.Log("录音时长：" + t + "position=" + position + ",lastPosition=" + lastPosition);
  231.             if (position <= lastPosition)
  232.             {
  233.                 Debug.LogWarning("字节流发送完毕！强制结束！");
  234.                 break;
  235.             }
  236.             int length = position - lastPosition > Maxlength ? Maxlength : position - lastPosition;
  237.             byte[] date = GetClipData(lastPosition, length, RecordedClip);
  238.             SendData(date);
  239.             lastPosition = lastPosition + length;
  240.         }
  241.         yield return new WaitForSecondsRealtime(waitTime);
  242.         SendEndMsg(null);
  243.         Microphone.End(micphoneName);
  244.     }
  245.     public byte[] GetClipData(int star, int length, AudioClip recordedClip)
  246.     {
  247.         float[] soundata = new float[length];
  248.         recordedClip.GetData(soundata, star);
  249.         int rescaleFactor = 32767;
  250.         byte[] outData = new byte[soundata.Length * 2];
  251.         for (int i = 0; i < soundata.Length; i++)
  252.         {
  253.             short temshort = (short)(soundata[i] * rescaleFactor);
  254.             byte[] temdata = BitConverter.GetBytes(temshort);
  255.             outData[i * 2] = temdata[0];
  256.             outData[i * 2 + 1] = temdata[1];
  257.         }
  258.         return outData;
  259.     }
  260.  
  261.     private string GetWebSocketUrl()
  262.     {
  263.         string appid = "appid";
  264.         string ts = GetCurrentUnixTimestampMillis().ToString();
  265.         string baseString = appid + ts;
  266.         string md5 = GetMD5Hash(baseString);
  267.         UnityEngine.Debug.Log("baseString:" + baseString + ",md5:" + md5);
  268.         string sha1 = CalculateHmacSha1(md5, "appkey");
  269.         string signa = sha1;
  270.         string url = string.Format("ws://rtasr.xfyun.cn/v1/ws?appid={0}&ts={1}&signa={2}", appid, ts, signa);
  271.         UnityEngine.Debug.Log(url);
  272.         return url;
  273.     }
  274.     private long GetCurrentUnixTimestampMillis()
  275.     {
  276.         DateTime unixStartTime = new DateTime(1970, 1, 1).ToLocalTime();
  277.         DateTime now = DateTime.Now;// DateTime.UtcNow;
  278.         TimeSpan timeSpan = now - unixStartTime;
  279.         long timestamp = (long)timeSpan.TotalSeconds;
  280.         return timestamp;
  281.     }
  282.     public string GetMD5Hash(string input)
  283.     {
  284.         MD5 md5Hasher = MD5.Create();
  285.         byte[] data = md5Hasher.ComputeHash(Encoding.Default.GetBytes(input));
  286.         StringBuilder sBuilder = new StringBuilder();
  287.         for (int i = 0; i < data.Length; i++)
  288.         {
  289.             sBuilder.Append(data[i].ToString("x2"));
  290.         }
  291.         return sBuilder.ToString();
  292.     }
  293.     public string CalculateHmacSha1(string data, string key)
  294.     {
  295.         HMACSHA1 hmac = new HMACSHA1(Encoding.UTF8.GetBytes(key));
  296.         byte[] hashBytes = hmac.ComputeHash(Encoding.UTF8.GetBytes(data));
  297.         return Convert.ToBase64String(hashBytes);
  298.     }
  299.  
  300.     private void Update()
  301.     {
  302.         if(messageQueue.Count > 0)
  303.         {
  304.             MainThreadOnMessage(messageQueue.Dequeue());
  305.         }
  306.     }
  307. }

Json解析类。

  1. [Serializable]
  2. public struct XFResponse
  3. {
  4.     public string action;
  5.     public int code;
  6.     public string data;
  7.     public string desc;
  8.     public string sid;
  9. }
  10. [Serializable]
  11. public struct SpeechRecognitionResult
  12. {
  13.     public string Text;        
  14.     public bool IsFinal;        
  15. }

值得注意的问题。

1、Microphone使用时传默认设备名比传null好使

2、握手阶段时间戳用的是秒（不是毫秒）

3、上传结束标志时，也要间隔40ms，否则讯飞像是没收到一样

4、如果Microphone.devices的长度为0，电脑确实又有麦克风设备，那么可能是麦克风的名字是中文导致的

遗留问题：

yield return new WaitForSecondsRealtime(0.04f)实际间隔时间0.1s左右，导致消息发送得很慢，语音识别慢。

2024.5.24更新第二篇，有效解决消息发送慢，识别慢的问题

2024.6.19更新：取消协程中发送数据，直接在Update中发送。解决消息发送很慢问题

  1.     private void Update()
  2.     {
  3.         if (isRunning)
  4.         {
  5.             byte[] voiceData = GetVoiveData();
  6.             if (voiceData != null)
  7.             {
  8.                 SendData(voiceData);
  9.             }
  10.         }
  11.         if (messageQueue.Count > 0)
  12.         {
  13.             MainThreadOnMessage(messageQueue.Dequeue());
  14.         }
  15.     }
  16.  
  17. private int last_length = -1;
  18.     private float[] volumeData = new float[9999];
  19.     private short[] intData = new short[9999];
  20.     bool isRunning;
  21.     private byte[] GetVoiveData()
  22.     {
  23.         if (RecordedClip == null)
  24.         {
  25.             return null;
  26.         }
  27.         int new_length = Microphone.GetPosition(null);
  28.         if (new_length == last_length)
  29.         {
  30.             if (Microphone.devices.Length == 0)
  31.             {
  32.                 isRunning = false;
  33.             }
  34.             return null;
  35.         }
  36.         int length = new_length - last_length;
  37.         int offset = last_length + 1;
  38.         last_length = new_length;
  39.         if (offset < 0)
  40.         {
  41.             return null;
  42.         }
  43.         if (length < 0)
  44.         {
  45.             float[] temp = new float[RecordedClip.samples];
  46.             RecordedClip.GetData(temp, 0);
  47.             int lengthTail = RecordedClip.samples - offset;
  48.             int lengthHead = new_length + 1;
  49.             try
  50.             {
  51.                 Array.Copy(temp, offset, volumeData, 0, lengthTail);
  52.                 Array.Copy(temp, 0, volumeData, lengthTail + 1, lengthHead);
  53.                 length = lengthTail + lengthHead;
  54.  
  55.             }
  56.             catch (Exception)
  57.             {
  58.                 return null;
  59.             }
  60.         }
  61.         else
  62.         {
  63.             if (length > volumeData.Length)
  64.             {
  65.                 volumeData = new float[length];
  66.                 intData = new short[length];
  67.             }
  68.             RecordedClip.GetData(volumeData, offset);
  69.         }
  70.         byte[] bytesData = new byte[length * 2];
  71.         int rescaleFactor = 32767; //to convert float to Int16
  72.         for (int i = 0; i < length; i++)
  73.         {
  74.             intData[i] = (short)(volumeData[i] * rescaleFactor);
  75.             byte[] byteArr = BitConverter.GetBytes(intData[i]);
  76.             byteArr.CopyTo(bytesData, i * 2);
  77.         }
  78.         return bytesData;
  79.     }

本文内容由网友自发贡献，转载请注明出处：https://www.wpsshop.cn/w/秋刀鱼在做梦/article/detail/904759
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号