devbox
羊村懒王
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

数字人系列四：Motionverse 接入chatgpt、文心一言等国内外大语言模型_添加外部语言模型

作者：羊村懒王 | 2024-04-25 07:11:16

赞

踩

添加外部语言模型

1. 下载插件：motionverse官网地址：概述 · Motionverse 接口文档 (deepscience.cn)

image.png


2. 按照官方文档新建Unity工程：对接说明 · Motionverse 接口文档 (deepscience.cn)


3. 通过我们自己的ASR，将语音转换为文本，文本通过大语言模型（chatgpt、文心一言以及其他大语言模型）生成的结果，直接通过生成式AI技术，把文本转化为AI智能体的声音、动作和表情，和大语言模型完美连接，只需要在获取到文本的时候调用以下代码即可：
代码示例：

  1. DriveTask task = new DriveTask();
  2. task.player = player;
  3. task.text = question;
  4. NLPDrive.GetDrive(task);


其中，player即为挂载motionverse插件中Player脚本的对象，question即为大语言模型获取到的答案。

还可以自己生成语音，通过语音链接调用以下代码：

  1. DriveTask task = new DriveTask();
  2.  
  3. task.player = player;
  4.  
  5. task.text = audioUrl;
  6.  
  7. AudioUrlDrive.GetDrive(task);

其中，player即为挂载motionverse插件中Player脚本的对象，audioUrl即为语音链接。

4. 新建脚本AskManager，并挂载到场景中（可新建空物体），脚本代码如下：

  1. using LitJson;
  2. using Motionverse;
  3. using MotionverseSDK;
  4. using System;
  5. using System.Collections;
  6. using System.Text;
  7. using UnityEngine;
  8. using UnityEngine.Networking;
  9. using UnityEngine.UI;
  10.  
  11. public class AskManager : MonoBehaviour
  12. {
  13.     public Player player;
  14.     public Dropdown dropdown;
  15.     public InputField inputField;
  16.     public Button btnSend;
  17.  
  18.     public string chatGptKey = "";
  19.     private string chatUrl = "https://chatgpt.kazava.io/v1/chat/completions";
  20.  
  21.     public string wenXinAPI = "";
  22.     public string wenXinSECRET = "";
  23.     private string wenXinToken = "";
  24.     private int curSelectNLP = 0;
  25.  
  26.     // Start is called before the first frame update
  27.     private void Awake()
  28.     {
  29.         for (int i = 0; i < Display.displays.Length; i++)
  30.         {
  31.             Display.displays[i].Activate();
  32.         }
  33.  
  34.     }
  35.     void Start()
  36.     {
  37.         dropdown.onValueChanged.AddListener((value) =>
  38.         {
  39.             curSelectNLP = value;
  40.         });
  41.         StartCoroutine(GetWenxinToken());
  42.         btnSend.onClick.AddListener(() =>
  43.         {
  44.             if (string.IsNullOrEmpty(inputField.text))
  45.             {
  46.                 Debug.Log("请输入内容！");
  47.             }
  48.             else
  49.             {
  50.                 GetAnswer(inputField.text);
  51.             }
  52.         });
  53.     }
  54.  
  55.     public void GetAnswer(string q)
  56.     {
  57.         StartCoroutine(RealAnswer(q));
  58.     }
  59.  
  60.  
  61.  
  62.     public IEnumerator RealAnswer(string question)
  63.     {
  64.         switch (curSelectNLP)
  65.         {
  66.             case 0:
  67.                 DriveTask task = new DriveTask();
  68.                 task.player = player;
  69.                 task.text = question;
  70.                 NLPDrive.GetDrive(task);
  71.                 break;
  72.             case 1:
  73.                 StartCoroutine(RequestChat(question));
  74.                 break;
  75.             case 2:
  76.                 StartCoroutine(ChatCompletions(question));
  77.                 break;
  78.             default:
  79.                 break;
  80.         }
  81.  
  82.         Invoke("restartRecording", 1);
  83.  
  84.         yield return null;
  85.  
  86.     }
  87.  
  88.     IEnumerator GetWenxinToken()
  89.     {
  90.         string url =$"https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id={wenXinAPI}&client_secret={wenXinSECRET}";
  91.         UnityWebRequest webRequest = UnityWebRequest.Get(url);
  92.         webRequest.timeout = 5000;
  93.  
  94.         yield return webRequest.SendWebRequest();
  95.  
  96.         if (webRequest.result == UnityWebRequest.Result.Success)
  97.         {
  98.             string response = webRequest.downloadHandler.text;
  99.             var result = JsonUtility.FromJson<AccessTokenResponse>(response);
  100.             wenXinToken = result.access_token;
  101.         }
  102.         else
  103.         {
  104.             Debug.LogError("Failed to get access token: " + webRequest.error);
  105.         }
  106.     }
  107.  
  108.     IEnumerator ChatCompletions(string content)
  109.     {
  110.         string url = $"https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/completions?access_token={wenXinToken}";
  111.         WenXinPostData postData = new WenXinPostData();
  112.         postData.messages = new WenXinMessage[1];
  113.         postData.messages[0] = new WenXinMessage();
  114.         postData.messages[0].content = content + "30字以内";
  115.         string data = JsonMapper.ToJson(postData);
  116.  
  117.         UnityWebRequest webRequest = UnityWebRequest.Post(url, data, "application/json");
  118.         webRequest.timeout = 5000;
  119.         yield return webRequest.SendWebRequest();
  120.  
  121.         if (webRequest.result == UnityWebRequest.Result.Success)
  122.         {
  123.             string response = webRequest.downloadHandler.text;
  124.             WenXinRequest requestData = JsonMapper.ToObject<WenXinRequest>(response);
  125.             DriveTask task = new DriveTask();
  126.             task.player = player;
  127.             task.text = requestData.result;
  128.             TextDrive.GetDrive(task);
  129.         }
  130.         else
  131.         {
  132.             Debug.LogError("Chat completions request failed: " + webRequest.error);
  133.         }
  134.     }
  135.  
  136.     private IEnumerator RequestChat(string content)
  137.     {
  138.         using (UnityWebRequest webRequest = new UnityWebRequest(chatUrl, "POST"))
  139.         {
  140.             webRequest.SetRequestHeader("Content-Type", "application/json");
  141.             ChatGPTPostData postData = new ChatGPTPostData();
  142.             postData.key = chatGptKey;
  143.             postData.messages = new PostMessage[1];
  144.             postData.messages[0] = new PostMessage();
  145.             postData.messages[0].content = content + "30字以内"; ;
  146.             string data = JsonMapper.ToJson(postData);
  147.             byte[] jsonToSend = new UTF8Encoding().GetBytes(data);
  148.             webRequest.uploadHandler = new UploadHandlerRaw(jsonToSend);
  149.             webRequest.downloadHandler = new DownloadHandlerBuffer();
  150.             yield return webRequest.SendWebRequest();
  151.  
  152.             if (webRequest.result != UnityWebRequest.Result.Success)
  153.             {
  154.                 Debug.LogError("ChatGPT request error: " + content + webRequest.error);
  155.             }
  156.             else
  157.             {
  158.                 string response = webRequest.downloadHandler.text;
  159.                 ChatGPTRequestData requestData = JsonMapper.ToObject<ChatGPTRequestData>(response);
  160.                 DriveTask task = new DriveTask();
  161.                 task.player = player;
  162.                 task.text = requestData.choices[0].message.content;
  163.                 TextDrive.GetDrive(task);
  164.             }
  165.         }
  166.     }
  167. }


5. 新建脚本RealtimeAsrManager，代码如下：

  1. using System;
  2. using System.Collections;
  3. using UnityEngine;
  4. using UnityEngine.UI;
  5. using UnityWebSocket;
  6.  
  7.  
  8. namespace Motionverse
  9. {
  10.     public class RealtimeAsrManager : MonoBehaviour
  11.     {
  12.         private IWebSocket webSocket;
  13.         private Status status = Status.FirstFrame;
  14.         private bool lockReconnect = false;
  15.         [HideInInspector]
  16.         public static Action<string> Asr;
  17.  
  18.         [SerializeField]
  19.         private GameObject TextBG;
  20.         //百度
  21.         public int AsrAppId;
  22.         public string AsrAppkey = "";
  23.  
  24.  
  25.         void Start()
  26.         {
  27.             CreateWebSocket();
  28.             RecorderManager.DataAvailable += OnDataAvailable;
  29.         }
  30.         private void OnDisable()
  31.         {
  32.             RecorderManager.DataAvailable -= OnDataAvailable;
  33.         }
  34.         void CreateWebSocket()
  35.         {
  36.             try
  37.             {
  38.                 string uril = "wss://vop.baidu.com/realtime_asr?sn=" + Guid.NewGuid().ToString();
  39.                 webSocket = new WebSocket(uril);
  40.                 InitHandle();
  41.                 webSocket.ConnectAsync();
  42.             }
  43.             catch (Exception e)
  44.             {
  45.                 Debug.Log("websocket连接异常:" + e.Message);
  46.                 ReConnect();
  47.             }
  48.  
  49.         }
  50.  
  51.  
  52.         private void InitHandle()
  53.         {
  54.             RemoveHandle();
  55.             webSocket.OnOpen += OnOpen;
  56.             webSocket.OnMessage += OnMessage;
  57.             webSocket.OnClose += OnClose;
  58.             webSocket.OnError += OnError;
  59.  
  60.  
  61.         }
  62.         void RemoveHandle()
  63.         {
  64.             webSocket.OnOpen -= OnOpen;
  65.             webSocket.OnMessage -= OnMessage;
  66.             webSocket.OnClose -= OnClose;
  67.             webSocket.OnError -= OnError;
  68.         }
  69.         //请求开始
  70.         private void OnDataAvailable(byte[] data)
  71.         {
  72.             if (webSocket == null || (webSocket != null && webSocket.ReadyState != WebSocketState.Open))
  73.                 return;
  74.  
  75.             switch (status)
  76.             {
  77.                 case Status.FirstFrame://握手
  78.                     {
  79.                         var firstFrame = new FirstFrame();
  80.                         firstFrame.data.appid= AsrAppId;
  81.                         firstFrame.data.appkey = AsrAppkey;
  82.  
  83.                         webSocket.SendAsync(JsonUtility.ToJson(firstFrame));
  84.                         status = Status.ContinueFrame;
  85.                     }
  86.                     break;
  87.                 case Status.ContinueFrame://开始发送
  88.                     {
  89.                         if (data.Length > 0)
  90.                         {
  91.                             webSocket.SendAsync(data);
  92.                         }
  93.                     }
  94.                     break;
  95.                 case Status.LastFrame://关闭
  96.                     {
  97.                         webSocket.SendAsync(JsonUtility.ToJson(new LastFrame()));
  98.                     }
  99.                     break;
  100.                 default:
  101.                     break;
  102.             }
  103.         }
  104.  
  105.         void ReConnect()
  106.         {
  107.             if (this.lockReconnect)
  108.                 return;
  109.             this.lockReconnect = true;
  110.             StartCoroutine(SetReConnect());
  111.         }
  112.  
  113.         private IEnumerator SetReConnect()
  114.         {
  115.             yield return new WaitForSeconds(1);
  116.             CreateWebSocket();
  117.             lockReconnect = false;
  118.         }
  119.  
  120.         #region WebSocket Event Handlers
  121.  
  122.         private void OnOpen(object sender, OpenEventArgs e)
  123.         {
  124.             status = Status.FirstFrame;
  125.         }
  126.  
  127.         private void OnMessage(object sender, MessageEventArgs e)
  128.         {
  129.             var err_msg = Utils.GetJsonValue(e.Data, "err_msg");
  130.             var type = Utils.GetJsonValue(e.Data, "type");
  131.            
  132.             if (err_msg == "OK" && type == "MID_TEXT")
  133.             {
  134.                 var result = Utils.GetJsonValue(e.Data, "result");
  135.                 TextBG.GetComponentInChildren<Text>().text = result;
  136.             }
  137.  
  138.             if (err_msg == "OK" && type == "FIN_TEXT")
  139.             {
  140.                 var result = Utils.GetJsonValue(e.Data, "result");
  141.                 TextBG.GetComponentInChildren<Text>().text = result;
  142.                 if (result.Length > 1)
  143.                 {
  144.                     RecorderManager.Instance.EndRecording();
  145.                     Asr?.Invoke(result);
  146.                 }
  147.                
  148.             }
  149.  
  150.         }
  151.  
  152.         private void OnClose(object sender, CloseEventArgs e)
  153.         {
  154.             Debug.Log("websocket关闭," + string.Format("Closed: StatusCode: {0}, Reason: {1}", e.StatusCode, e.Reason));
  155.             webSocket = null;
  156.             ReConnect();
  157.         }
  158.  
  159.         private void OnError(object sender, ErrorEventArgs e)
  160.         {
  161.             if (e != null)
  162.                 Debug.Log("websocket连接异常:" + e.Message);
  163.             webSocket = null;
  164.             ReConnect();
  165.         }
  166.  
  167.         #endregion
  168.     }
  169. }


6. 新建脚本RecorderManager，代码如下：

  1. using UnityEngine;
  2. using System;
  3. using System.Collections;
  4. using UnityEngine.UI;
  5. using Unity.VisualScripting;
  6. namespace Motionverse
  7. {
  8.     public class RecorderManager : Singleton<RecorderManager>
  9.     {
  10.         //标记是否有麦克风
  11.         private bool isHaveMic = false;
  12.         //当前录音设备名称
  13.         private string currentDeviceName = string.Empty;
  14.         //表示录音的最大时长
  15.         int recordMaxLength = 600;
  16.         //录音频率,控制录音质量(16000)
  17.         int recordFrequency = 16000;
  18.         [HideInInspector]
  19.         public static Action<byte[]> DataAvailable;
  20.         [SerializeField]
  21.         private GameObject micON;
  22.         [SerializeField]
  23.         private GameObject micOFF;
  24.         [SerializeField]
  25.         private Image micAmount;
  26.         [SerializeField]
  27.         private GameObject TextBG;
  28.         private AudioClip saveAudioClip;
  29.  
  30.         int offsetSamples = 0;
  31.         private void Start()
  32.         {
  33.             Debug.Log(Microphone.devices[0]);
  34.             if (Microphone.devices.Length > 0)
  35.             {
  36.                 isHaveMic = true;
  37.                 currentDeviceName = Microphone.devices[0];
  38.                 StartCoroutine(GetAudioFrames());
  39.                 StartRecording();
  40.             }
  41.  
  42.         }
  43.  
  44.         /// <summary>
  45.         /// 开始录音
  46.         /// </summary>
  47.         /// <returns></returns>
  48.         public void StartRecording() //16000
  49.         {
  50.             if (isHaveMic == false || Microphone.IsRecording(currentDeviceName))
  51.             {
  52.                 return;
  53.             }
  54.             micOFF.gameObject.SetActive(false);
  55.             micON.gameObject.SetActive(true);
  56.             TextBG.GetComponentInChildren<Text>().text = null;
  57.             offsetSamples = 0;
  58.             saveAudioClip = Microphone.Start(currentDeviceName, true, recordMaxLength, recordFrequency);
  59.  
  60.         }
  61.         public void OnButtonClick()
  62.         {
  63.             if (Microphone.IsRecording(currentDeviceName))
  64.             {
  65.                 EndRecording();
  66.             }
  67.             else
  68.             {
  69.                 StartRecording();
  70.             }
  71.         }
  72.         public void EndRecording()
  73.         {
  74.  
  75.             if (isHaveMic == false || !Microphone.IsRecording(currentDeviceName))
  76.             {
  77.                 return;
  78.             }
  79.             micOFF.gameObject.SetActive(true);
  80.             micON.gameObject.SetActive(false);
  81.             //结束录音
  82.             Microphone.End(currentDeviceName);
  83.         }
  84.         public bool IsRecording()
  85.         {
  86.             return Microphone.IsRecording(currentDeviceName);
  87.         }
  88.         IEnumerator GetAudioFrames()
  89.         {
  90.  
  91.             while (true)
  92.             {
  93.                 if (Microphone.IsRecording(currentDeviceName))
  94.                 {
  95.                     int lenght = Microphone.GetPosition(currentDeviceName) * saveAudioClip.channels - offsetSamples;
  96.                     if (lenght > 0)
  97.                     {
  98.                         float[] samples = new float[lenght];
  99.                         saveAudioClip.GetData(samples, offsetSamples);
  100.  
  101.                         var samplesShort = new short[samples.Length];
  102.                         for (var index = 0; index < samples.Length; index++)
  103.                         {
  104.                             samplesShort[index] = (short)(samples[index] * short.MaxValue);
  105.                         }
  106.                         byte[] binaryData = new byte[samplesShort.Length * 2];
  107.                         Buffer.BlockCopy(samplesShort, 0, binaryData, 0, binaryData.Length);
  108.  
  109.                         offsetSamples += lenght;
  110.                         DataAvailable?.Invoke(binaryData);
  111.                     }
  112.  
  113.                     yield return new WaitForSeconds(0.16f);
  114.                 }
  115.                 else
  116.                 {
  117.                     yield return new WaitForSeconds(0.16f);
  118.                     byte[] binaryData = new byte[2];
  119.                     DataAvailable?.Invoke(binaryData);
  120.                 }
  121.  
  122.             }
  123.         }
  124.  
  125.         /// <summary>
  126.         /// 获取毫秒级别的时间戳,用于计算按下录音时长
  127.         /// </summary>
  128.         /// <returns></returns>
  129.         public double GetTimestampOfNowWithMillisecond()
  130.         {
  131.             return (DateTime.Now.ToUniversalTime().Ticks - 621355968000000000) / 10000;
  132.         }
  133.  
  134.         private void LateUpdate()
  135.         {
  136.             if (isHaveMic == true && Microphone.IsRecording(currentDeviceName))
  137.             {
  138.                 micAmount.fillAmount = Volume;
  139.             }
  140.         }
  141.         public float Volume
  142.         {
  143.             get
  144.             {
  145.                 if (Microphone.IsRecording(currentDeviceName))
  146.                 {
  147.                     // 采样数
  148.                     int sampleSize = 128;
  149.                     float[] samples = new float[sampleSize];
  150.                     int startPosition = Microphone.GetPosition(currentDeviceName) - (sampleSize + 1);
  151.                     // 得到数据
  152.                     if (startPosition < 0)
  153.                         return 0;
  154.  
  155.                     saveAudioClip.GetData(samples, startPosition);
  156.  
  157.                     // Getting a peak on the last 128 samples
  158.                     float levelMax = 0;
  159.                     for (int i = 0; i < sampleSize; ++i)
  160.                     {
  161.                         float wavePeak = samples[i];
  162.                         if (levelMax < wavePeak)
  163.                             levelMax = wavePeak;
  164.                     }
  165.  
  166.                     return levelMax;
  167.                 }
  168.                 return 0;
  169.             }
  170.         }
  171.         void OnGUI()
  172.         {
  173.             GUIStyle guiStyle = GUIStyle.none;
  174.             guiStyle.fontSize = 10;
  175.             guiStyle.normal.textColor = Color.white;
  176.             guiStyle.alignment = TextAnchor.UpperLeft;
  177.             Rect tr = new Rect(0, 0, 100, 100);
  178.             GUI.Label(tr, currentDeviceName, guiStyle);
  179.         }
  180.     }
  181. }


7. 新建空物体，挂载脚本RecorderManager和RealtimeAsrManager
8. 输入百度asr的appId和secretKey：

image.png


9. 输入GPTkey、问心一眼appId和secretKey：

image.png


10. 根据需求选择相应的NLP方式：

image.png


注意：如果缺少Singleton文件，可使用如下代码：

  1. using UnityEngine;
  2. namespace Motionverse
  3. {
  4.     public abstract class Singleton<T> : MonoBehaviour where T : Singleton<T>
  5.     {
  6.  
  7.         private static T sInstance = null;
  8.  
  9.         public static T Instance
  10.         {
  11.             get
  12.             {
  13.                 if (sInstance == null)
  14.                 {
  15.                     GameObject gameObject = new(typeof(T).FullName);
  16.                     sInstance = gameObject.AddComponent<T>();
  17.                 }
  18.  
  19.                 return sInstance;
  20.             }
  21.         }
  22.  
  23.         public static void Clear()
  24.         {
  25.             sInstance = null;
  26.         }
  27.  
  28.         protected virtual void Awake()
  29.         {
  30.             if (sInstance != null) Debug.LogError(name + "error: already initialized", this);
  31.  
  32.             sInstance = (T)this;
  33.         }
  34.     }
  35. }

若有收获，就点个赞吧

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/羊村懒王/article/detail/483943
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号