Unity & FACEGOOD Audio2Face 通过音频驱动面部BlendShape

目录

Audio2Face简介

在Unity中应用

Audio2Face简介

在元宇宙的热潮下，为了让AI数字人渗透到更多的领域中，FACEGOOD已经将语音驱动口型的算法技术开源，开源地址：

https://github.com/FACEGOOD/FACEGOOD-Audio2Face

该技术可以实时将音频数据转换为驱动数字人面部BlendShape的权重数据，不同于ARKit中的52个BlendShape，它的数量多达116个，我们可以通过对应关系得到相应的数值，对应关系如下：

生产的数据结果如下图所示，可见是116个取值范围为-1~1的小数：

这116个数值依次对应下面116个BlendShape名称：

brow_lower_l
tongue_Scale__X
tongue_Scale_Y
tongue_Scale__Y
tongue_Scale_Z
tongue_Scale__Z
nose_out_l
nose_out_r
tongue_u
tongue_u_u
brow_raise_d
cheek_suck_r
mouth_stretch_u
tongue_u_d
tooth_d_d
tongue_d
tooth_r
tooth_d_u
cheek_UP
eye_blink1_l
eye_blink1_r
eye_blink2_l
eye_blink2_r
eye_lidTight_l
eye_lidTight_r
eye_shutTight_l
eye_shutTight_r
brow_lower_r
eye_upperLidRaise_l
eye_upperLidRaise_r
eye_downLidRaise_l
eye_downLidRaise_r
jaw_sideways_l
jaw_sideways_r
jaw_thrust_c
mouth_chew_c
mouth_chinRaise_d
mouth_chinRaise_u
brow_raise_c
mouth_dimple_l
mouth_dimple_r
mouth_funnel_dl
mouth_funnel_dr
mouth_funnel_ul
mouth_funnel_ur
mouth_lipCornerDepressFix_l
mouth_lipCornerDepressFix_r
mouth_lipCornerDepress_l
mouth_lipCornerDepress_r
brow_raise_l
mouth_lipCornerPullOpen_l
mouth_lipCornerPullOpen_r
mouth_lipCornerPull_l
mouth_lipCornerPull_r
mouth_lipStretchOpen_l
mouth_lipStretchOpen_r
mouth_lipStretch_l
mouth_lipStretch_r
mouth_lowerLipDepress_l
mouth_lowerLipDepress_r
brow_raise_r
mouth_lowerLipProtrude_c
mouth_oh_c
mouth_oo_c
mouth_pressFix_c
mouth_press_l
mouth_press_r
mouth_pucker_l
mouth_pucker_r
mouth_screamFix_c
mouth_sideways_l
cheek_puff_l
mouth_sideways_r
mouth_stretch_c
mouth_suck_dl
mouth_suck_dr
mouth_suck_ul
mouth_suck_ur
mouth_upperLipRaise_l
mouth_upperLipRaise_r
nose_wrinkle_l
nose_wrinkle_r
cheek_puff_r
tooth_l
eye_lookDown1_l
eye_lookDown2_l
eye_lookLeft_l
eye_lookRight_l
eye_lookUp_l
eye_lookDown1_r
eye_lookDown2_r
eye_lookLeft_r
eye_lookRight_r
cheek_raise_l
eye_lookUp_r
tongue_Rot_1X
tongue_Rot__1X
tongue_Rot_2X
tongue_Rot__2X
tongue_Rot_3X
tongue_Rot__3X
tongue_Rot_1Y
tongue_Rot__1Y
tongue_Rot_2Y
cheek_raise_r
tongue_Rot__2Y
tongue_Rot_3Y
tongue_Rot__3Y
tongue_Rot_1Z
tongue_Rot__1Z
tongue_Rot_2Z
tongue_Rot__2Z
tongue_Rot_3Z
tongue_Rot__3Z
tongue_Scale_X
cheek_suck_l

在Unity中应用

可以用过构建python服务，Unity客户端开启麦克风录制音频，将音频数据发送给python服务端，服务端转换为驱动BlendShape的权重数据后，返回给Unity客户端进行驱动。需要注意的是Unity中BlendShape的权重范围并不是[-1,1]，因此需要进行映射。

 例如：

//将[-1,1]映射到[-100,100]
private float Remap(float v)
{
    return v * 100f;
}

下面是一段测试音频产生的bs权重数据文件，每一行包含116个权重数值，我们拿来进行测试，将其放到StreamingAssets文件夹下。

测试模型：

测试代码：

using System.IO;
using System.Collections;
using System.Collections.Generic;
 
using UnityEngine;
 
public class TEST : MonoBehaviour
{
    private Coroutine coroutine;
    private SkinnedMeshRenderer smr;
    private readonly List<List<float>> valueList = new List<List<float>>();
 
    private IEnumerator Start()
    {
        smr = GetComponent<SkinnedMeshRenderer>();
        string path = Path.Combine(Application.streamingAssetsPath, "weight.txt");
        using (StreamReader streamReader = new StreamReader(path))
        {
            string content;
            while ((content = streamReader.ReadLine()) != null)
            {
                List<float> list = new List<float>();
                content = content.Trim();
                string[] splitArray = content.Split(' ');
                for (int i = 0; i < splitArray.Length; i++)
                {
                    float.TryParse(splitArray[i], out float result);
                    list.Add(result);
                }
                valueList.Add(list);
                yield return null;
            }
        }
    }
 
    private IEnumerator ExecuteCoroutine()
    {
        for (int i = 0; i < valueList.Count; i++)
        {
            List<float> list = valueList[i];
 
            smr.SetBlendShapeWeight(0, Remap(list[49]));    //brow_raise_l
            smr.SetBlendShapeWeight(1, Remap(list[60]));    //brow_raise_r
            smr.SetBlendShapeWeight(2, Remap(list[25]));    //eye_shutTight_l
            smr.SetBlendShapeWeight(3, Remap(list[26]));    //eye_shutTight_r
            smr.SetBlendShapeWeight(4, Remap(list[87]));    //eye_lookRight_l
            smr.SetBlendShapeWeight(5, Remap(list[86]));    //eye_lookLeft_l    
            smr.SetBlendShapeWeight(6, Remap(list[92]));    //eye_lookRight_r
            smr.SetBlendShapeWeight(7, Remap(list[91]));    //eye_lookLeft_r
            smr.SetBlendShapeWeight(8, Remap(list[88]));    //eye_lookUp_l
            smr.SetBlendShapeWeight(9, Remap(list[94]));    //eye_lookUp_r
            smr.SetBlendShapeWeight(10, Remap(list[85]));   //eye_lookDown2_l
            smr.SetBlendShapeWeight(11, Remap(list[90]));   //eye_lookDown2_r
            smr.SetBlendShapeWeight(12, Mathf.Max(Remap(list[71]), Remap(list[82])));   //cheek_pull_l cheek_pull_r
            smr.SetBlendShapeWeight(13, Remap(list[18]));   //cheek_UP
            smr.SetBlendShapeWeight(14, Remap(list[18]));   //cheek_UP
            smr.SetBlendShapeWeight(15, Remap(list[6]));    //nose_out_l   
            smr.SetBlendShapeWeight(16, Remap(list[7]));    //nose_out_r
            smr.SetBlendShapeWeight(17, Remap(list[70]));   //mouth_sideways_l
            smr.SetBlendShapeWeight(18, Remap(list[72]));   //mouth_sideways_r
            smr.SetBlendShapeWeight(19, Mathf.Max(Remap(list[67]), Remap(list[68])));   //mouth_pucker_l mouth_pucker_2
            smr.SetBlendShapeWeight(20, Mathf.Max(Remap(list[41]), Remap(list[42]), Remap(list[43]), Remap(list[44])));   //mouth_funnel_dl dr ul ur
            smr.SetBlendShapeWeight(21, Remap(list[52]));   //mouth_lipCornerPull_l
            smr.SetBlendShapeWeight(22, Remap(list[53]));   //mouth_lipCornerPull_r
            smr.SetBlendShapeWeight(23, Mathf.Max(Remap(list[47]), Remap(list[45])));   //mouth_lipCornerDepress_l mouth_lipCornerDepressFix_l
            smr.SetBlendShapeWeight(24, Mathf.Max(Remap(list[48]), Remap(list[46])));   //mouth_lipCornerDepress_r mouth_lipCornerDepressFix_r
            smr.SetBlendShapeWeight(25, Remap(list[39]));   //mouth_dimple_l
            smr.SetBlendShapeWeight(26, Remap(list[40]));   //mouth_dimple_r
            smr.SetBlendShapeWeight(27, Remap(list[65]));   //mouth_press_l
            smr.SetBlendShapeWeight(28, Remap(list[66]));   //mouth_press_r
            smr.SetBlendShapeWeight(29, Remap(list[36]));   //mouth_chinRaise_d
            smr.SetBlendShapeWeight(30, Remap(list[37]));   //mouth_chinRaise_u
            smr.SetBlendShapeWeight(31, Remap(list[56]));   //mouth_lipStretch_l
            smr.SetBlendShapeWeight(32, Remap(list[57]));   //mouth_lipStretch_r
            smr.SetBlendShapeWeight(33, Remap(list[78]));   //mouth_upperLipRaise_l
            smr.SetBlendShapeWeight(34, Remap(list[79]));   //mouth_upperLipRaise_r
            smr.SetBlendShapeWeight(35, Remap(list[58]));   //mouth_lowerLipDepress_l
            smr.SetBlendShapeWeight(36, Remap(list[59]));   //mouth_lowerLipDepress_r
            smr.SetBlendShapeWeight(37, Mathf.Max(Remap(list[76]), Remap(list[77])));    //mouth_suck_ul mouth_suck_ur
            smr.SetBlendShapeWeight(38, Mathf.Max(Remap(list[74]), Remap(list[75])));    //mouth_suck_dl mouth_suck_dr
            smr.SetBlendShapeWeight(39, Remap(list[35]));   //mouth_chew_c
            smr.SetBlendShapeWeight(40, Remap(list[34]));   //jaw_thrust_c
            smr.SetBlendShapeWeight(41, Remap(list[73]));   //mouth_stretch_c
            smr.SetBlendShapeWeight(42, Remap(list[32]));   //jaw_sideways_l
            smr.SetBlendShapeWeight(43, Remap(list[33]));   //jaw_sideways_r
            smr.SetBlendShapeWeight(44, Remap(list[38]));   //brow_raise_c
            smr.SetBlendShapeWeight(45, Remap(list[22]));   //eye_blink2_r
            smr.SetBlendShapeWeight(46, Remap(list[21]));   //eye_blink2_l
            smr.SetBlendShapeWeight(47, Remap(list[0]));   //brow_lower_l
            smr.SetBlendShapeWeight(48, Remap(list[27]));   //brow_lower_r
            smr.SetBlendShapeWeight(49, Mathf.Max(Remap(list[31]), Remap(list[29])));   //eye_downLidRaise_r eye_upLidRaise_r
            smr.SetBlendShapeWeight(50, Mathf.Max(Remap(list[30]), Remap(list[28])));   //eye_downLidRaise_l eye_upLidRaise_l
            yield return new WaitForSeconds(.07f);
        }
        coroutine = null;
    }
    private float Remap(float v)
    {
        return v * 100f;
    }
 
    private void OnGUI()
    {
        GUI.enabled = coroutine == null;
        if (GUILayout.Button("Begin", GUILayout.Width(200f), GUILayout.Height(50f)))
        {
            coroutine = StartCoroutine(ExecuteCoroutine());
        }
        GUI.enabled = coroutine != null;
        if (GUILayout.Button("Stop", GUILayout.Width(200f), GUILayout.Height(50f)))
        {
            StopCoroutine(coroutine);
            coroutine = null;
        }
    }
}