Initialer Upload neues Unity-Projekt

Assets/Convai/Scripts/Runtime/Features/LipSync/ConvaiLipSync.cs

@@ -0,0 +1,203 @@
+using System;
+using System.Text.RegularExpressions;
+using Convai.Scripts.Utils.LipSync.Types;
+using Service;
+using UnityEngine;
+
+namespace Convai.Scripts.Utils.LipSync
+{
+    public class ConvaiLipSync : MonoBehaviour
+    {
+        public enum LipSyncBlendshapeType
+        {
+            None, // Default Value
+            OVR, // Oculus 
+            ReallusionPlus, // Reallusion Extended
+            ARKit, // AR Kit - Translated from Oculus
+        }
+
+        [Tooltip(
+            "The type of facial blend-shapes in the character. Select OVR for Oculus and ReallusionPlus for Reallusion Extended visemes.")]
+        public LipSyncBlendshapeType BlendshapeType = LipSyncBlendshapeType.OVR;
+        [Tooltip("Skinned Mesh Renderer Component for the head of the character.")]
+        public SkinnedMeshRenderer HeadSkinnedMeshRenderer;
+        [Tooltip("Skinned Mesh Renderer Component for the teeth of the character, if available. Leave empty if not.")]
+        public SkinnedMeshRenderer TeethSkinnedMeshRenderer;
+        [Tooltip("Skinned Mesh Renderer Component for the tongue of the character, if available. Leave empty if not.")]
+        public SkinnedMeshRenderer TongueSkinnedMeshRenderer;
+        [Tooltip("Game object with the bone of the jaw for the character, if available. Leave empty if not.")]
+        public GameObject jawBone;
+        [Tooltip("Game object with the bone of the tongue for the character, if available. Leave empty if not.")]
+        public GameObject tongueBone; // even though actually tongue doesn't have a bone
+
+        [HideInInspector]
+        public FaceModel faceModel = FaceModel.OvrModelName;
+        [Tooltip("The index of the first blendshape that will be manipulated.")]
+        public int firstIndex;
+        [Tooltip("This will multiply the weights of the incoming frames to the lipsync")]
+        [field: SerializeField] public float WeightMultiplier { get; private set; } = 1f;
+
+        private ConvaiNPC _convaiNPC;
+        public event Action<bool> OnCharacterLipSyncing;
+        private ConvaiLipSyncApplicationBase convaiLipSyncApplicationBase;
+        public ConvaiLipSyncApplicationBase ConvaiLipSyncApplicationBase { get => convaiLipSyncApplicationBase; private set => convaiLipSyncApplicationBase = value; }
+
+        private void Awake()
+        {
+            switch (BlendshapeType)
+            {
+                case LipSyncBlendshapeType.None:
+                    break;
+                case LipSyncBlendshapeType.OVR:
+                    ConvaiLipSyncApplicationBase = gameObject.GetOrAddComponent<ConvaiOVRLipsync>();
+                    break;
+                case LipSyncBlendshapeType.ReallusionPlus:
+                    ConvaiLipSyncApplicationBase = gameObject.GetOrAddComponent<ConvaiReallusionLipSync>();
+                    break;
+                case LipSyncBlendshapeType.ARKit:
+                    ConvaiLipSyncApplicationBase = gameObject.GetOrAddComponent<ConvaiARKitLipSync>();
+                    break;
+            }
+        }
+
+        /// <summary>
+        ///     This function will automatically set any of the unassigned skinned mesh renderers
+        ///     to appropriate values using regex based functions.
+        ///     It also invokes the LipSyncCharacter() function every one hundredth of a second.
+        /// </summary>
+        private void Start()
+        {
+            // regex search for SkinnedMeshRenderers: head, teeth, tongue
+            if (HeadSkinnedMeshRenderer == null)
+                HeadSkinnedMeshRenderer = GetHeadSkinnedMeshRendererWithRegex(transform);
+            if (TeethSkinnedMeshRenderer == null)
+                TeethSkinnedMeshRenderer = GetTeethSkinnedMeshRendererWithRegex(transform);
+            if (TongueSkinnedMeshRenderer == null)
+                TongueSkinnedMeshRenderer = GetTongueSkinnedMeshRendererWithRegex(transform);
+
+            _convaiNPC = GetComponent<ConvaiNPC>();
+            ConvaiLipSyncApplicationBase.Initialize(this, _convaiNPC);
+            SetCharacterLipSyncing(true);
+        }
+        /// <summary>
+        /// Fires an event with update the Character Lip Syncing State
+        /// </summary>
+        /// <param name="value"></param>
+        private void SetCharacterLipSyncing(bool value)
+        {
+            OnCharacterLipSyncing?.Invoke(value);
+        }
+
+
+        private void OnApplicationQuit()
+        {
+            StopLipSync();
+        }
+
+        /// <summary>
+        ///     This function finds the Head skinned mesh renderer components, if present,
+        ///     in the children of the parentTransform using regex.
+        /// </summary>
+        /// <param name="parentTransform">The parent transform whose children are searched.</param>
+        /// <returns>The SkinnedMeshRenderer component of the Head, if found; otherwise, null.</returns>
+        private SkinnedMeshRenderer GetHeadSkinnedMeshRendererWithRegex(Transform parentTransform)
+        {
+            // Initialize a variable to store the found SkinnedMeshRenderer.
+            SkinnedMeshRenderer findFaceSkinnedMeshRenderer = null;
+
+            // Define a regular expression pattern for matching child object names.
+            Regex regexPattern = new("(.*_Head|CC_Base_Body)");
+
+            // Iterate through each child of the parentTransform.
+            foreach (Transform child in parentTransform)
+                // Check if the child's name matches the regex pattern.
+                if (regexPattern.IsMatch(child.name))
+                {
+                    // If a match is found, get the SkinnedMeshRenderer component of the child.
+                    findFaceSkinnedMeshRenderer = child.GetComponent<SkinnedMeshRenderer>();
+
+                    // If a SkinnedMeshRenderer is found, break out of the loop.
+                    if (findFaceSkinnedMeshRenderer != null) break;
+                }
+
+            // Return the found SkinnedMeshRenderer (or null if none is found).
+            return findFaceSkinnedMeshRenderer;
+        }
+
+
+        /// <summary>
+        ///     This function finds the Teeth skinned mesh renderer components, if present,
+        ///     in the children of the parentTransform using regex.
+        /// </summary>
+        /// <param name="parentTransform">The parent transform whose children are searched.</param>
+        /// <returns>The SkinnedMeshRenderer component of the Teeth, if found; otherwise, null.</returns>
+        private SkinnedMeshRenderer GetTeethSkinnedMeshRendererWithRegex(Transform parentTransform)
+        {
+            // Initialize a variable to store the found SkinnedMeshRenderer for teeth.
+            SkinnedMeshRenderer findTeethSkinnedMeshRenderer = null;
+
+            // Define a regular expression pattern for matching child object names.
+            Regex regexPattern = new("(.*_Teeth|CC_Base_Body)");
+
+            // Iterate through each child of the parentTransform.
+            foreach (Transform child in parentTransform)
+                // Check if the child's name matches the regex pattern.
+                if (regexPattern.IsMatch(child.name))
+                {
+                    // If a match is found, get the SkinnedMeshRenderer component of the child.
+                    findTeethSkinnedMeshRenderer = child.GetComponent<SkinnedMeshRenderer>();
+
+                    // If a SkinnedMeshRenderer is found, break out of the loop.
+                    if (findTeethSkinnedMeshRenderer != null) break;
+                }
+
+            // Return the found SkinnedMeshRenderer for teeth (or null if none is found).
+            return findTeethSkinnedMeshRenderer;
+        }
+
+
+        /// <summary>
+        ///     This function finds the Tongue skinned mesh renderer components, if present,
+        ///     in the children of the parentTransform using regex.
+        /// </summary>
+        /// <param name="parentTransform">The parent transform whose children are searched.</param>
+        /// <returns>The SkinnedMeshRenderer component of the Tongue, if found; otherwise, null.</returns>
+        private SkinnedMeshRenderer GetTongueSkinnedMeshRendererWithRegex(Transform parentTransform)
+        {
+            // Initialize a variable to store the found SkinnedMeshRenderer for the tongue.
+            SkinnedMeshRenderer findTongueSkinnedMeshRenderer = null;
+
+            // Define a regular expression pattern for matching child object names.
+            Regex regexPattern = new("(.*_Tongue|CC_Base_Body)");
+
+            // Iterate through each child of the parentTransform.
+            foreach (Transform child in parentTransform)
+                // Check if the child's name matches the regex pattern.
+                if (regexPattern.IsMatch(child.name))
+                {
+                    // If a match is found, get the SkinnedMeshRenderer component of the child.
+                    findTongueSkinnedMeshRenderer = child.GetComponent<SkinnedMeshRenderer>();
+
+                    // If a SkinnedMeshRenderer is found, break out of the loop.
+                    if (findTongueSkinnedMeshRenderer != null) break;
+                }
+
+            // Return the found SkinnedMeshRenderer for the tongue (or null if none is found).
+            return findTongueSkinnedMeshRenderer;
+        }
+        /// <summary>
+        /// Purges the latest chuck of lipsync frames
+        /// </summary>
+        public void PurgeExcessFrames()
+        {
+            ConvaiLipSyncApplicationBase?.PurgeExcessBlendShapeFrames();
+        }
+        /// <summary>
+        /// Stops the Lipsync by clearing the frames queue
+        /// </summary>
+        public void StopLipSync()
+        {
+            ConvaiLipSyncApplicationBase?.ClearQueue();
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/LipSync/ConvaiLipSync.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 88bce56f6985ef84f8835a0152628fa1
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/LipSync/ConvaiLipSyncApplicationBase.cs

@@ -0,0 +1,159 @@
+using System;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+using Service;
+using UnityEngine;
+
+
+namespace Convai.Scripts.Utils.LipSync
+{
+
+    /// <summary>
+    /// This Class will serve as a base for any method of Lipsync that Convai will develop or use
+    /// </summary>
+    public abstract class ConvaiLipSyncApplicationBase : MonoBehaviour
+    {
+        /// <summary>
+        /// This stores a dictionary of blendshape name and index of the Blendweight it will affect
+        /// </summary>
+        protected Dictionary<string, int> _headMapping;
+        /// <summary>
+        /// Reference to the Head Skin Mesh Renderer used for lipsync
+        /// </summary>
+        protected SkinnedMeshRenderer _headSkinMeshRenderer;
+        /// <summary>
+        /// Reference to the Teeth Skin Mesh Renderer used for lipsync
+        /// </summary>
+        protected SkinnedMeshRenderer _teethSkinMeshRenderer;
+        /// <summary>
+        /// Reference to the Jaw bone gameobject used for lipsync
+        /// </summary>
+        private GameObject _jawBone;
+        /// <summary>
+        /// Reference to the Tongue bone gameobject used for lipsync
+        /// </summary>
+        private GameObject _tongueBone;
+        /// <summary>
+        /// Reference to the NPC on which lipsync will be applied
+        /// </summary>
+        protected ConvaiNPC _convaiNPC;
+        protected float _weightMultiplier { get; private set; }
+        #region Null States of References
+        protected bool HasHeadSkinnedMeshRenderer { get; private set; }
+        protected bool HasTeethSkinnedMeshRenderer { get; private set; }
+        protected bool HasJawBone { get; private set; }
+        protected bool HasTongueBone { get; private set; }
+        #endregion
+        /// <summary>
+        /// Initializes and setup up of the things necessary for lipsync to work
+        /// </summary>
+        /// <param name="convaiLipSync"></param>
+        /// <param name="convaiNPC"></param>
+        public virtual void Initialize(ConvaiLipSync convaiLipSync, ConvaiNPC convaiNPC)
+        {
+            _headSkinMeshRenderer = convaiLipSync.HeadSkinnedMeshRenderer;
+            HasHeadSkinnedMeshRenderer = _headSkinMeshRenderer != null;
+
+            _teethSkinMeshRenderer = convaiLipSync.TeethSkinnedMeshRenderer;
+            HasTeethSkinnedMeshRenderer = _teethSkinMeshRenderer != null;
+
+            _jawBone = convaiLipSync.jawBone;
+            HasJawBone = _jawBone != null;
+
+            _tongueBone = convaiLipSync.tongueBone;
+            HasTongueBone = _tongueBone != null;
+
+            _convaiNPC = convaiNPC;
+            _weightMultiplier = convaiLipSync != null ? convaiLipSync.WeightMultiplier : 1;
+
+            if (HasHeadSkinnedMeshRenderer)
+                _headMapping = SetupMapping(GetHeadRegexMapping, _headSkinMeshRenderer);
+        }
+        /// <summary>
+        /// Creates the mapping of blendshape and index it affects during lipsync
+        /// </summary>
+        protected Dictionary<string, int> SetupMapping(Func<Dictionary<string, string>> finder, SkinnedMeshRenderer skinnedMeshRenderer)
+        {
+            Dictionary<string, int> mapping = new Dictionary<string, int>();
+            Dictionary<string, string> regexMapping = finder();
+
+            foreach (KeyValuePair<string, string> pair in regexMapping)
+            {
+                for (int i = 0; i < skinnedMeshRenderer.sharedMesh.blendShapeCount; i++)
+                {
+                    string blendShapeName = skinnedMeshRenderer.sharedMesh.GetBlendShapeName(i);
+                    Regex regex = new(pair.Value);
+                    if (regex.IsMatch(blendShapeName))
+                    {
+                        mapping.TryAdd(pair.Key, i);
+                    }
+                }
+            }
+            return mapping;
+        }
+        /// <summary>
+        /// Returns a dictionary of blendshape name and regex string used to find the index
+        /// TODO Modify the override to fit your version of the mapping
+        /// </summary>
+        /// <returns></returns>
+        protected virtual Dictionary<string, string> GetHeadRegexMapping()
+        {
+            return new Dictionary<string, string>();
+        }
+        /// <summary>
+        /// Updates the tongue bone rotation to the new rotation
+        /// </summary>
+        /// <param name="newRotation"></param>
+        protected void UpdateTongueBoneRotation(Vector3 newRotation)
+        {
+            if (!HasTongueBone) return;
+            _tongueBone.transform.localEulerAngles = newRotation;
+        }
+        /// <summary>
+        /// Updates the jaw bone rotation to the new rotation
+        /// </summary>
+        /// <param name="newRotation"></param>
+        protected void UpdateJawBoneRotation(Vector3 newRotation)
+        {
+            if (!HasJawBone) return;
+            _jawBone.transform.localEulerAngles = newRotation;
+        }
+        /// <summary>
+        /// Updates the current blendshape or visemes frame
+        /// </summary>
+        protected abstract void UpdateBlendShape();
+        /// <summary>
+        /// This removes the excess frames in the queue
+        /// </summary>
+        public abstract void PurgeExcessBlendShapeFrames();
+        /// <summary>
+        /// This resets the whole queue of the frames
+        /// </summary>
+        protected bool CanPurge<T>(Queue<T> queue)
+        {
+            // ? Should I hardcode the limiter for this check
+            return queue.Count < 10;
+        }
+        public abstract void ClearQueue();
+        /// <summary>
+        /// Adds blendshape frames in the queue
+        /// </summary>
+        /// <param name="blendshapeFrames"></param>
+        public virtual void EnqueueQueue(Queue<BlendshapeFrame> blendshapeFrames) { }
+        /// <summary>
+        /// Adds Visemes frames in the list
+        /// </summary>
+        /// <param name="visemesFrames"></param>
+        public virtual void EnqueueQueue(Queue<VisemesData> visemesFrames) { }
+        /// <summary>
+        /// Adds a blendshape frame in the last queue
+        /// </summary>
+        /// <param name="blendshapeFrame"></param>
+        public virtual void EnqueueFrame(BlendshapeFrame blendshapeFrame) { }
+        /// <summary>
+        /// Adds a viseme frame to the last element of the list
+        /// </summary>
+        /// <param name="viseme"></param>
+        public virtual void EnqueueFrame(VisemesData viseme) { }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/LipSync/ConvaiLipSyncApplicationBase.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: bb231034f5b2dee4494498fe9117bda1
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/LipSync/LipSyncBlendFrameData.cs

@@ -0,0 +1,96 @@
+using System.Collections.Generic;
+using Service;
+using UnityEngine;
+
+namespace Convai.Scripts.Utils.LipSync
+{
+    public class LipSyncBlendFrameData
+    {
+        public enum FrameType
+        {
+            Visemes,
+            Blendshape
+        }
+        private readonly int _totalFrames;
+        private readonly Queue<BlendshapeFrame> _blendShapeFrames = new Queue<BlendshapeFrame>();
+        private readonly Queue<VisemesData> _visemesFrames = new Queue<VisemesData>();
+        private readonly GetResponseResponse _getResponseResponse;
+        private readonly FrameType _frameType;
+
+        private int _framesCaptured;
+        private bool _partiallyProcessed;
+
+        public LipSyncBlendFrameData(int totalFrames, GetResponseResponse response, FrameType frameType)
+        {
+            _totalFrames = totalFrames;
+            _framesCaptured = 0;
+            _getResponseResponse = response;
+            _frameType = frameType;
+            //Logger.DebugLog($"Total Frames: {_totalFrames} | {response.AudioResponse.TextData}", Logger.LogCategory.LipSync);
+        }
+
+        public void Enqueue(BlendshapeFrame blendShapeFrame)
+        {
+            _blendShapeFrames.Enqueue(blendShapeFrame);
+            _framesCaptured++;
+        }
+
+        public void Enqueue(VisemesData visemesData)
+        {
+            _visemesFrames.Enqueue(visemesData);
+        }
+
+        public void Process(ConvaiNPC npc)
+        {
+            if (!_partiallyProcessed)
+                npc.EnqueueResponse(_getResponseResponse);
+            switch (_frameType)
+            {
+                case FrameType.Visemes:
+                    npc.convaiLipSync.ConvaiLipSyncApplicationBase.EnqueueQueue(new Queue<VisemesData>(_visemesFrames));
+                    break;
+                case FrameType.Blendshape:
+                    npc.convaiLipSync.ConvaiLipSyncApplicationBase.EnqueueQueue(new Queue<BlendshapeFrame>(_blendShapeFrames));
+                    break;
+            }
+
+            npc.AudioManager.SetWaitForCharacterLipSync(false);
+        }
+
+        public void ProcessPartially(ConvaiNPC npc)
+        {
+            if (!_partiallyProcessed)
+            {
+                _partiallyProcessed = true;
+                npc.EnqueueResponse(_getResponseResponse);
+                npc.AudioManager.SetWaitForCharacterLipSync(false);
+            }
+
+            switch (_frameType)
+            {
+                case FrameType.Visemes:
+                    while (_visemesFrames.Count != 0)
+                    {
+                        npc.convaiLipSync.ConvaiLipSyncApplicationBase.EnqueueFrame(_visemesFrames.Dequeue());
+                    }
+                    break;
+                case FrameType.Blendshape:
+                    while (_blendShapeFrames.Count != 0)
+                    {
+                        npc.convaiLipSync.ConvaiLipSyncApplicationBase.EnqueueFrame(_blendShapeFrames.Dequeue());
+                    }
+                    break;
+            }
+        }
+
+        public bool CanPartiallyProcess()
+        {
+            return _framesCaptured > Mathf.Min(21, _totalFrames * 0.7f);
+        }
+
+        public bool CanProcess()
+        {
+            return _framesCaptured == _totalFrames;
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/LipSync/LipSyncBlendFrameData.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 31db1a9457d64f3d936ff7f5aabfb193
+timeCreated: 1708491067

Assets/Convai/Scripts/Runtime/Features/LipSync/Types.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 6158a9323c720f5408c5b7caa77405cc
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiARKitLipSync.cs

@@ -0,0 +1,112 @@
+using System.Collections.Generic;
+using System.Reflection;
+using Service;
+
+
+namespace Convai.Scripts.Utils.LipSync.Types
+{
+    public class ConvaiARKitLipSync : ConvaiVisemesLipSync
+    {
+        private Dictionary<string, int> _teethMapping;
+        public override void Initialize(ConvaiLipSync convaiLipSync, ConvaiNPC convaiNPC)
+        {
+            base.Initialize(convaiLipSync, convaiNPC);
+            if (HasTeethSkinnedMeshRenderer)
+                _teethMapping = SetupMapping(GetTeethRegexMapping, _teethSkinMeshRenderer);
+        }
+        private Dictionary<string, string> GetTeethRegexMapping()
+        {
+            string prefix = "(?:[A-Z]\\d{1,2}_)?";
+            string spacer = "[\\s_]*";
+            string open = "[Oo]pen";
+            return new Dictionary<string, string>()
+            {
+                {"KK", $"{prefix}[Jj]aw{spacer}{open}"},
+                {"AA", $"{prefix}[Jj]aw{spacer}[Ff]orward"}
+            };
+        }
+        protected override Dictionary<string, string> GetHeadRegexMapping()
+        {
+            string mouth = "[Mm]outh";
+            string spacer = "[\\s_]*";
+            string left = "[Ll]eft";
+            string right = "[Rr]ight";
+            string lower = "[Ll]ower";
+            string upper = "[Uu]pper";
+            string open = "[Oo]pen";
+            string funnel = "[Ff]unnel";
+            string pucker = "[Pp]ucker";
+            string prefix = "(?:[A-Z]\\d{1,2}_)?";
+
+            return new Dictionary<string, string>()
+            {
+                {"PP", $"{prefix}{mouth}{spacer}{pucker}"},
+                {"FF", $"{prefix}{mouth}{spacer}{funnel}"},
+                {"THL", $"{prefix}{mouth}{spacer}{lower}{spacer}[Dd]own{spacer}{left}"},
+                {"THR", $"{prefix}{mouth}{spacer}{lower}{spacer}[Dd]own{spacer}{right}"},
+                {"DDL", $"{prefix}{mouth}{spacer}[Pp]ress{spacer}{left}"},
+                {"DDR", $"{prefix}{mouth}{spacer}[Pp]ress{spacer}{right}"},
+                {"KK", $"{prefix}[Jj]aw{spacer}{open}"},
+                {"CHL",$"{prefix}{mouth}{spacer}[Ss]tretch{spacer}{left}"},
+                {"CHR",$"{prefix}{mouth}{spacer}[Ss]tretch{spacer}{right}"},
+                {"SSL", $"{prefix}{mouth}{spacer}[Ss]mile{spacer}{left}"},
+                {"SSR", $"{prefix}{mouth}{spacer}[Ss]mile{spacer}{right}"},
+                {"NNL", $"{prefix}[Nn]ose{spacer}[Ss]neer{spacer}{left}"},
+                {"NNR", $"{prefix}[Nn]ose{spacer}[Ss]neer{spacer}{right}"},
+                {"RRU",$"{prefix}{mouth}{spacer}[Rr]oll{spacer}{upper}"},
+                {"RRL", $"{prefix}{mouth}{spacer}[Rr]oll{spacer}{lower}"},
+                {"AA", $"{prefix}[Jj]aw{spacer}{open}"},
+                {"EL", $"{prefix}{mouth}{spacer}{upper}{spacer}[Uu]p{spacer}{left}"},
+                {"ER", $"{prefix}{mouth}{spacer}{upper}{spacer}[Uu]p{spacer}{right}"},
+                {"IHL", $"{prefix}{mouth}{spacer}[Ff]rown{spacer}{left}"},
+                {"IHR",$"{prefix}{mouth}{spacer}[Ff]rown{spacer}{right}"},
+                {"OU", $"{prefix}{mouth}{spacer}{pucker}"},
+                {"OH", $"{prefix}{mouth}{spacer}{funnel}"},
+            };
+        }
+        private void Update()
+        {
+            // Check if the dequeued frame is not null.
+            if (_currentViseme == null) return;
+            // Check if the frame represents silence (-2 is a placeholder for silence).
+            if (_currentViseme.Sil == -2) return;
+
+            float weight;
+            List<int> knownHeadIndexs = new List<int>();
+            List<int> knownTeethIndexs = new List<int>();
+            foreach (PropertyInfo propertyInfo in typeof(Viseme).GetProperties())
+            {
+                if (propertyInfo.PropertyType != typeof(float)) continue;
+                string fieldName = propertyInfo.Name.ToUpper();
+                float value = (float)propertyInfo.GetValue(_currentViseme);
+                weight = fieldName switch
+                {
+                    "KK" => 1.0f / 1.5f,
+                    "DD" => 1.0f / 0.7f,
+                    "CH" => 1.0f / 2.7f,
+                    "SS" => 1.0f / 1.5f,
+                    "NN" => 1.0f / 2.0f,
+                    "RR" => 1.0f / 0.9f,
+                    "AA" => 1.0f / 2.0f,
+                    "II" => 1.0f / 1.2f,
+                    "OH" => 1.2f,
+                    _ => 1.0f
+                };
+
+                foreach (string s in _possibleCombinations)
+                {
+                    float weightThisFrame = value * weight * _weightMultiplier;
+                    string modifiedFieldName = fieldName + s;
+                    if (HasHeadSkinnedMeshRenderer)
+                    {
+                        FindAndUpdateBlendWeight(_headSkinMeshRenderer, modifiedFieldName, weightThisFrame, knownHeadIndexs, _headMapping);
+                    }
+                    if (HasTeethSkinnedMeshRenderer)
+                    {
+                        FindAndUpdateBlendWeight(_teethSkinMeshRenderer, modifiedFieldName, weightThisFrame, knownTeethIndexs, _teethMapping);
+                    }
+                }
+            }
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiARKitLipSync.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: a78929d391d407d46ab3288e15a0700d
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiBlendShapeLipSync.cs

@@ -0,0 +1,202 @@
+using System.Collections.Generic;
+using System.Reflection;
+using Service;
+using UnityEngine;
+
+/*
+! This class is a Work in progress and can produce un expected results, Convai does not advise to use this class in production, please use this with extreme caution
+*/
+
+namespace Convai.Scripts.Utils.LipSync.Types
+{
+    public class ConvaiBlendShapeLipSync : ConvaiLipSyncApplicationBase
+    {
+        private const float A2XFRAMERATE = 1f / 30f;
+        private Queue<Queue<BlendshapeFrame>> _blendShapesQueue = new();
+        private ARKitBlendShapes _currentBlendshape;
+
+        protected override Dictionary<string, string> GetHeadRegexMapping()
+        {
+            #region Regex Finders
+            string prefix = "(?:[A-Z]\\d{1,2}_)?";
+            string spacer = "[\\s_]*";
+            string mouth = "[Mm]outh";
+            string nose = "[Nn]ose";
+            string left = "[Ll]eft";
+            string right = "[Rr]ight";
+            string up = "[Uu]p";
+            string down = "[Dd]own";
+            string lower = "[Ll]ower";
+            string upper = "[Uu]pper";
+            string open = "[Oo]pen";
+            string funnel = "[Ff]unnel";
+            string pucker = "[Pp]ucker";
+            string sneer = "[Ss]neer";
+            string cheek = "[Cc]heek";
+            string squint = "[Ss]quint";
+            string brow = "[Bb]row";
+            string outer = "[Oo]uter";
+            string inner = "[Ii]nner";
+            string eye = "[Ee]ye";
+            string blink = "[Bb]link";
+            string look = "[Ll]ook";
+            string In = "[Ii]n";
+            string Out = "[Oo]ut";
+            string wide = "[Ww]ide";
+            string forward = "[Ff]orward";
+            string jaw = "[Jj]aw";
+            string close = "[Cc]lose";
+            string smile = "[Ss]mile";
+            string frown = "[Ff]rown";
+            string dimple = "[Dd]imple";
+            string stretch = "[Ss]tretch";
+            string roll = "[Rr]oll";
+            string shrug = "[Ss]hrug";
+            string press = "[Pp]ress";
+            #endregion
+
+            return new Dictionary<string, string>(){
+                {"TougueOut", $"{prefix}[Tt]ougue{spacer}[Oo]ut"},
+
+                {"NoseSneerRight", $"{prefix}{nose}{spacer}{sneer}{spacer}{right}"},
+                {"NoseSneerLeft", $"{prefix}{nose}{spacer}{sneer}{spacer}{left}"},
+
+                {"CheekSquintRight", $"{prefix}{cheek}{spacer}{squint}{spacer}{right}"},
+                {"CheekSquintLeft", $"{prefix}{cheek}{spacer}{squint}{spacer}{left}"},
+                {"CheekPuff", $"{prefix}{cheek}{spacer}[Pp]uff"},
+
+                {"BrowDownLeft", $"{prefix}{brow}{spacer}{down}{spacer}{left}"},
+                {"BrowDownRight", $"{prefix}{brow}{spacer}{down}{spacer}{right}"},
+                {"BrowInnerUp", $"{prefix}{brow}{spacer}{inner}{spacer}{up}"},
+                {"BrowOuterUpLeft", $"{prefix}{brow}{spacer}{outer}{spacer}{up}{spacer}{left}"},
+                {"BrowOuterUpRight", $"{prefix}{brow}{spacer}{outer}{spacer}{up}{spacer}{right}"},
+
+                {"EyeBlinkLeft", $"{prefix}{eye}{spacer}{blink}{spacer}{left}"},
+                {"EyeLookDownLeft",$"{prefix}{eye}{spacer}{look}{spacer}{In}{left}"},
+                {"EyeLookInLeft", $"{prefix}{eye}{spacer}{look}{spacer}{In}{spacer}{left}"},
+                {"EyeLookOutLeft", $"{prefix}{eye}{spacer}{look}{spacer}{Out}{spacer}{left}"},
+                {"EyeLookUpLeft", $"{prefix}{eye}{spacer}{look}{spacer}{up}{spacer}{left}"},
+                {"EyeSquintLeft", $"{prefix}{eye}{spacer}{squint}{spacer}{left}"},
+                {"EyeWideLeft", $"{prefix}{eye}{spacer}{wide}{spacer}{left}"},
+
+                {"EyeBlinkRight", $"{prefix}{eye}{spacer}{blink}{spacer}{right}"},
+                {"EyeLookDownRight",$"{prefix}{eye}{spacer}{look}{spacer}{In}{right}"},
+                {"EyeLookInRight", $"{prefix}{eye}{spacer}{look}{spacer}{In}{spacer}{right}"},
+                {"EyeLookOutRight", $"{prefix}{eye}{spacer}{look}{spacer}{Out}{spacer}{right}"},
+                {"EyeLookUpRight", $"{prefix}{eye}{spacer}{look}{spacer}{up}{spacer}{right}"},
+                {"EyeSquintRight", $"{prefix}{eye}{spacer}{squint}{spacer}{right}"},
+                {"EyeWideRight", $"{prefix}{eye}{spacer}{wide}{spacer}{right}"},
+
+                {"JawForward", $"{prefix}{jaw}{spacer}{forward}"},
+                {"JawLeft", $"{prefix}{jaw}{spacer}{left}"},
+                {"JawRight", $"{prefix}{jaw}{spacer}{right}"},
+                {"JawOpen", $"{prefix}{jaw}{spacer}{open}"},
+
+                {"MouthClose", $"{prefix}{mouth}{spacer}{close}"},
+                {"MouthFunnel", $"{prefix}{mouth}{spacer}{funnel}"},
+                {"MouthPucker", $"{prefix}{mouth}{spacer}{pucker}"},
+
+                {"Mouthleft", $"{prefix}{mouth}{spacer}{left}"},
+                {"MouthRight", $"{prefix}{mouth}{spacer}{right}"},
+
+                {"MouthSmileLeft", $"{prefix}{mouth}{spacer}{smile}{spacer}{left}"},
+                {"MouthSmileRight", $"{prefix}{mouth}{spacer}{smile}{spacer}{right}"},
+
+                {"MouthFrownLeft", $"{prefix}{mouth}{spacer}{frown}{spacer}{left}"},
+                {"MouthFrownRight", $"{prefix}{mouth}{spacer}{frown}{spacer}{right}"},
+
+                {"MouthDimpleLeft", $"{prefix}{mouth}{spacer}{dimple}{spacer}{left}"},
+                {"MouthDimpleRight", $"{prefix}{mouth}{spacer}{dimple}{spacer}{right}"},
+
+                {"MouthStretchLeft", $"{prefix}{mouth}{spacer}{stretch}{spacer}{left}"},
+                {"MouthStretchRight", $"{prefix}{mouth}{spacer}{stretch}{spacer}{right}"},
+
+                {"MouthRollLower", $"{prefix}{mouth}{spacer}{roll}{spacer}{lower}"},
+                {"MouthRollUpper", $"{prefix}{mouth}{spacer}{roll}{spacer}{upper}"},
+
+                {"MouthShrugLower", $"{prefix}{mouth}{spacer}{shrug}{spacer}{lower}"},
+                {"MouthShrugUpper", $"{prefix}{mouth}{spacer}{shrug}{spacer}{upper}"},
+
+                {"MouthPressLeft", $"{prefix}{mouth}{spacer}{press}{spacer}{left}"},
+                {"MouthPressRight", $"{prefix}{mouth}{spacer}{press}{spacer}{right}"},
+
+                {"MouthLowerDownLeft", $"{prefix}{mouth}{spacer}{lower}{spacer}{down}{spacer}{left}"},
+                {"MouthLowerDownRight", $"{prefix}{mouth}{spacer}{lower}{spacer}{down}{spacer}{right}"},
+
+                {"MouthUpperUpLeft", $"{prefix}{mouth}{spacer}{upper}{spacer}{up}{spacer}{left}"},
+                {"MouthUpperUpRight", $"{prefix}{mouth}{spacer}{upper}{spacer}{up}{spacer}{right}"},
+            };
+        }
+
+        public override void Initialize(ConvaiLipSync convaiLipSync, ConvaiNPC convaiNPC)
+        {
+            base.Initialize(convaiLipSync, convaiNPC);
+            InvokeRepeating(nameof(UpdateBlendShape), 0, A2XFRAMERATE);
+        }
+
+        protected override void UpdateBlendShape()
+        {
+            if (_blendShapesQueue == null || _blendShapesQueue.Count <= 0)
+            {
+                _currentBlendshape = new ARKitBlendShapes();
+                return;
+            }
+            if (_blendShapesQueue.Peek().Count == 0)
+            {
+                _blendShapesQueue.Dequeue();
+                return;
+            }
+            if (!_convaiNPC.IsCharacterTalking) return;
+            _currentBlendshape = _blendShapesQueue.Peek().Dequeue().Blendshapes;
+        }
+        private void Update()
+        {
+            if (_currentBlendshape == null) return;
+            UpdateJawBoneRotation(new Vector3(0.0f, 0.0f, -90.0f - _currentBlendshape.JawOpen * 30f));
+            UpdateTongueBoneRotation(new Vector3(0.0f, 0.0f, -5.0f * _currentBlendshape.TongueOut));
+            if (!HasHeadSkinnedMeshRenderer) return;
+            foreach (PropertyInfo propertyInfo in typeof(ARKitBlendShapes).GetProperties())
+            {
+                if (propertyInfo.PropertyType != typeof(float)) continue;
+                string fieldName = propertyInfo.Name;
+                float value = (float)propertyInfo.GetValue(_currentBlendshape);
+                if (_headMapping.TryGetValue(fieldName, out int index))
+                {
+                    _headSkinMeshRenderer.SetBlendShapeWeightInterpolate(
+                        index,
+                        value * _weightMultiplier,
+                        Time.deltaTime
+                    );
+                }
+            }
+        }
+
+        public override void PurgeExcessBlendShapeFrames()
+        {
+            if (_blendShapesQueue.Count <= 0) return;
+            if (!CanPurge<BlendshapeFrame>(_blendShapesQueue.Peek())) return;
+            Logger.Info($"Purging {_blendShapesQueue.Peek().Count} frames", Logger.LogCategory.LipSync);
+            _blendShapesQueue.Dequeue();
+        }
+
+        public override void ClearQueue()
+        {
+            _blendShapesQueue = new Queue<Queue<BlendshapeFrame>>();
+            _currentBlendshape = new ARKitBlendShapes();
+        }
+
+        public override void EnqueueQueue(Queue<BlendshapeFrame> blendshapeFrames)
+        {
+            _blendShapesQueue.Enqueue(blendshapeFrames);
+        }
+
+        public override void EnqueueFrame(BlendshapeFrame blendshapeFrame)
+        {
+            if (_blendShapesQueue.Count == 0)
+            {
+                EnqueueQueue(new Queue<BlendshapeFrame>());
+            }
+            _blendShapesQueue.Peek().Enqueue(blendshapeFrame);
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiBlendShapeLipSync.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: ca3ca8129b12656449558f306c86f70d
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiOVRLipsync.cs

@@ -0,0 +1,162 @@
+using System.Collections.Generic;
+using System.Reflection;
+using Service;
+using UnityEngine;
+
+namespace Convai.Scripts.Utils.LipSync.Types
+{
+    public class ConvaiOVRLipsync : ConvaiVisemesLipSync
+    {
+        private int _firstIndex;
+
+        public override void Initialize(ConvaiLipSync convaiLipSync, ConvaiNPC convaiNPC)
+        {
+            base.Initialize(convaiLipSync, convaiNPC);
+            _firstIndex = convaiLipSync.firstIndex;
+        }
+
+        protected override Dictionary<string, string> GetHeadRegexMapping()
+        {
+            const string mouth = "[Mm]outh";
+            const string spacer = "[\\s_]*";
+            const string left = "[Ll]eft";
+            const string right = "[Rr]ight";
+            const string lower = "[Ll]ower";
+            const string upper = "[Uu]pper";
+            const string open = "[Oo]pen";
+            const string funnel = "[Ff]unnel";
+            const string pucker = "[Pp]ucker";
+            const string prefix = "(?:[A-Z]\\d{1,2}_)?";
+
+            return new Dictionary<string, string>
+            {
+                {"PP", $"{prefix}{mouth}{spacer}{pucker}"},
+                {"FF", $"{prefix}{mouth}{spacer}{funnel}"},
+                {"THL", $"{prefix}{mouth}{spacer}{lower}{spacer}[Dd]own{spacer}{left}"},
+                {"THR", $"{prefix}{mouth}{spacer}{lower}{spacer}[Dd]own{spacer}{right}"},
+                {"DDL", $"{prefix}{mouth}{spacer}[Pp]ress{spacer}{left}"},
+                {"DDR", $"{prefix}{mouth}{spacer}[Pp]ress{spacer}{right}"},
+                {"KK", $"{prefix}[Jj]aw{spacer}{open}"},
+                {"CHL", $"{prefix}{mouth}{spacer}[Ss]tretch{spacer}{left}"},
+                {"CHR", $"{prefix}{mouth}{spacer}[Ss]tretch{spacer}{right}"},
+                {"SSL", $"{prefix}{mouth}{spacer}[Ss]mile{spacer}{left}"},
+                {"SSR", $"{prefix}{mouth}{spacer}[Ss]mile{spacer}{right}"},
+                {"NNL", $"{prefix}[Nn]ose{spacer}[Ss]neer{spacer}{left}"},
+                {"NNR", $"{prefix}[Nn]ose{spacer}[Ss]neer{spacer}{right}"},
+                {"RRU", $"{prefix}{mouth}{spacer}[Rr]oll{spacer}{upper}"},
+                {"RRL", $"{prefix}{mouth}{spacer}[Rr]oll{spacer}{lower}"},
+                {"AA", $"{prefix}[Jj]aw{spacer}[Oo]pen"},
+                {"EL", $"{prefix}{mouth}{spacer}{upper}{spacer}[Uu]p{spacer}{left}"},
+                {"ER", $"{prefix}{mouth}{spacer}{upper}{spacer}[Uu]p{spacer}{right}"},
+                {"IHL", $"{prefix}{mouth}{spacer}[Ff]rown{spacer}{left}"},
+                {"IHR", $"{prefix}{mouth}{spacer}[Ff]rown{spacer}{right}"},
+                {"OU", $"{prefix}{mouth}{spacer}{pucker}"},
+                {"OH", $"{prefix}{mouth}{spacer}{funnel}"},
+            };
+        }
+
+        private void Update()
+        {
+            if (_currentViseme == null || _currentViseme.Sil == -2) return;
+
+            float weight;
+            List<int> knownIndexes = new List<int>();
+
+            UpdateJawBoneRotation(new Vector3(0.0f, 0.0f, -90.0f));
+            UpdateTongueBoneRotation(new Vector3(0.0f, 0.0f, -5.0f));
+
+            if (HasHeadSkinnedMeshRenderer)
+            {
+                foreach (PropertyInfo propertyInfo in typeof(Viseme).GetProperties())
+                {
+                    if (propertyInfo.PropertyType != typeof(float)) continue;
+
+                    string fieldName = propertyInfo.Name.ToUpper();
+                    float value = (float)propertyInfo.GetValue(_currentViseme);
+
+                    weight = fieldName switch
+                    {
+                        "KK" => 1.0f / 1.5f,
+                        "DD" => 1.0f / 0.7f,
+                        "CH" => 1.0f / 2.7f,
+                        "SS" => 1.0f / 1.5f,
+                        "NN" => 1.0f / 2.0f,
+                        "RR" => 1.0f / 0.9f,
+                        "AA" => 1.0f / 2.0f,
+                        "II" => 1.0f / 1.2f,
+                        "OH" => 1.2f,
+                        _ => 1.0f
+                    };
+
+                    foreach (string s in _possibleCombinations)
+                    {
+                        float weightThisFrame = value * weight * _weightMultiplier;
+                        string modifiedFieldName = fieldName + s;
+                        FindAndUpdateBlendWeight(_headSkinMeshRenderer, modifiedFieldName, weightThisFrame, knownIndexes, _headMapping);
+                    }
+                }
+            }
+
+            UpdateJawBoneRotation(new Vector3(0.0f, 0.0f, CalculateJawRotation()));
+            UpdateTongueBoneRotation(new Vector3(0.0f, 0.0f, CalculateTongueRotation()));
+
+            if (_teethSkinMeshRenderer.sharedMesh.blendShapeCount < (_firstIndex + 15)) return;
+
+            for (int i = 0; i < 15; i++)
+            {
+                float visemeValue = GetVisemeValueByIndex(i);
+                _teethSkinMeshRenderer.SetBlendShapeWeightInterpolate(_firstIndex + i, visemeValue * _weightMultiplier, Time.deltaTime);
+            }
+        }
+
+        private float CalculateJawRotation()
+        {
+            float totalWeight = 0.2f + 0.1f + 0.5f + 0.2f + 0.2f + 1.0f + 0.2f + 0.3f + 0.8f + 0.3f;
+            float rotation = (0.2f * _currentViseme.Th
+                             + 0.1f * _currentViseme.Dd
+                             + 0.5f * _currentViseme.Kk
+                             + 0.2f * _currentViseme.Nn
+                             + 0.2f * _currentViseme.Rr
+                             + 1.0f * _currentViseme.Aa
+                             + 0.2f * _currentViseme.E
+                             + 0.3f * _currentViseme.Ih
+                             + 0.8f * _currentViseme.Oh
+                             + 0.3f * _currentViseme.Ou) / totalWeight;
+
+            return -90.0f - rotation * 30f;
+        }
+
+        private float CalculateTongueRotation()
+        {
+            float totalWeight = 0.1f + 0.2f + 0.15f;
+            float rotation = (0.1f * _currentViseme.Th
+                             + 0.2f * _currentViseme.Nn
+                             + 0.15f * _currentViseme.Rr) / totalWeight;
+
+            return rotation * 80f - 5f;
+        }
+
+        private float GetVisemeValueByIndex(int index)
+        {
+            return index switch
+            {
+                0 => _currentViseme.Sil,
+                1 => _currentViseme.Pp,
+                2 => _currentViseme.Ff,
+                3 => _currentViseme.Th,
+                4 => _currentViseme.Dd,
+                5 => _currentViseme.Kk,
+                6 => _currentViseme.Ch,
+                7 => _currentViseme.Ss,
+                8 => _currentViseme.Nn,
+                9 => _currentViseme.Rr,
+                10 => _currentViseme.Aa,
+                11 => _currentViseme.E,
+                12 => _currentViseme.Ih,
+                13 => _currentViseme.Oh,
+                14 => _currentViseme.Ou,
+                _ => 0.0f
+            };
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiOVRLipsync.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 769391e6890ecb0459ada7f3c4fb1400
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiReallusionLipSync.cs

@@ -0,0 +1,117 @@
+using System.Collections.Generic;
+using System.Reflection;
+using Service;
+using UnityEngine;
+namespace Convai.Scripts.Utils.LipSync.Types
+{
+    public class ConvaiReallusionLipSync : ConvaiVisemesLipSync
+    {
+        protected override Dictionary<string, string> GetHeadRegexMapping()
+        {
+            string mouth = "[Mm]outh";
+            string lower = "[Ll]ower";
+            string spacer = "[\\s_]*";
+            string prefix = "(?:[A-Z]\\d{1,2}_)?";
+
+            return new Dictionary<string, string>()
+            {
+                {"PP", $"{prefix}[Vv]{spacer}[Ee]xplosive"},
+                {"FF", $"{prefix}[Vv]{spacer}[Dd]ental{spacer}[Ll]ip"},
+                {"TH", $"{prefix}{mouth}{spacer}[Dd]rop{spacer}{lower}"},
+                {"DDL", $"{prefix}{mouth}{spacer}[Dd]rop{spacer}[Ll]ower"},
+                {"DDU", $"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Uu]pper"},
+                {"KKL", $"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Ll]ower"},
+                {"KKU", $"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Uu]pper"},
+                {"CHL",$"{prefix}{mouth}[Dd]rop{spacer}[Ll]ower"},
+                {"CHU",$"{prefix}{mouth}[Dd]rop{spacer}[Uu]pper"},
+                {"CHO",$"{prefix}[Vv]{spacer}[Ll]ip{spacer}[Oo]pen"},
+                {"SSL", $"{prefix}{mouth}{spacer}[Dd]rop{spacer}[Ll]ower"},
+                {"SSU", $"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Uu]pper"},
+                {"NNL", $"{prefix}{mouth}{spacer}[Dd]rop{spacer}[Ll]ower"},
+                {"NNU", $"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Uu]pper"},
+                {"RR",$"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Uu]pper"},
+                {"AA", $"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Uu]pper"},
+                {"EL", $"{prefix}{mouth}{spacer}[Dd]rop{spacer}[Ll]ower"},
+                {"EU", $"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Uu]pper"},
+                {"IHL", $"{prefix}{mouth}{spacer}[Dd]rop{spacer}[Ll]ower"},
+                {"IHU",$"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Uu]pper"},
+                {"OH", $"{prefix}[Vv]{spacer}[Tt]ight{spacer}[Oo]"},
+                {"OU", $"{prefix}[Vv]{spacer}[Tt]ight{spacer}[Oo]"},
+            };
+        }
+
+        private void Update()
+        {
+            // Check if the dequeued frame is not null.
+            if (_currentViseme == null) return;
+            // Check if the frame represents silence (-2 is a placeholder for silence).
+            if (_currentViseme.Sil == -2) return;
+
+            float weight;
+            List<int> knownIndexs = new List<int>();
+            UpdateJawBoneRotation(new Vector3(0.0f, 0.0f, -90.0f));
+            UpdateTongueBoneRotation(new Vector3(0.0f, 0.0f, -5.0f));
+            if (HasHeadSkinnedMeshRenderer)
+            {
+                foreach (PropertyInfo propertyInfo in typeof(Viseme).GetProperties())
+                {
+                    if (propertyInfo.PropertyType != typeof(float)) continue;
+                    string fieldName = propertyInfo.Name.ToUpper();
+                    float value = (float)propertyInfo.GetValue(_currentViseme);
+                    weight = fieldName switch
+                    {
+                        "TH" => 0.5f,
+                        "DDL" => 0.2f / 0.7f,
+                        "DDU" => 0.5f / 2.7f,
+                        "KKL" => 0.5f / 1.5f,
+                        "KKU" => 1.0f / 1.5f,
+                        "CHL" => 0.7f / 2.7f,
+                        "CHU" => 1.0f / 2.7f,
+                        "CHO" => 1.0f / 2.7f,
+                        "SSL" => 0.5f / 1.5f,
+                        "SSU" => 1.0f / 1.5f,
+                        "NNL" => 0.5f / 2.0f,
+                        "NNU" => 1.0f / 2.0f,
+                        "RR" => 0.5f / 0.9f,
+                        "AA" => 1.0f / 2.0f,
+                        "EL" => 0.7f,
+                        "EU" => 0.3f,
+                        "IHL" => 0.7f / 1.2f,
+                        "IHU" => 0.5f / 1.2f,
+                        "OH" => 1.2f,
+                        _ => 1.0f
+                    };
+                    foreach (string s in _possibleCombinations)
+                    {
+                        float weightThisFrame = value * weight * _weightMultiplier;
+                        string modifiedFieldName = fieldName + s;
+                        FindAndUpdateBlendWeight(_headSkinMeshRenderer, modifiedFieldName, weightThisFrame, knownIndexs, _headMapping);
+                    }
+                }
+            }
+
+
+            UpdateJawBoneRotation(new Vector3(0.0f, 0.0f, -90.0f - (
+                        0.2f * _currentViseme.Th
+                        + 0.1f * _currentViseme.Dd
+                        + 0.5f * _currentViseme.Kk
+                        + 0.2f * _currentViseme.Nn
+                        + 0.2f * _currentViseme.Rr
+                        + 1.0f * _currentViseme.Aa
+                        + 0.2f * _currentViseme.E
+                        + 0.3f * _currentViseme.Ih
+                        + 0.8f * _currentViseme.Oh
+                        + 0.3f * _currentViseme.Ou
+                    )
+                    / (0.2f + 0.1f + 0.5f + 0.2f + 0.2f + 1.0f + 0.2f + 0.3f + 0.8f + 0.3f)
+                    * 30f));
+            UpdateTongueBoneRotation(new Vector3(0.0f, 0.0f, (
+                        0.1f * _currentViseme.Th
+                        + 0.2f * _currentViseme.Nn
+                        + 0.15f * _currentViseme.Rr
+                    )
+                    / (0.1f + 0.2f + 0.15f)
+                    * 80f - 5f));
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiReallusionLipSync.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 04cd2132229efcc4f9848b088c03e55b
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiVisemesLipSync.cs

@@ -0,0 +1,102 @@
+using System.Collections.Generic;
+using Service;
+using UnityEngine;
+namespace Convai.Scripts.Utils.LipSync.Types
+{
+    public abstract class ConvaiVisemesLipSync : ConvaiLipSyncApplicationBase
+    {
+        private const float FRAMERATE = 1f / 100.0f;
+        protected Queue<Queue<VisemesData>> _visemesDataQueue = new Queue<Queue<VisemesData>>();
+        protected Viseme _currentViseme;
+        protected List<string> _possibleCombinations = new List<string>();
+
+        public override void Initialize(ConvaiLipSync convaiLipSync, ConvaiNPC convaiNPC)
+        {
+            base.Initialize(convaiLipSync, convaiNPC);
+            _possibleCombinations = new List<string> { "", "R", "U", "L", "O" };
+            InvokeRepeating(nameof(UpdateBlendShape), 0, FRAMERATE);
+        }
+
+        public override void ClearQueue()
+        {
+            _visemesDataQueue = new Queue<Queue<VisemesData>>();
+            _currentViseme = new Viseme();
+        }
+
+        public override void PurgeExcessBlendShapeFrames()
+        {
+            if (_visemesDataQueue.Count == 0) return;
+            if (!CanPurge<VisemesData>(_visemesDataQueue.Peek())) return;
+            Logger.Info($"Purging {_visemesDataQueue.Peek().Count} Frames", Logger.LogCategory.LipSync);
+            _visemesDataQueue.Dequeue();
+        }
+
+
+
+        protected override void UpdateBlendShape()
+        {
+            if (_visemesDataQueue == null || _visemesDataQueue.Count <= 0)
+            {
+                _currentViseme = new Viseme();
+                return;
+            }
+            // Dequeue the next frame of visemes data from the faceDataList.
+            if (_visemesDataQueue.Peek().Count <= 0 || _visemesDataQueue.Peek() == null)
+            {
+                _visemesDataQueue.Dequeue();
+                return;
+            }
+            if (!_convaiNPC.IsCharacterTalking) return;
+
+            _currentViseme = _visemesDataQueue.Peek().Dequeue().Visemes;
+        }
+
+        protected void FindAndUpdateBlendWeight(SkinnedMeshRenderer renderer, string fieldName, float value, List<int> knownIndexs, Dictionary<string, int> mapping)
+        {
+            if (mapping.TryGetValue(fieldName, out int index))
+            {
+                if (!knownIndexs.Contains(index))
+                {
+                    knownIndexs.Add(index);
+                    UpdateWeight(renderer, index, value, true);
+                }
+                else
+                {
+                    UpdateWeight(_headSkinMeshRenderer, index, value, false);
+                }
+            }
+        }
+
+        protected virtual void UpdateWeight(SkinnedMeshRenderer renderer, int index, float value, bool firstTime)
+        {
+            if (value == 0f)
+            {
+                renderer.SetBlendShapeWeight(index, 0);
+                return;
+            }
+
+            if (FRAMERATE > Time.deltaTime)
+            {
+                renderer.SetBlendShapeWeight(index, (firstTime ? 0 : renderer.GetBlendShapeWeight(index)) + value);
+            }
+            else
+            {
+                renderer.SetBlendShapeWeightInterpolate(index, (firstTime ? 0 : renderer.GetBlendShapeWeight(index)) + value, Time.deltaTime);
+            }
+        }
+
+        public override void EnqueueQueue(Queue<VisemesData> visemesFrames)
+        {
+            _visemesDataQueue.Enqueue(visemesFrames);
+        }
+
+        public override void EnqueueFrame(VisemesData viseme)
+        {
+            if (_visemesDataQueue.Count == 0)
+            {
+                EnqueueQueue(new Queue<VisemesData>());
+            }
+            _visemesDataQueue.Peek().Enqueue(viseme);
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiVisemesLipSync.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 1917f6a14d9682440a27a434f0210496
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: