restructure

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/ConvaiLipSync.cs

@@ -0,0 +1,112 @@
+using System;
+using System.Collections.Generic;
+using Convai.Scripts.Runtime.Core;
+using Convai.Scripts.Runtime.Extensions;
+using Convai.Scripts.Runtime.Features.LipSync.Models;
+using Convai.Scripts.Runtime.Features.LipSync.Types;
+using Service;
+using UnityEngine;
+
+namespace Convai.Scripts.Runtime.Features.LipSync
+{
+    public class ConvaiLipSync : MonoBehaviour
+    {
+        [HideInInspector] public FaceModel faceModel = FaceModel.OvrModelName;
+
+        [field: SerializeField]
+        [field: Tooltip("Assign the skin renderers and its respective effectors, along with the bones used for Facial Expression")]
+        public FacialExpressionData FacialExpressionData { get; private set; } = new();
+
+        [field: SerializeField]
+        [field: Range(0f, 1f)]
+        [field: Tooltip("This decides how much blending will occur between two different blendshape frames")]
+        public float WeightBlendingPower { get; private set; } = 0.5f;
+
+        [SerializeField] private List<string> characterEmotions;
+
+        private ConvaiNPC _convaiNPC;
+        public ConvaiLipSyncApplicationBase ConvaiLipSyncApplicationBase { get; private set; }
+
+        /// <summary>
+        ///     This function will automatically set any of the unassigned skinned mesh renderers to appropriate values using regex
+        ///     based functions.
+        ///     Sets the references of the required variables
+        ///     Sets wait for lipsync to true
+        /// </summary>
+        private void Start()
+        {
+            FindSkinMeshRenderer();
+            _convaiNPC = GetComponent<ConvaiNPC>();
+            ConvaiLipSyncApplicationBase = gameObject.GetOrAddComponent<ConvaiVisemesLipSync>();
+            ConvaiLipSyncApplicationBase.Initialize(this, _convaiNPC);
+            SetCharacterLipSyncing(true);
+        }
+
+        private void OnDisable()
+        {
+            StopLipSync();
+        }
+
+
+        private void OnApplicationQuit()
+        {
+            StopLipSync();
+        }
+
+        public event Action<bool> OnCharacterLipSyncing;
+
+        private void FindSkinMeshRenderer()
+        {
+            if (FacialExpressionData.Head.Renderer == null)
+                FacialExpressionData.Head.Renderer = transform.GetComponentOnChildWithMatchingRegex<SkinnedMeshRenderer>("(.*_Head|CC_Base_Body)");
+            if (FacialExpressionData.Teeth.Renderer == null)
+                FacialExpressionData.Teeth.Renderer = transform.GetComponentOnChildWithMatchingRegex<SkinnedMeshRenderer>("(.*_Teeth|CC_Base_Teeth)");
+            if (FacialExpressionData.Tongue.Renderer == null)
+                FacialExpressionData.Tongue.Renderer = transform.GetComponentOnChildWithMatchingRegex<SkinnedMeshRenderer>("(.*_Tongue|CC_Base_Tongue)");
+        }
+
+        /// <summary>
+        ///     Overrides the character emotions list
+        /// </summary>
+        /// <param name="newEmotions">list of new emotions</param>
+        public void SetCharacterEmotions(List<string> newEmotions)
+        {
+            characterEmotions = new List<string>(newEmotions);
+        }
+
+        /// <summary>
+        ///     Returns Direct reference of the character emotions [Not Recommended to directly change this list]
+        /// </summary>
+        /// <returns></returns>
+        public List<string> GetCharacterEmotions()
+        {
+            return characterEmotions;
+        }
+
+
+        /// <summary>
+        ///     Fires an event with update the Character Lip Syncing State
+        /// </summary>
+        /// <param name="value"></param>
+        private void SetCharacterLipSyncing(bool value)
+        {
+            OnCharacterLipSyncing?.Invoke(value);
+        }
+
+        /// <summary>
+        ///     Purges the latest chuck of lipsync frames
+        /// </summary>
+        public void PurgeExcessFrames()
+        {
+            ConvaiLipSyncApplicationBase?.PurgeExcessBlendShapeFrames();
+        }
+
+        /// <summary>
+        ///     Stops the Lipsync by clearing the frames queue
+        /// </summary>
+        public void StopLipSync()
+        {
+            ConvaiLipSyncApplicationBase?.ClearQueue();
+        }
+    }
+}

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/ConvaiLipSync.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 88bce56f6985ef84f8835a0152628fa1
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/ConvaiLipSyncApplicationBase.cs

@@ -0,0 +1,126 @@
+using System.Collections.Generic;
+using Convai.Scripts.Runtime.Core;
+using Convai.Scripts.Runtime.Features.LipSync;
+using Convai.Scripts.Runtime.Features.LipSync.Models;
+using Service;
+using UnityEngine;
+
+namespace Convai.Scripts.Runtime.Features
+{
+    /// <summary>
+    ///     This Class will serve as a base for any method of Lipsync that Convai will develop or use
+    /// </summary>
+    public abstract class ConvaiLipSyncApplicationBase : MonoBehaviour
+    {
+        /// <summary>
+        ///     Reference to the NPC on which lipsync will be applied
+        /// </summary>
+        protected ConvaiNPC ConvaiNPC;
+
+        /// <summary>
+        ///     Cached Reference of Facial Expression Data
+        /// </summary>
+        protected FacialExpressionData FacialExpressionData;
+
+        /// <summary>
+        ///     Cached Reference of WeightBlendingPower
+        /// </summary>
+        protected float WeightBlendingPower;
+
+        /// <summary>
+        ///     Initializes and setup up of the things necessary for lipsync to work
+        /// </summary>
+        /// <param name="convaiLipSync"></param>
+        /// <param name="convaiNPC"></param>
+        public virtual void Initialize(ConvaiLipSync convaiLipSync, ConvaiNPC convaiNPC)
+        {
+            FacialExpressionData = convaiLipSync.FacialExpressionData;
+            WeightBlendingPower = convaiLipSync.WeightBlendingPower;
+            HasHeadSkinnedMeshRenderer = FacialExpressionData.Head.Renderer != null;
+            HasTeethSkinnedMeshRenderer = FacialExpressionData.Teeth.Renderer != null;
+            HasTongueSkinnedMeshRenderer = FacialExpressionData.Tongue.Renderer != null;
+            HasJawBone = FacialExpressionData.JawBone != null;
+            HasTongueBone = FacialExpressionData.TongueBone != null;
+            ConvaiNPC = convaiNPC;
+        }
+
+        /// <summary>
+        ///     Updates the tongue bone rotation to the new rotation
+        /// </summary>
+        /// <param name="newRotation"></param>
+        protected void UpdateTongueBoneRotation(Vector3 newRotation)
+        {
+            if (!HasTongueBone) return;
+            FacialExpressionData.TongueBone.transform.localEulerAngles = newRotation;
+        }
+
+        /// <summary>
+        ///     Updates the jaw bone rotation to the new rotation
+        /// </summary>
+        /// <param name="newRotation"></param>
+        protected void UpdateJawBoneRotation(Vector3 newRotation)
+        {
+            if (!HasJawBone) return;
+            FacialExpressionData.JawBone.transform.localEulerAngles = newRotation;
+        }
+
+
+        /// <summary>
+        ///     This removes the excess frames in the queue
+        /// </summary>
+        public abstract void PurgeExcessBlendShapeFrames();
+
+        /// <summary>
+        ///     This resets the whole queue of the frames
+        /// </summary>
+        protected bool CanPurge<T>(Queue<T> queue)
+        {
+            // ? Should I hardcode the limiter for this check
+            return queue.Count < 10;
+        }
+
+        public abstract void ClearQueue();
+
+        /// <summary>
+        ///     Adds blendshape frames in the queue
+        /// </summary>
+        /// <param name="blendshapeFrames"></param>
+        public virtual void EnqueueQueue(Queue<ARKitBlendShapes> blendshapeFrames)
+        {
+        }
+
+        /// <summary>
+        ///     Adds Visemes frames in the list
+        /// </summary>
+        /// <param name="visemesFrames"></param>
+        public virtual void EnqueueQueue(Queue<VisemesData> visemesFrames)
+        {
+        }
+
+        /// <summary>
+        ///     Adds a blendshape frame in the last queue
+        /// </summary>
+        /// <param name="blendshapeFrame"></param>
+        public virtual void EnqueueFrame(ARKitBlendShapes blendshapeFrame)
+        {
+        }
+
+        /// <summary>
+        ///     Adds a viseme frame to the last element of the list
+        /// </summary>
+        /// <param name="viseme"></param>
+        public virtual void EnqueueFrame(VisemesData viseme)
+        {
+        }
+
+        #region Null States of References
+
+        protected bool HasHeadSkinnedMeshRenderer { get; private set; }
+        protected bool HasTeethSkinnedMeshRenderer { get; private set; }
+        protected bool HasTongueSkinnedMeshRenderer { get; private set; }
+        private bool HasJawBone { get; set; }
+        private bool HasTongueBone { get; set; }
+
+        #endregion
+    }
+}

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/ConvaiLipSyncApplicationBase.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: bb231034f5b2dee4494498fe9117bda1
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/LipSyncBlendFrameData.cs

@@ -0,0 +1,96 @@
+using System.Collections.Generic;
+using Convai.Scripts.Runtime.Core;
+using Service;
+using UnityEngine;
+
+namespace Convai.Scripts.Runtime.Features
+{
+    public class LipSyncBlendFrameData
+    {
+        #region FrameType enum
+
+        public enum FrameType
+        {
+            Visemes,
+            Blendshape
+        }
+
+        #endregion
+
+        private readonly Queue<ARKitBlendShapes> _blendShapeFrames = new();
+        private readonly FrameType _frameType;
+        private readonly GetResponseResponse _getResponseResponse;
+        private readonly int _totalFrames;
+        private readonly Queue<VisemesData> _visemesFrames = new();
+
+        private int _framesCaptured;
+        private bool _partiallyProcessed;
+
+        public LipSyncBlendFrameData(int totalFrames, GetResponseResponse response, FrameType frameType)
+        {
+            _totalFrames = totalFrames;
+            _framesCaptured = 0;
+            _getResponseResponse = response;
+            _frameType = frameType;
+            //ConvaiLogger.DebugLog($"Total Frames: {_totalFrames} | {response.AudioResponse.TextData}", ConvaiLogger.LogCategory.LipSync);
+        }
+
+        public void Enqueue(ARKitBlendShapes blendShapeFrame)
+        {
+            _blendShapeFrames.Enqueue(blendShapeFrame);
+            _framesCaptured++;
+        }
+
+        public void Enqueue(VisemesData visemesData)
+        {
+            _visemesFrames.Enqueue(visemesData);
+        }
+
+        public void Process(ConvaiNPC npc)
+        {
+            if (!_partiallyProcessed)
+                npc.EnqueueResponse(_getResponseResponse);
+            switch (_frameType)
+            {
+                case FrameType.Visemes:
+                    npc.convaiLipSync.ConvaiLipSyncApplicationBase.EnqueueQueue(new Queue<VisemesData>(_visemesFrames));
+                    break;
+                case FrameType.Blendshape:
+                    npc.convaiLipSync.ConvaiLipSyncApplicationBase.EnqueueQueue(new Queue<ARKitBlendShapes>(_blendShapeFrames));
+                    break;
+            }
+
+            npc.AudioManager.SetWaitForCharacterLipSync(false);
+        }
+
+        public void ProcessPartially(ConvaiNPC npc)
+        {
+            if (!_partiallyProcessed)
+            {
+                _partiallyProcessed = true;
+                npc.EnqueueResponse(_getResponseResponse);
+                npc.AudioManager.SetWaitForCharacterLipSync(false);
+            }
+
+            switch (_frameType)
+            {
+                case FrameType.Visemes:
+                    while (_visemesFrames.Count != 0) npc.convaiLipSync.ConvaiLipSyncApplicationBase.EnqueueFrame(_visemesFrames.Dequeue());
+                    break;
+                case FrameType.Blendshape:
+                    while (_blendShapeFrames.Count != 0) npc.convaiLipSync.ConvaiLipSyncApplicationBase.EnqueueFrame(_blendShapeFrames.Dequeue());
+                    break;
+            }
+        }
+
+        public bool CanPartiallyProcess()
+        {
+            return _framesCaptured > Mathf.Min(21, _totalFrames * 0.7f);
+        }
+
+        public bool CanProcess()
+        {
+            return _framesCaptured == _totalFrames;
+        }
+    }
+}

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/LipSyncBlendFrameData.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 31db1a9457d64f3d936ff7f5aabfb193
+timeCreated: 1708491067

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Models.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: c45a9d44677146f4eb628784af6e9461
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Models/BlendShapesIndexEffector.cs

@@ -0,0 +1,13 @@
+using System;
+using UnityEngine;
+
+namespace Convai.Scripts.Runtime.Features.LipSync.Models
+{
+    [Serializable]
+    public class BlendShapesIndexEffector
+    {
+        [SerializeField] public int index;
+
+        [SerializeField] public float effectPercentage;
+    }
+}

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Models/BlendShapesIndexEffector.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 2e635a84c98545eca5224853f9d59618
+timeCreated: 1722024855

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Models/FacialExpressionData.cs

@@ -0,0 +1,33 @@
+using System;
+using Convai.Scripts.Runtime.Features.LipSync.Visemes;
+using UnityEngine;
+
+// ReSharper disable InconsistentNaming
+
+namespace Convai.Scripts.Runtime.Features.LipSync.Models
+{
+    [Serializable]
+    public class FacialExpressionData
+    {
+        [Tooltip("Assign the Skin Renderer and Effector for Head")]
+        public SkinMeshRendererData Head;
+
+        [Tooltip("Assign the Skin Renderer and Effector for Teeth")]
+        public SkinMeshRendererData Teeth;
+
+        [Tooltip("Assign the Skin Renderer and Effector for Tongue")]
+        public SkinMeshRendererData Tongue;
+
+        [Tooltip("Assign the Viseme Bone Effector List for Jaw")]
+        public VisemeBoneEffectorList JawBoneEffector;
+
+        [Tooltip("Assign the Viseme Bone Effector List for Tongue")]
+        public VisemeBoneEffectorList TongueBoneEffector;
+
+        [Tooltip("Assign the bone which effects movement of jaw")]
+        public GameObject JawBone;
+
+        [Tooltip("Assign the bone which effects movement of tongue")]
+        public GameObject TongueBone;
+    }
+}

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Models/FacialExpressionData.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: eeb1390214d44f679a8f26e8e95a40e1
+timeCreated: 1722028711

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Models/SkinMeshRendererData.cs

@@ -0,0 +1,16 @@
+using System;
+using Convai.Scripts.Runtime.Features.LipSync.Visemes;
+using UnityEngine;
+
+namespace Convai.Scripts.Runtime.Features.LipSync.Models
+{
+    [Serializable]
+    public class SkinMeshRendererData
+    {
+        public SkinnedMeshRenderer Renderer;
+        public VisemeEffectorsList VisemeEffectorsList;
+
+        [Tooltip("Lower and Upper bound of the Blendshape weight, Ex: 0-1, or 0-100")]
+        public Vector2 WeightBounds = new(0, 1);
+    }
+}

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Models/SkinMeshRendererData.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: a23bc1b310f54ba7849b92bcb381cf4d
+timeCreated: 1722028729

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Types.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 6158a9323c720f5408c5b7caa77405cc
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiBlendShapeLipSync.cs

@@ -0,0 +1,204 @@
+using System.Collections.Generic;
+using Convai.Scripts.Runtime.Core;
+using Convai.Scripts.Runtime.Features.LipSync;
+using Convai.Scripts.Runtime.LoggerSystem;
+using Service;
+
+/*
+! This class is a Work in progress and can produce un expected results, Convai does not advise to use this class in production, please use this with extreme caution
+*/
+
+namespace Convai.Scripts.Runtime.Features
+{
+    public class ConvaiBlendShapeLipSync : ConvaiLipSyncApplicationBase
+    {
+        private const float A2XFRAMERATE = 1f / 30f;
+        private Queue<Queue<ARKitBlendShapes>> _blendShapesQueue = new();
+        private ARKitBlendShapes _currentBlendshape;
+
+        // private void Update()
+        // {
+        //     if (_currentBlendshape == null) return;
+        //     UpdateJawBoneRotation(new Vector3(0.0f, 0.0f, -90.0f - _currentBlendshape.JawOpen * 30f));
+        //     UpdateTongueBoneRotation(new Vector3(0.0f, 0.0f, -5.0f * _currentBlendshape.TongueOut));
+        //     if (!HasHeadSkinnedMeshRenderer) return;
+        //     foreach (PropertyInfo propertyInfo in typeof(ARKitBlendShapes).GetProperties())
+        //     {
+        //         if (propertyInfo.PropertyType != typeof(float)) continue;
+        //         string fieldName = propertyInfo.Name;
+        //         float value = (float)propertyInfo.GetValue(_currentBlendshape);
+        //         if (HeadMapping.TryGetValue(fieldName, out int index))
+        //             HeadSkinMeshRenderer.SetBlendShapeWeightInterpolate(
+        //                 index,
+        //                 value * WeightMultiplier,
+        //                 Time.deltaTime
+        //             );
+        //     }
+        // }
+
+        protected Dictionary<string, string> GetHeadRegexMapping()
+        {
+            #region Regex Finders
+
+            string prefix = "(?:[A-Z]\\d{1,2}_)?";
+            string spacer = "[\\s_]*";
+            string mouth = "[Mm]outh";
+            string nose = "[Nn]ose";
+            string left = "[Ll]eft";
+            string right = "[Rr]ight";
+            string up = "[Uu]p";
+            string down = "[Dd]own";
+            string lower = "[Ll]ower";
+            string upper = "[Uu]pper";
+            string open = "[Oo]pen";
+            string funnel = "[Ff]unnel";
+            string pucker = "[Pp]ucker";
+            string sneer = "[Ss]neer";
+            string cheek = "[Cc]heek";
+            string squint = "[Ss]quint";
+            string brow = "[Bb]row";
+            string outer = "[Oo]uter";
+            string inner = "[Ii]nner";
+            string eye = "[Ee]ye";
+            string blink = "[Bb]link";
+            string look = "[Ll]ook";
+            string In = "[Ii]n";
+            string Out = "[Oo]ut";
+            string wide = "[Ww]ide";
+            string forward = "[Ff]orward";
+            string jaw = "[Jj]aw";
+            string close = "[Cc]lose";
+            string smile = "[Ss]mile";
+            string frown = "[Ff]rown";
+            string dimple = "[Dd]imple";
+            string stretch = "[Ss]tretch";
+            string roll = "[Rr]oll";
+            string shrug = "[Ss]hrug";
+            string press = "[Pp]ress";
+
+            #endregion
+
+            return new Dictionary<string, string>
+            {
+                { "TougueOut", $"{prefix}[Tt]ougue{spacer}[Oo]ut" },
+
+                { "NoseSneerRight", $"{prefix}{nose}{spacer}{sneer}{spacer}{right}" },
+                { "NoseSneerLeft", $"{prefix}{nose}{spacer}{sneer}{spacer}{left}" },
+
+                { "CheekSquintRight", $"{prefix}{cheek}{spacer}{squint}{spacer}{right}" },
+                { "CheekSquintLeft", $"{prefix}{cheek}{spacer}{squint}{spacer}{left}" },
+                { "CheekPuff", $"{prefix}{cheek}{spacer}[Pp]uff" },
+
+                { "BrowDownLeft", $"{prefix}{brow}{spacer}{down}{spacer}{left}" },
+                { "BrowDownRight", $"{prefix}{brow}{spacer}{down}{spacer}{right}" },
+                { "BrowInnerUp", $"{prefix}{brow}{spacer}{inner}{spacer}{up}" },
+                { "BrowOuterUpLeft", $"{prefix}{brow}{spacer}{outer}{spacer}{up}{spacer}{left}" },
+                { "BrowOuterUpRight", $"{prefix}{brow}{spacer}{outer}{spacer}{up}{spacer}{right}" },
+
+                { "EyeBlinkLeft", $"{prefix}{eye}{spacer}{blink}{spacer}{left}" },
+                { "EyeLookDownLeft", $"{prefix}{eye}{spacer}{look}{spacer}{In}{left}" },
+                { "EyeLookInLeft", $"{prefix}{eye}{spacer}{look}{spacer}{In}{spacer}{left}" },
+                { "EyeLookOutLeft", $"{prefix}{eye}{spacer}{look}{spacer}{Out}{spacer}{left}" },
+                { "EyeLookUpLeft", $"{prefix}{eye}{spacer}{look}{spacer}{up}{spacer}{left}" },
+                { "EyeSquintLeft", $"{prefix}{eye}{spacer}{squint}{spacer}{left}" },
+                { "EyeWideLeft", $"{prefix}{eye}{spacer}{wide}{spacer}{left}" },
+
+                { "EyeBlinkRight", $"{prefix}{eye}{spacer}{blink}{spacer}{right}" },
+                { "EyeLookDownRight", $"{prefix}{eye}{spacer}{look}{spacer}{In}{right}" },
+                { "EyeLookInRight", $"{prefix}{eye}{spacer}{look}{spacer}{In}{spacer}{right}" },
+                { "EyeLookOutRight", $"{prefix}{eye}{spacer}{look}{spacer}{Out}{spacer}{right}" },
+                { "EyeLookUpRight", $"{prefix}{eye}{spacer}{look}{spacer}{up}{spacer}{right}" },
+                { "EyeSquintRight", $"{prefix}{eye}{spacer}{squint}{spacer}{right}" },
+                { "EyeWideRight", $"{prefix}{eye}{spacer}{wide}{spacer}{right}" },
+
+                { "JawForward", $"{prefix}{jaw}{spacer}{forward}" },
+                { "JawLeft", $"{prefix}{jaw}{spacer}{left}" },
+                { "JawRight", $"{prefix}{jaw}{spacer}{right}" },
+                { "JawOpen", $"{prefix}{jaw}{spacer}{open}" },
+
+                { "MouthClose", $"{prefix}{mouth}{spacer}{close}" },
+                { "MouthFunnel", $"{prefix}{mouth}{spacer}{funnel}" },
+                { "MouthPucker", $"{prefix}{mouth}{spacer}{pucker}" },
+
+                { "Mouthleft", $"{prefix}{mouth}{spacer}{left}" },
+                { "MouthRight", $"{prefix}{mouth}{spacer}{right}" },
+
+                { "MouthSmileLeft", $"{prefix}{mouth}{spacer}{smile}{spacer}{left}" },
+                { "MouthSmileRight", $"{prefix}{mouth}{spacer}{smile}{spacer}{right}" },
+
+                { "MouthFrownLeft", $"{prefix}{mouth}{spacer}{frown}{spacer}{left}" },
+                { "MouthFrownRight", $"{prefix}{mouth}{spacer}{frown}{spacer}{right}" },
+
+                { "MouthDimpleLeft", $"{prefix}{mouth}{spacer}{dimple}{spacer}{left}" },
+                { "MouthDimpleRight", $"{prefix}{mouth}{spacer}{dimple}{spacer}{right}" },
+
+                { "MouthStretchLeft", $"{prefix}{mouth}{spacer}{stretch}{spacer}{left}" },
+                { "MouthStretchRight", $"{prefix}{mouth}{spacer}{stretch}{spacer}{right}" },
+
+                { "MouthRollLower", $"{prefix}{mouth}{spacer}{roll}{spacer}{lower}" },
+                { "MouthRollUpper", $"{prefix}{mouth}{spacer}{roll}{spacer}{upper}" },
+
+                { "MouthShrugLower", $"{prefix}{mouth}{spacer}{shrug}{spacer}{lower}" },
+                { "MouthShrugUpper", $"{prefix}{mouth}{spacer}{shrug}{spacer}{upper}" },
+
+                { "MouthPressLeft", $"{prefix}{mouth}{spacer}{press}{spacer}{left}" },
+                { "MouthPressRight", $"{prefix}{mouth}{spacer}{press}{spacer}{right}" },
+
+                { "MouthLowerDownLeft", $"{prefix}{mouth}{spacer}{lower}{spacer}{down}{spacer}{left}" },
+                { "MouthLowerDownRight", $"{prefix}{mouth}{spacer}{lower}{spacer}{down}{spacer}{right}" },
+
+                { "MouthUpperUpLeft", $"{prefix}{mouth}{spacer}{upper}{spacer}{up}{spacer}{left}" },
+                { "MouthUpperUpRight", $"{prefix}{mouth}{spacer}{upper}{spacer}{up}{spacer}{right}" }
+            };
+        }
+
+        public override void Initialize(ConvaiLipSync convaiLipSync, ConvaiNPC convaiNPC)
+        {
+            base.Initialize(convaiLipSync, convaiNPC);
+            InvokeRepeating(nameof(UpdateBlendShape), 0, A2XFRAMERATE);
+        }
+
+        protected virtual void UpdateBlendShape()
+        {
+            if (_blendShapesQueue == null || _blendShapesQueue.Count <= 0)
+            {
+                _currentBlendshape = new ARKitBlendShapes();
+                return;
+            }
+
+            if (_blendShapesQueue.Peek().Count == 0)
+            {
+                _blendShapesQueue.Dequeue();
+                return;
+            }
+
+            if (!ConvaiNPC.IsCharacterTalking) return;
+            _currentBlendshape = _blendShapesQueue.Peek().Dequeue();
+        }
+
+        public override void PurgeExcessBlendShapeFrames()
+        {
+            if (_blendShapesQueue.Count <= 0) return;
+            if (!CanPurge(_blendShapesQueue.Peek())) return;
+            ConvaiLogger.Info($"Purging {_blendShapesQueue.Peek().Count} frames", ConvaiLogger.LogCategory.LipSync);
+            _blendShapesQueue.Dequeue();
+        }
+
+        public override void ClearQueue()
+        {
+            _blendShapesQueue = new Queue<Queue<ARKitBlendShapes>>();
+            _currentBlendshape = new ARKitBlendShapes();
+        }
+
+        public override void EnqueueQueue(Queue<ARKitBlendShapes> blendshapeFrames)
+        {
+            _blendShapesQueue.Enqueue(blendshapeFrames);
+        }
+
+        public override void EnqueueFrame(ARKitBlendShapes blendshapeFrame)
+        {
+            if (_blendShapesQueue.Count == 0) EnqueueQueue(new Queue<ARKitBlendShapes>());
+            _blendShapesQueue.Peek().Enqueue(blendshapeFrame);
+        }
+    }
+}

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiBlendShapeLipSync.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: ca3ca8129b12656449558f306c86f70d
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiVisemesLipSync.cs

@@ -0,0 +1,150 @@
+using System.Collections.Generic;
+using Convai.Scripts.Runtime.Core;
+using Convai.Scripts.Runtime.Extensions;
+using Convai.Scripts.Runtime.Features.LipSync.Models;
+using Convai.Scripts.Runtime.Features.LipSync.Visemes;
+using Convai.Scripts.Runtime.LoggerSystem;
+using Service;
+using UnityEngine;
+
+namespace Convai.Scripts.Runtime.Features.LipSync.Types
+{
+    public class ConvaiVisemesLipSync : ConvaiLipSyncApplicationBase
+    {
+        private const float FRAMERATE = 1f / 100.0f;
+        private readonly Viseme _defaultViseme = new();
+        private Viseme _currentViseme;
+        private Queue<Queue<VisemesData>> _visemesDataQueue = new();
+
+        private void LateUpdate()
+        {
+            // Check if the dequeued frame is not null.
+            if (_currentViseme == null) return;
+            // Check if the frame represents silence (-2 is a placeholder for silence).
+            if (Mathf.Approximately(_currentViseme.Sil, -2)) return;
+
+
+            UpdateJawBoneRotation(new Vector3(0.0f, 0.0f, -90.0f));
+            UpdateTongueBoneRotation(new Vector3(0.0f, 0.0f, -5.0f));
+
+            if (HasHeadSkinnedMeshRenderer)
+                UpdateMeshRenderer(FacialExpressionData.Head);
+            if (HasTeethSkinnedMeshRenderer)
+                UpdateMeshRenderer(FacialExpressionData.Teeth);
+            if (HasTongueSkinnedMeshRenderer)
+                UpdateMeshRenderer(FacialExpressionData.Tongue);
+
+            UpdateJawBoneRotation(new Vector3(0.0f, 0.0f, -90.0f - CalculateBoneEffect(FacialExpressionData.JawBoneEffector) * 30f));
+            UpdateTongueBoneRotation(new Vector3(0.0f, 0.0f, CalculateBoneEffect(FacialExpressionData.TongueBoneEffector) * 80f - 5f));
+        }
+
+        public override void Initialize(ConvaiLipSync convaiLipSync, ConvaiNPC convaiNPC)
+        {
+            base.Initialize(convaiLipSync, convaiNPC);
+            InvokeRepeating(nameof(UpdateBlendShape), 0, FRAMERATE);
+        }
+
+        public override void ClearQueue()
+        {
+            _visemesDataQueue = new Queue<Queue<VisemesData>>();
+            _currentViseme = new Viseme();
+        }
+
+        public override void PurgeExcessBlendShapeFrames()
+        {
+            if (_visemesDataQueue.Count == 0) return;
+            if (!CanPurge(_visemesDataQueue.Peek())) return;
+            ConvaiLogger.Info($"Purging {_visemesDataQueue.Peek().Count} Frames", ConvaiLogger.LogCategory.LipSync);
+            _visemesDataQueue.Dequeue();
+        }
+
+        public override void EnqueueQueue(Queue<VisemesData> visemesFrames)
+        {
+            _visemesDataQueue.Enqueue(visemesFrames);
+        }
+
+        public override void EnqueueFrame(VisemesData viseme)
+        {
+            if (_visemesDataQueue.Count == 0) EnqueueQueue(new Queue<VisemesData>());
+            _visemesDataQueue.Peek().Enqueue(viseme);
+        }
+
+
+        protected void UpdateBlendShape()
+        {
+            if (_visemesDataQueue is not { Count: > 0 })
+            {
+                _currentViseme = _defaultViseme;
+                return;
+            }
+
+            // Dequeue the next frame of visemes data from the faceDataList.
+            if (_visemesDataQueue.Peek() == null || _visemesDataQueue.Peek().Count <= 0)
+            {
+                _visemesDataQueue.Dequeue();
+                return;
+            }
+
+            if (!ConvaiNPC.IsCharacterTalking) return;
+
+            _currentViseme = _visemesDataQueue.Peek().Dequeue().Visemes;
+        }
+
+        private float CalculateBoneEffect(VisemeBoneEffectorList boneEffectorList)
+        {
+            if (boneEffectorList is null) return 0;
+            return (
+                       boneEffectorList.sil * _currentViseme.Sil +
+                       boneEffectorList.pp * _currentViseme.Pp +
+                       boneEffectorList.ff * _currentViseme.Ff +
+                       boneEffectorList.th * _currentViseme.Th +
+                       boneEffectorList.dd * _currentViseme.Dd +
+                       boneEffectorList.kk * _currentViseme.Kk +
+                       boneEffectorList.ch * _currentViseme.Ch +
+                       boneEffectorList.ss * _currentViseme.Ss +
+                       boneEffectorList.nn * _currentViseme.Nn +
+                       boneEffectorList.rr * _currentViseme.Rr +
+                       boneEffectorList.aa * _currentViseme.Aa +
+                       boneEffectorList.e * _currentViseme.E +
+                       boneEffectorList.ih * _currentViseme.Ih +
+                       boneEffectorList.oh * _currentViseme.Oh +
+                       boneEffectorList.ou * _currentViseme.Ou
+                   )
+                   / boneEffectorList.Total;
+        }
+
+        private void UpdateMeshRenderer(SkinMeshRendererData data)
+        {
+            VisemeEffectorsList effectorsList = data.VisemeEffectorsList;
+            SkinnedMeshRenderer skinnedMesh = data.Renderer;
+            Vector2 bounds = data.WeightBounds;
+            if (effectorsList == null) return;
+            Dictionary<int, float> finalModifiedValuesDictionary = new();
+            CalculateBlendShapeEffect(effectorsList.pp, ref finalModifiedValuesDictionary, _currentViseme.Pp);
+            CalculateBlendShapeEffect(effectorsList.ff, ref finalModifiedValuesDictionary, _currentViseme.Ff);
+            CalculateBlendShapeEffect(effectorsList.th, ref finalModifiedValuesDictionary, _currentViseme.Th);
+            CalculateBlendShapeEffect(effectorsList.dd, ref finalModifiedValuesDictionary, _currentViseme.Dd);
+            CalculateBlendShapeEffect(effectorsList.kk, ref finalModifiedValuesDictionary, _currentViseme.Kk);
+            CalculateBlendShapeEffect(effectorsList.ch, ref finalModifiedValuesDictionary, _currentViseme.Ch);
+            CalculateBlendShapeEffect(effectorsList.ss, ref finalModifiedValuesDictionary, _currentViseme.Ss);
+            CalculateBlendShapeEffect(effectorsList.nn, ref finalModifiedValuesDictionary, _currentViseme.Nn);
+            CalculateBlendShapeEffect(effectorsList.rr, ref finalModifiedValuesDictionary, _currentViseme.Rr);
+            CalculateBlendShapeEffect(effectorsList.aa, ref finalModifiedValuesDictionary, _currentViseme.Aa);
+            CalculateBlendShapeEffect(effectorsList.e, ref finalModifiedValuesDictionary, _currentViseme.E);
+            CalculateBlendShapeEffect(effectorsList.ih, ref finalModifiedValuesDictionary, _currentViseme.Ih);
+            CalculateBlendShapeEffect(effectorsList.oh, ref finalModifiedValuesDictionary, _currentViseme.Oh);
+            CalculateBlendShapeEffect(effectorsList.ou, ref finalModifiedValuesDictionary, _currentViseme.Ou);
+            foreach (KeyValuePair<int, float> keyValuePair in finalModifiedValuesDictionary)
+                skinnedMesh.SetBlendShapeWeightInterpolate(keyValuePair.Key, keyValuePair.Value * bounds.y - bounds.x, WeightBlendingPower);
+        }
+
+        private static void CalculateBlendShapeEffect(List<BlendShapesIndexEffector> effectors, ref Dictionary<int, float> dictionary, float value)
+        {
+            foreach (BlendShapesIndexEffector blendShapesIndexEffector in effectors)
+                if (dictionary.ContainsKey(blendShapesIndexEffector.index))
+                    dictionary[blendShapesIndexEffector.index] += value * blendShapesIndexEffector.effectPercentage;
+                else
+                    dictionary[blendShapesIndexEffector.index] = value * blendShapesIndexEffector.effectPercentage;
+        }
+    }
+}

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiVisemesLipSync.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 1917f6a14d9682440a27a434f0210496
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Visemes.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 7d67d884b024af84cb3e3a450c6a742a
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Visemes/VisemeBoneEffectorList.cs

@@ -0,0 +1,50 @@
+using Convai.Scripts.Runtime.Attributes;
+using UnityEngine;
+
+namespace Convai.Scripts.Runtime.Features.LipSync.Visemes
+{
+    [CreateAssetMenu(fileName = "Convai Viseme Bone Effector", menuName = "Convai/Expression/Visemes Bone Effector", order = 0)]
+    public class VisemeBoneEffectorList : ScriptableObject
+    {
+        [SerializeField] public float sil;
+        [SerializeField] public float pp;
+        [SerializeField] public float ff;
+        [SerializeField] public float th;
+        [SerializeField] public float dd;
+        [SerializeField] public float kk;
+        [SerializeField] public float ch;
+        [SerializeField] public float ss;
+        [SerializeField] public float nn;
+        [SerializeField] public float rr;
+        [SerializeField] public float aa;
+        [SerializeField] public float e;
+        [SerializeField] public float ih;
+        [SerializeField] public float oh;
+        [SerializeField] public float ou;
+
+
+        [field: SerializeField]
+        [field: ReadOnly]
+        public float Total { get; private set; }
+
+        private void OnValidate()
+        {
+            Total = 0;
+            Total += sil;
+            Total += pp;
+            Total += ff;
+            Total += th;
+            Total += dd;
+            Total += kk;
+            Total += ch;
+            Total += ss;
+            Total += nn;
+            Total += rr;
+            Total += aa;
+            Total += e;
+            Total += ih;
+            Total += oh;
+            Total += ou;
+        }
+    }
+}

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Visemes/VisemeBoneEffectorList.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: df06673497ba4952ab788f87edbf7950
+timeCreated: 1722030678

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Visemes/VisemeEffectorsList.cs

@@ -0,0 +1,26 @@
+using System.Collections.Generic;
+using Convai.Scripts.Runtime.Features.LipSync.Models;
+using UnityEngine;
+
+namespace Convai.Scripts.Runtime.Features.LipSync.Visemes
+{
+    [CreateAssetMenu(fileName = "Convai Viseme Effectors", menuName = "Convai/Expression/Visemes Skin Effector", order = 0)]
+    public class VisemeEffectorsList : ScriptableObject
+    {
+        [SerializeField] public List<BlendShapesIndexEffector> sil;
+        [SerializeField] public List<BlendShapesIndexEffector> pp;
+        [SerializeField] public List<BlendShapesIndexEffector> ff;
+        [SerializeField] public List<BlendShapesIndexEffector> th;
+        [SerializeField] public List<BlendShapesIndexEffector> dd;
+        [SerializeField] public List<BlendShapesIndexEffector> kk;
+        [SerializeField] public List<BlendShapesIndexEffector> ch;
+        [SerializeField] public List<BlendShapesIndexEffector> ss;
+        [SerializeField] public List<BlendShapesIndexEffector> nn;
+        [SerializeField] public List<BlendShapesIndexEffector> rr;
+        [SerializeField] public List<BlendShapesIndexEffector> aa;
+        [SerializeField] public List<BlendShapesIndexEffector> e;
+        [SerializeField] public List<BlendShapesIndexEffector> ih;
+        [SerializeField] public List<BlendShapesIndexEffector> oh;
+        [SerializeField] public List<BlendShapesIndexEffector> ou;
+    }
+}

Unity/Assets/Convai/Scripts/Runtime/Features/LipSync/Visemes/VisemeEffectorsList.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 60d464fb2d5c4b959eb83e0010e7b3a8
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: