using System; using System.Text.RegularExpressions; using Convai.Scripts.Utils.LipSync.Types; using Service; using UnityEngine; namespace Convai.Scripts.Utils.LipSync { public class ConvaiLipSync : MonoBehaviour { public enum LipSyncBlendshapeType { None, // Default Value OVR, // Oculus ReallusionPlus, // Reallusion Extended ARKit, // AR Kit - Translated from Oculus } [Tooltip( "The type of facial blend-shapes in the character. Select OVR for Oculus and ReallusionPlus for Reallusion Extended visemes.")] public LipSyncBlendshapeType BlendshapeType = LipSyncBlendshapeType.OVR; [Tooltip("Skinned Mesh Renderer Component for the head of the character.")] public SkinnedMeshRenderer HeadSkinnedMeshRenderer; [Tooltip("Skinned Mesh Renderer Component for the teeth of the character, if available. Leave empty if not.")] public SkinnedMeshRenderer TeethSkinnedMeshRenderer; [Tooltip("Skinned Mesh Renderer Component for the tongue of the character, if available. Leave empty if not.")] public SkinnedMeshRenderer TongueSkinnedMeshRenderer; [Tooltip("Game object with the bone of the jaw for the character, if available. Leave empty if not.")] public GameObject jawBone; [Tooltip("Game object with the bone of the tongue for the character, if available. Leave empty if not.")] public GameObject tongueBone; // even though actually tongue doesn't have a bone [HideInInspector] public FaceModel faceModel = FaceModel.OvrModelName; [Tooltip("The index of the first blendshape that will be manipulated.")] public int firstIndex; [Tooltip("This will multiply the weights of the incoming frames to the lipsync")] [field: SerializeField] public float WeightMultiplier { get; private set; } = 1f; private ConvaiNPC _convaiNPC; public event Action OnCharacterLipSyncing; private ConvaiLipSyncApplicationBase convaiLipSyncApplicationBase; public ConvaiLipSyncApplicationBase ConvaiLipSyncApplicationBase { get => convaiLipSyncApplicationBase; private set => convaiLipSyncApplicationBase = value; } private void Awake() { switch (BlendshapeType) { case LipSyncBlendshapeType.None: break; case LipSyncBlendshapeType.OVR: ConvaiLipSyncApplicationBase = gameObject.GetOrAddComponent(); break; case LipSyncBlendshapeType.ReallusionPlus: ConvaiLipSyncApplicationBase = gameObject.GetOrAddComponent(); break; case LipSyncBlendshapeType.ARKit: ConvaiLipSyncApplicationBase = gameObject.GetOrAddComponent(); break; } } /// /// This function will automatically set any of the unassigned skinned mesh renderers /// to appropriate values using regex based functions. /// It also invokes the LipSyncCharacter() function every one hundredth of a second. /// private void Start() { // regex search for SkinnedMeshRenderers: head, teeth, tongue if (HeadSkinnedMeshRenderer == null) HeadSkinnedMeshRenderer = GetHeadSkinnedMeshRendererWithRegex(transform); if (TeethSkinnedMeshRenderer == null) TeethSkinnedMeshRenderer = GetTeethSkinnedMeshRendererWithRegex(transform); if (TongueSkinnedMeshRenderer == null) TongueSkinnedMeshRenderer = GetTongueSkinnedMeshRendererWithRegex(transform); _convaiNPC = GetComponent(); ConvaiLipSyncApplicationBase.Initialize(this, _convaiNPC); SetCharacterLipSyncing(true); } /// /// Fires an event with update the Character Lip Syncing State /// /// private void SetCharacterLipSyncing(bool value) { OnCharacterLipSyncing?.Invoke(value); } private void OnApplicationQuit() { StopLipSync(); } /// /// This function finds the Head skinned mesh renderer components, if present, /// in the children of the parentTransform using regex. /// /// The parent transform whose children are searched. /// The SkinnedMeshRenderer component of the Head, if found; otherwise, null. private SkinnedMeshRenderer GetHeadSkinnedMeshRendererWithRegex(Transform parentTransform) { // Initialize a variable to store the found SkinnedMeshRenderer. SkinnedMeshRenderer findFaceSkinnedMeshRenderer = null; // Define a regular expression pattern for matching child object names. Regex regexPattern = new("(.*_Head|CC_Base_Body)"); // Iterate through each child of the parentTransform. foreach (Transform child in parentTransform) // Check if the child's name matches the regex pattern. if (regexPattern.IsMatch(child.name)) { // If a match is found, get the SkinnedMeshRenderer component of the child. findFaceSkinnedMeshRenderer = child.GetComponent(); // If a SkinnedMeshRenderer is found, break out of the loop. if (findFaceSkinnedMeshRenderer != null) break; } // Return the found SkinnedMeshRenderer (or null if none is found). return findFaceSkinnedMeshRenderer; } /// /// This function finds the Teeth skinned mesh renderer components, if present, /// in the children of the parentTransform using regex. /// /// The parent transform whose children are searched. /// The SkinnedMeshRenderer component of the Teeth, if found; otherwise, null. private SkinnedMeshRenderer GetTeethSkinnedMeshRendererWithRegex(Transform parentTransform) { // Initialize a variable to store the found SkinnedMeshRenderer for teeth. SkinnedMeshRenderer findTeethSkinnedMeshRenderer = null; // Define a regular expression pattern for matching child object names. Regex regexPattern = new("(.*_Teeth|CC_Base_Body)"); // Iterate through each child of the parentTransform. foreach (Transform child in parentTransform) // Check if the child's name matches the regex pattern. if (regexPattern.IsMatch(child.name)) { // If a match is found, get the SkinnedMeshRenderer component of the child. findTeethSkinnedMeshRenderer = child.GetComponent(); // If a SkinnedMeshRenderer is found, break out of the loop. if (findTeethSkinnedMeshRenderer != null) break; } // Return the found SkinnedMeshRenderer for teeth (or null if none is found). return findTeethSkinnedMeshRenderer; } /// /// This function finds the Tongue skinned mesh renderer components, if present, /// in the children of the parentTransform using regex. /// /// The parent transform whose children are searched. /// The SkinnedMeshRenderer component of the Tongue, if found; otherwise, null. private SkinnedMeshRenderer GetTongueSkinnedMeshRendererWithRegex(Transform parentTransform) { // Initialize a variable to store the found SkinnedMeshRenderer for the tongue. SkinnedMeshRenderer findTongueSkinnedMeshRenderer = null; // Define a regular expression pattern for matching child object names. Regex regexPattern = new("(.*_Tongue|CC_Base_Body)"); // Iterate through each child of the parentTransform. foreach (Transform child in parentTransform) // Check if the child's name matches the regex pattern. if (regexPattern.IsMatch(child.name)) { // If a match is found, get the SkinnedMeshRenderer component of the child. findTongueSkinnedMeshRenderer = child.GetComponent(); // If a SkinnedMeshRenderer is found, break out of the loop. if (findTongueSkinnedMeshRenderer != null) break; } // Return the found SkinnedMeshRenderer for the tongue (or null if none is found). return findTongueSkinnedMeshRenderer; } /// /// Purges the latest chuck of lipsync frames /// public void PurgeExcessFrames() { ConvaiLipSyncApplicationBase?.PurgeExcessBlendShapeFrames(); } /// /// Stops the Lipsync by clearing the frames queue /// public void StopLipSync() { ConvaiLipSyncApplicationBase?.ClearQueue(); } } }