Initialer Upload neues Unity-Projekt

Assets/Convai/Scripts/Runtime/Features/Actions.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: f7ee91cfe70e2af439a2ed9b7557855d
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/Actions/ConvaiActionsHandler.cs

@@ -0,0 +1,748 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using Service;
+using UnityEngine;
+using UnityEngine.AI;
+using UnityEngine.Serialization;
+
+namespace Convai.Scripts.Utils
+{
+    // STEP 1: Add the enum for your custom action here. 
+    public enum ActionChoice
+    {
+        None,
+        Jump,
+        Crouch,
+        MoveTo,
+        PickUp,
+        Drop,
+        OpenWeb
+    }
+
+    /// <summary>
+    ///     DISCLAIMER: The action API is in experimental stages and can misbehave. Meanwhile, feel free to try it out and play
+    ///     around with it.
+    /// </summary>
+    [DisallowMultipleComponent]
+    [AddComponentMenu("Convai/Convai Actions Handler")]
+    [HelpURL(
+        "https://docs.convai.com/api-docs/plugins-and-integrations/unity-plugin/scripts-overview/convaiactionshandler.cs")]
+    public class ConvaiActionsHandler : MonoBehaviour
+    {
+        [SerializeField] public ActionMethod[] actionMethods;
+        public List<string> actionResponseList = new();
+        private readonly List<ConvaiAction> _actionList = new();
+        public readonly ActionConfig ActionConfig = new();
+        private List<string> _actions = new();
+        private ConvaiNPC _currentNPC;
+        private ConvaiInteractablesData _interactablesData;
+
+        // Awake is called when the script instance is being loaded
+        private void Awake()
+        {
+            // Find the global action settings object in the scene
+            _interactablesData = FindObjectOfType<ConvaiInteractablesData>();
+
+            // Check if the global action settings object is missing
+            if (_interactablesData == null)
+                // Log an error message to indicate missing Convai Action Settings
+                Logger.Error("Convai Action Settings missing. Please create a game object that handles actions.",
+                    Logger.LogCategory.Character);
+
+            // Check if this GameObject has a ConvaiNPC component attached
+            if (TryGetComponent(out ConvaiNPC npc))
+                // If it does, set the current NPC to this GameObject
+                _currentNPC = npc;
+
+
+            // Iterate through each action method and add its name to the action configuration
+            foreach (ActionMethod actionMethod in actionMethods) ActionConfig.Actions.Add(actionMethod.action);
+
+            if (_interactablesData != null)
+            {
+                // Iterate through each character in global action settings and add them to the action configuration
+                foreach (ConvaiInteractablesData.Character character in _interactablesData.Characters)
+                {
+                    ActionConfig.Types.Character rpcCharacter = new()
+                    {
+                        Name = character.Name,
+                        Bio = character.Bio
+                    };
+
+                    ActionConfig.Characters.Add(rpcCharacter);
+                }
+
+                // Iterate through each object in global action settings and add them to the action configuration
+                foreach (ConvaiInteractablesData.Object eachObject in _interactablesData.Objects)
+                {
+                    ActionConfig.Types.Object rpcObject = new()
+                    {
+                        Name = eachObject.Name,
+                        Description = eachObject.Description
+                    };
+                    ActionConfig.Objects.Add(rpcObject);
+                }
+            }
+        }
+
+        private void Reset()
+        {
+            actionMethods = new ActionMethod[]
+            {
+                new() { action = "Move To", actionChoice = ActionChoice.MoveTo },
+                new() { action = "Pick Up", actionChoice = ActionChoice.PickUp },
+                new() { action = "Dance", animationName = "Dance", actionChoice = ActionChoice.None },
+                new() { action = "Drop", actionChoice = ActionChoice.Drop },
+                new() { action = "Jump", actionChoice = ActionChoice.Jump }
+            };
+        }
+
+        // Start is called before the first frame update
+        private void Start()
+        {
+            // Set up the action configuration
+
+            #region Actions Setup
+
+            // Set the classification of the action configuration to "multistep"
+            ActionConfig.Classification = "multistep";
+
+            // Log the configured action information
+            Logger.DebugLog(ActionConfig, Logger.LogCategory.Actions);
+
+            #endregion
+
+            // Start playing the action list using a coroutine
+            StartCoroutine(PlayActionList());
+        }
+
+        private void Update()
+        {
+            if (actionResponseList.Count > 0)
+            {
+                ParseActions(actionResponseList[0]);
+                actionResponseList.RemoveAt(0);
+            }
+        }
+
+        public void ParseActions(string actionsString)
+        {
+            // Trim the input string to remove leading and trailing spaces
+            actionsString = actionsString.Trim();
+            Logger.DebugLog($"Parsing actions from: {actionsString}", Logger.LogCategory.Actions);
+
+            // Split the trimmed actions string into a list of individual actions
+            _actions = new List<string>(actionsString.Split(", "));
+
+            // Iterate through each action in the list of actions
+            foreach (List<string> actionWords in _actions.Select(t => new List<string>(t.Split(" "))))
+                // Iterate through the words in the current action
+            {
+                Logger.Info(
+                    $"Processing action: {string.Join(" ", actionWords)}",
+                    Logger.LogCategory.Actions); // Info: Checking each action being processed
+                for (int j = 0; j < actionWords.Count; j++)
+                {
+                    // Separate the words into two parts: verb and object
+                    string[] tempString1 = new string[j + 1];
+                    string[] tempString2 = new string[actionWords.Count - j - 1];
+
+                    Array.Copy(actionWords.ToArray(), tempString1, j + 1);
+                    Array.Copy(actionWords.ToArray(), j + 1, tempString2,
+                        0, actionWords.Count - j - 1);
+
+                    // Check if any verb word ends with "s" and remove it
+                    for (int k = 0; k < tempString1.Length; k++)
+                        if (tempString1[k].EndsWith("s"))
+                            tempString1[k] = tempString1[k].Remove(tempString1[k].Length - 1);
+
+                    // Iterate through each defined Convai action
+                    foreach (ActionMethod convaiAction in actionMethods)
+                        // Check if the parsed verb matches any defined action
+                        if (string.Equals(convaiAction.action, string.Join(" ", tempString1),
+                                StringComparison.CurrentCultureIgnoreCase))
+                        {
+                            GameObject tempGameObject = null;
+
+                            // Iterate through each object in global action settings to find a match
+                            foreach (ConvaiInteractablesData.Object @object in _interactablesData.Objects)
+                                if (string.Equals(@object.Name, string.Join(" ", tempString2),
+                                        StringComparison.CurrentCultureIgnoreCase))
+                                {
+                                    Logger.DebugLog($"Active Target: {string.Join(" ", tempString2).ToLower()}",
+                                        Logger.LogCategory.Actions);
+                                    tempGameObject = @object.gameObject;
+                                }
+
+                            // Iterate through each character in global action settings to find a match
+                            foreach (ConvaiInteractablesData.Character character in _interactablesData.Characters)
+                                if (string.Equals(character.Name, string.Join(" ", tempString2),
+                                        StringComparison.CurrentCultureIgnoreCase))
+                                {
+                                    Logger.DebugLog($"Active Target: {string.Join(" ", tempString2).ToLower()}",
+                                        Logger.LogCategory.Actions);
+                                    tempGameObject = character.gameObject;
+                                }
+
+                            if (tempGameObject != null)
+                                Logger.DebugLog(
+                                    $"Found matching target: {tempGameObject.name} for action: {string.Join(" ", tempString1).ToLower()}",
+                                    Logger.LogCategory.Actions); // DebugLog: For successful matching
+                            else
+                                Logger.Warn(
+                                    $"No matching target found for action: {string.Join(" ", tempString1).ToLower()}",
+                                    Logger.LogCategory.Actions); // Warning: When expected matches aren't found
+
+                            // Add the parsed action to the action list
+                            _actionList.Add(new ConvaiAction(convaiAction.actionChoice, tempGameObject,
+                                convaiAction.animationName));
+
+                            break; // Break the loop as the action is found
+                        }
+                }
+            }
+        }
+
+        /// <summary>
+        ///     Event that is triggered when an action starts.
+        /// </summary>
+        /// <remarks>
+        ///     This event can be subscribed to in order to perform custom logic when an action starts.
+        ///     The event provides the name of the action and the GameObject that the action is targeting.
+        /// </remarks>
+        public event Action<string, GameObject> ActionStarted;
+
+        /// <summary>
+        ///     Event that is triggered when an action ends.
+        /// </summary>
+        /// <remarks>
+        ///     This event can be subscribed to in order to perform custom logic when an action ends.
+        ///     The event provides the name of the action and the GameObject that the action was targeting.
+        /// </remarks>
+        public event Action<string, GameObject> ActionEnded;
+
+        /// <summary>
+        ///     This coroutine handles playing the actions in the action list.
+        /// </summary>
+        /// <returns></returns>
+        private IEnumerator PlayActionList()
+        {
+            while (true)
+                // Check if there are actions in the action list
+                if (_actionList.Count > 0)
+                {
+                    // Call the DoAction function for the first action in the list and wait until it's done
+                    yield return DoAction(_actionList[0]);
+
+                    // Remove the completed action from the list
+                    _actionList.RemoveAt(0);
+                }
+                else
+                {
+                    // If there are no actions in the list, yield to wait for the next frame
+                    yield return null;
+                }
+        }
+
+        private IEnumerator DoAction(ConvaiAction action)
+        {
+            // STEP 2: Add the function call for your action here corresponding to your enum.
+            //         Remember to yield until its return if it is a Enumerator function.
+
+            // Use a switch statement to handle different action choices based on the ActionChoice enum
+            switch (action.Verb)
+            {
+                case ActionChoice.MoveTo:
+                    // Call the MoveTo function and yield until it's completed
+                    yield return MoveTo(action.Target);
+                    break;
+
+                case ActionChoice.PickUp:
+                    // Call the PickUp function and yield until it's completed
+                    yield return PickUp(action.Target);
+                    break;
+
+                case ActionChoice.Drop:
+                    // Call the Drop function
+                    Drop(action.Target);
+                    break;
+
+                case ActionChoice.Jump:
+                    // Call the Jump function
+                    Jump();
+                    break;
+
+                case ActionChoice.Crouch:
+                    // Call the Crouch function and yield until it's completed
+                    yield return Crouch();
+                    break;
+
+                case ActionChoice.None:
+                    // Call the AnimationActions function and yield until it's completed
+                    yield return AnimationActions(action.Animation);
+                    break;
+
+                case ActionChoice.OpenWeb:
+                    OpenWebsite();
+                    break;
+            }
+
+            // Yield once to ensure the coroutine advances to the next frame
+            yield return null;
+        }
+
+        private void OpenWebsite()
+        {
+            // Specify the URL you want to open
+            string url = "https://www.convai.com";
+
+            // Use Application.OpenURL to open the web page
+            Application.OpenURL(url);
+        }
+
+        /// <summary>
+        ///     This method is a coroutine that handles playing an animation for Convai NPC.
+        ///     The method takes in the name of the animation to be played as a string parameter.
+        /// </summary>
+        /// <param name="animationName"> The name of the animation to be played. </param>
+        /// <returns> A coroutine that plays the animation. </returns>
+        private IEnumerator AnimationActions(string animationName)
+        {
+            // Logging the action of initiating the animation with the provided animation name.
+            Logger.DebugLog("Doing animation: " + animationName, Logger.LogCategory.Actions);
+
+            // Attempting to get the Animator component attached to the current NPC object.
+            // The Animator component is responsible for controlling animations on the GameObject.
+            Animator animator = _currentNPC.GetComponent<Animator>();
+
+            // Converting the provided animation name to its corresponding hash code.
+            // This is a more efficient way to refer to animations and Animator states.
+            int animationHash = Animator.StringToHash(animationName);
+
+            // Check if the Animator component has a state with the provided hash code.
+            // This is a safety check to prevent runtime errors if the animation is not found.
+            if (!animator.HasState(0, animationHash))
+            {
+                // Logging a message to indicate that the animation was not found.
+                Logger.DebugLog("Could not find an animator state named: " + animationName, Logger.LogCategory.Actions);
+
+                // Exiting the coroutine early since the animation is not available.
+                yield break;
+            }
+
+            // Playing the animation with a cross-fade transition.
+            // The second parameter '0.1f' specifies the duration of the cross-fade.
+            animator.CrossFadeInFixedTime(animationHash, 0.1f);
+
+            // Waiting for a short duration (just over the cross-fade time) to allow the animation transition to start.
+            // This ensures that subsequent code runs after the animation has started playing.
+            yield return new WaitForSeconds(0.11f);
+
+            // Getting information about the current animation clip that is playing.
+            AnimatorClipInfo[] clipInfo = animator.GetCurrentAnimatorClipInfo(0);
+
+            // Checking if there is no animation clip information available.
+            if (clipInfo == null || clipInfo.Length == 0)
+            {
+                // Logging a message to indicate that there are no animation clips associated with the state.
+                Logger.DebugLog("Animator state named: " + animationName + " has no associated animation clips",
+                    Logger.LogCategory.Actions);
+
+                // Exiting the coroutine as there is no animation to play.
+                yield break;
+            }
+
+            // Defining variables to store the length and name of the animation clip.
+            float length = 0;
+            string animationClipName = "";
+
+            // Iterating through the array of animation clips to find the one that is currently playing.
+            foreach (AnimatorClipInfo clipInf in clipInfo)
+            {
+                // Logging the name of the animation clip for debugging purposes.
+                Logger.DebugLog("Clip name: " + clipInf.clip.name, Logger.LogCategory.Actions);
+
+                // Storing the current animation clip in a local variable for easier access.
+                AnimationClip clip = clipInf.clip;
+
+                // Checking if the animation clip is valid.
+                if (clip != null)
+                {
+                    // Storing the length and name of the animation clip.
+                    length = clip.length;
+                    animationClipName = clip.name;
+
+                    // Exiting the loop as we've found the information we need.
+                    break;
+                }
+            }
+
+            // Checking if a valid animation clip was found.
+            if (length > 0.0f)
+            {
+                // Logging a message indicating that the animation is now playing.
+                Logger.DebugLog(
+                    "Playing the animation " + animationClipName + " from the Animator State " + animationName +
+                    " for " + length + " seconds", Logger.LogCategory.Actions);
+
+                // Waiting for the duration of the animation to allow it to play out.
+                yield return new WaitForSeconds(length);
+            }
+            else
+            {
+                // Logging a message to indicate that no valid animation clips were found or their length was zero.
+                Logger.DebugLog(
+                    "Animator state named: " + animationName +
+                    " has no valid animation clips or they have a length of 0", Logger.LogCategory.Actions);
+
+                // Exiting the coroutine early.
+                yield break;
+            }
+
+            // Transitioning back to the idle animation.
+            // It is assumed that an "Idle" animation exists and is set up in your Animator Controller.
+            animator.CrossFadeInFixedTime(Animator.StringToHash("Idle"), 0.1f);
+
+            // Yielding to wait for one frame to ensure that the coroutine progresses to the next frame.
+            // This is often done at the end of a coroutine to prevent issues with Unity's execution order.
+            yield return null;
+        }
+
+        /// <summary>
+        ///     Registers the provided methods to the ActionStarted and ActionEnded events.
+        ///     This allows external code to subscribe to these events and react when they are triggered.
+        /// </summary>
+        /// <param name="onActionStarted">
+        ///     The method to be called when an action starts. It should accept a string (the action
+        ///     name) and a GameObject (the target of the action).
+        /// </param>
+        /// <param name="onActionEnded">
+        ///     The method to be called when an action ends. It should accept a string (the action name)
+        ///     and a GameObject (the target of the action).
+        /// </param>
+        public void RegisterForActionEvents(Action<string, GameObject> onActionStarted,
+            Action<string, GameObject> onActionEnded)
+        {
+            ActionStarted += onActionStarted;
+            ActionEnded += onActionEnded;
+        }
+
+        /// <summary>
+        ///     Unregisters the provided methods from the ActionStarted and ActionEnded events.
+        ///     This allows external code to unsubscribe from these events when they are no longer interested in them.
+        /// </summary>
+        /// <param name="onActionStarted">
+        ///     The method to be removed from the ActionStarted event. It should be the same method that
+        ///     was previously registered.
+        /// </param>
+        /// <param name="onActionEnded">
+        ///     The method to be removed from the ActionEnded event. It should be the same method that was
+        ///     previously registered.
+        /// </param>
+        public void UnregisterForActionEvents(Action<string, GameObject> onActionStarted,
+            Action<string, GameObject> onActionEnded)
+        {
+            ActionStarted -= onActionStarted;
+            ActionEnded -= onActionEnded;
+        }
+
+        [Serializable]
+        public class ActionMethod
+        {
+            [FormerlySerializedAs("Action")] [SerializeField]
+            public string action;
+
+            // feels unnecessary 
+            // [SerializeField] public ActionType actionType;
+            [SerializeField] public string animationName;
+            [SerializeField] public ActionChoice actionChoice;
+        }
+
+        private class ConvaiAction
+        {
+            public ConvaiAction(ActionChoice verb, GameObject target, string animation)
+            {
+                Verb = verb;
+                Target = target;
+                Animation = animation;
+            }
+
+            #region 04. Public variables
+
+            public readonly string Animation;
+            public readonly GameObject Target;
+            public readonly ActionChoice Verb;
+
+            #endregion
+        }
+
+        // STEP 3: Add the function for your action here.
+
+        #region Action Implementation Methods
+
+        private IEnumerator Crouch()
+        {
+            ActionStarted?.Invoke("Crouch", _currentNPC.gameObject);
+            Logger.DebugLog("Crouching!", Logger.LogCategory.Actions);
+            Animator animator = _currentNPC.GetComponent<Animator>();
+            animator.CrossFadeInFixedTime(Animator.StringToHash("Crouch"), 0.1f);
+
+            // Wait for the next frame to ensure the Animator has transitioned to the new state.
+            yield return new WaitForSeconds(0.11f);
+
+            AnimatorClipInfo[] clipInfo = animator.GetCurrentAnimatorClipInfo(0);
+            if (clipInfo == null || clipInfo.Length == 0)
+            {
+                Logger.DebugLog("No animation clips found for crouch state!", Logger.LogCategory.Actions);
+                yield break;
+            }
+
+            float length = clipInfo[0].clip.length;
+
+            _currentNPC.GetComponents<CapsuleCollider>()[0].height = 1.2f;
+            _currentNPC.GetComponents<CapsuleCollider>()[0].center = new Vector3(0, 0.6f, 0);
+
+            if (_currentNPC.GetComponents<CapsuleCollider>().Length > 1)
+            {
+                _currentNPC.GetComponents<CapsuleCollider>()[1].height = 1.2f;
+                _currentNPC.GetComponents<CapsuleCollider>()[1].center = new Vector3(0, 0.6f, 0);
+            }
+
+            yield return new WaitForSeconds(length);
+            animator.CrossFadeInFixedTime(Animator.StringToHash("Idle"), 0.1f);
+
+            yield return null;
+            ActionEnded?.Invoke("Crouch", _currentNPC.gameObject);
+        }
+
+        /// <summary>
+        ///     Coroutine to move the NPC towards a specified target using NavMeshAgent.
+        /// </summary>
+        /// <param name="target">The target GameObject to move towards.</param>
+        private IEnumerator MoveTo(GameObject target)
+        {
+            // Invoke the ActionStarted event with the "MoveTo" action and the target GameObject.
+            ActionStarted?.Invoke("MoveTo", target);
+
+            // Check if the target is null or inactive, and log an error if so.
+            if (target == null || !target.activeInHierarchy)
+            {
+                // Log an error if the target is null or inactive.
+                Logger.DebugLog("MoveTo target is null or inactive.", Logger.LogCategory.Actions);
+                yield break; // Exit the coroutine.
+            }
+
+            // Log that the NPC is starting the movement towards the target.
+            Logger.DebugLog($"Moving to Target: {target.name}", Logger.LogCategory.Actions);
+
+            // Start the "Walking" animation.
+            Animator animator = _currentNPC.GetComponent<Animator>();
+
+            // Crossfade to the "Walking" animation state with a slight transition delay for smoother visual effect.
+            animator.CrossFade(Animator.StringToHash("Walking"), 0.01f);
+
+            // Disable root motion to prevent conflicts between AI navigation and the automatic application of root motion.
+            // This helps avoid unexpected behaviors and ensures smoother control during AI navigation.
+            animator.applyRootMotion = false;
+
+            // Get the NavMeshAgent component from the NPC.
+            NavMeshAgent navMeshAgent = _currentNPC.GetComponent<NavMeshAgent>();
+            navMeshAgent.updateRotation = false;
+
+            // Calculate the target destination, considering the target's forward direction or default offset.
+            Vector3 targetDestination = target.transform.position;
+
+            // Check if the target has a Renderer component.
+            if (target.TryGetComponent(out Renderer renderer))
+            {
+                // If yes, calculate the offset based on the size of the renderer along the z-axis.
+                float zOffset = renderer.bounds.size.z;
+                targetDestination += zOffset * target.transform.forward;
+            }
+            else
+            {
+                // If no Renderer component is found, use a default offset of half the target's forward direction.
+                // This is useful when the target object does not have a visual representation (Renderer).
+                Vector3 standardOffset = 0.5f * target.transform.forward;
+                targetDestination += standardOffset;
+            }
+
+            // Set the destination for the NavMeshAgent and yield for one frame.
+            navMeshAgent.SetDestination(targetDestination);
+            yield return null;
+
+            // Set the rotation speed for the character's rotation towards the movement direction.
+            float rotationSpeed = 5;
+
+            // Loop until the character is within the stopping distance to the target.
+            while (navMeshAgent.remainingDistance > navMeshAgent.stoppingDistance)
+            {
+                // Ensure the target is still active during the movement.
+                if (!target.activeInHierarchy)
+                {
+                    // Log and break if the target has been deactivated during the movement.
+                    Logger.DebugLog("Target deactivated during movement.", Logger.LogCategory.Actions);
+                    yield break;
+                }
+
+                // Check if the NPC is not moving, and yield until the next frame.
+                if (navMeshAgent.velocity.sqrMagnitude < Mathf.Epsilon) yield return null;
+
+                // Calculate the rotation towards the movement direction.
+                Quaternion rotation = Quaternion.LookRotation(navMeshAgent.velocity.normalized);
+                rotation.x = 0;
+                rotation.z = 0;
+
+                // Smoothly rotate the NPC towards the calculated rotation.
+                transform.rotation = Quaternion.Slerp(transform.rotation, rotation, rotationSpeed * Time.deltaTime);
+
+                // Yield until the next frame.
+                yield return null;
+            }
+
+            // Transition to the "Idle" animation once the NPC has reached the target.
+            animator.CrossFade(Animator.StringToHash("Idle"), 0.1f);
+
+            // If this is the only action in the queue, rotate the NPC to face the camera.
+            if (_actions.Count == 1)
+            {
+                Vector3 direction = (Camera.main.transform.position - transform.position).normalized;
+                Quaternion targetRotation = Quaternion.LookRotation(direction);
+                float elapsedTime = 0f;
+                float rotationTime = 2f;
+
+                // Smoothly rotate the NPC towards the camera direction.
+                while (elapsedTime < rotationTime)
+                {
+                    targetRotation.x = 0;
+                    targetRotation.z = 0;
+                    transform.rotation =
+                        Quaternion.Slerp(transform.rotation, targetRotation, elapsedTime / rotationTime);
+
+                    elapsedTime += Time.deltaTime;
+                    yield return null;
+                }
+            }
+
+            // Re-enabling root motion, which was disabled during AI navigation movement to avoid conflicts.
+            animator.applyRootMotion = true;
+
+            // Invoke the ActionEnded event with the "MoveTo" action and the target GameObject.
+            ActionEnded?.Invoke("MoveTo", target);
+        }
+
+        /// <summary>
+        ///     Coroutine to pick up a target GameObject, adjusting the NPC's rotation and playing animations.
+        /// </summary>
+        /// <param name="target">The target GameObject to pick up.</param>
+        private IEnumerator PickUp(GameObject target)
+        {
+            // Invoke the ActionStarted event with the "PickUp" action and the target GameObject.
+            ActionStarted?.Invoke("PickUp", target);
+
+            // Check if the target GameObject is null. If it is, log an error and exit the coroutine.
+            if (target == null)
+            {
+                Logger.DebugLog("Target is null! Exiting PickUp coroutine.", Logger.LogCategory.Actions);
+                yield break;
+            }
+
+            // Check if the target GameObject is active. If not, log an error and exit the coroutine.
+            if (!target.activeInHierarchy)
+            {
+                Logger.DebugLog($"Target: {target.name} is inactive! Exiting PickUp coroutine.",
+                    Logger.LogCategory.Actions);
+                yield break;
+            }
+
+            // Calculate the direction from the NPC to the target, ignoring the vertical (y) component.
+            Vector3 direction = (target.transform.position - transform.position).normalized;
+            direction.y = 0;
+
+            // Calculate the target rotation to face the target direction.
+            Quaternion targetRotation = Quaternion.LookRotation(direction);
+            float elapsedTime = 0f;
+            float rotationTime = 0.5f;
+
+            // Smoothly rotate the NPC towards the target direction over a specified time.
+            while (elapsedTime < rotationTime)
+            {
+                targetRotation.x = 0;
+                targetRotation.z = 0;
+                transform.rotation = Quaternion.Slerp(transform.rotation, targetRotation, elapsedTime / rotationTime);
+
+                elapsedTime += Time.deltaTime;
+                yield return null;
+            }
+
+            // Log the action of picking up the target along with its name.
+            Logger.DebugLog($"Picking up Target: {target.name}", Logger.LogCategory.Actions);
+
+            // Retrieve the Animator component from the current NPC.
+            Animator animator = _currentNPC.GetComponent<Animator>();
+
+            // Start the "Picking Up" animation with a cross-fade transition.
+            animator.CrossFade(Animator.StringToHash("Picking Up"), 0.1f);
+
+            // Wait for one second to ensure that the Animator has had time to transition to the "Picking Up" animation state.
+            yield return new WaitForSeconds(1);
+
+            // Define the time it takes for the hand to reach the object in the "Picking Up" animation.
+            // This is a specific point in time during the animation that we are interested in.
+            float timeToReachObject = 1f;
+
+            // Wait for the time it takes for the hand to reach the object.
+            yield return new WaitForSeconds(timeToReachObject);
+
+            // Check again if the target is still active before attempting to pick it up.
+            if (!target.activeInHierarchy)
+            {
+                Logger.DebugLog(
+                    $"Target: {target.name} became inactive during the pick up animation! Exiting PickUp coroutine.",
+                    Logger.LogCategory.Actions);
+                yield break;
+            }
+
+            // Once the hand has reached the object, set the target's parent to the NPC's transform,
+            // effectively "picking up" the object, and then deactivate the object.
+            target.transform.parent = gameObject.transform;
+            target.SetActive(false);
+
+            // Transition back to the "Idle" animation.
+            animator.CrossFade(Animator.StringToHash("Idle"), 0.4f);
+
+            // Invoke the ActionEnded event with the "PickUp" action and the target GameObject.
+            ActionEnded?.Invoke("PickUp", target);
+        }
+
+        private void Drop(GameObject target)
+        {
+            ActionStarted?.Invoke("Drop", target);
+
+            if (target == null) return;
+
+            Logger.DebugLog($"Dropping Target: {target.name}", Logger.LogCategory.Actions);
+            target.transform.parent = null;
+            target.SetActive(true);
+
+            ActionEnded?.Invoke("Drop", target);
+        }
+
+        private void Jump()
+        {
+            ActionStarted?.Invoke("Jump", _currentNPC.gameObject);
+
+            float jumpForce = 5f;
+            GetComponent<Rigidbody>().AddForce(new Vector3(0f, jumpForce, 0f), ForceMode.Impulse);
+            _currentNPC.GetComponent<Animator>().CrossFade(Animator.StringToHash("Dance"), 1);
+
+            ActionEnded?.Invoke("Jump", _currentNPC.gameObject);
+        }
+
+        // STEP 3: Add the function for your action here.
+
+        #endregion
+    }
+}

Assets/Convai/Scripts/Runtime/Features/Actions/ConvaiActionsHandler.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 744cc0ebc534aa44b8b0871ca72570b9
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/Actions/ConvaiInteractablesData.cs

@@ -0,0 +1,40 @@
+using System;
+using UnityEngine;
+
+namespace Convai.Scripts.Utils
+{
+    /// <summary>
+    ///     This script defines global actions and settings for Convai.
+    /// </summary>
+    [AddComponentMenu("Convai/Convai Interactables Data")]
+    [HelpURL("https://docs.convai.com/api-docs/plugins-and-integrations/unity-plugin/scripts-overview")]
+    public class ConvaiInteractablesData : MonoBehaviour
+    {
+        [Tooltip("Array of Characters in the environment")]
+        [SerializeField] public Character[] Characters;
+
+        [Tooltip("Array of Objects in the environment")]
+        [SerializeField] public Object[] Objects;
+
+        public Transform DynamicMoveTargetIndicator;
+
+        /// <summary>
+        ///     Represents a character in the environment.
+        /// </summary>
+        [Serializable]
+        public class Character
+        {
+            [SerializeField] public string Name;
+            [SerializeField] public string Bio;
+            [SerializeField] public GameObject gameObject;
+        }
+
+        [Serializable]
+        public class Object
+        {
+            [SerializeField] public string Name;
+            [SerializeField] public string Description;
+            [SerializeField] public GameObject gameObject;
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/Actions/ConvaiInteractablesData.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: b3b04a8e56d4a394585355d57021ba7a
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/LipSync.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 79e17fbbffadd7f47b7b345e0be9a750
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/LipSync/ConvaiLipSync.cs

@@ -0,0 +1,203 @@
+using System;
+using System.Text.RegularExpressions;
+using Convai.Scripts.Utils.LipSync.Types;
+using Service;
+using UnityEngine;
+
+namespace Convai.Scripts.Utils.LipSync
+{
+    public class ConvaiLipSync : MonoBehaviour
+    {
+        public enum LipSyncBlendshapeType
+        {
+            None, // Default Value
+            OVR, // Oculus 
+            ReallusionPlus, // Reallusion Extended
+            ARKit, // AR Kit - Translated from Oculus
+        }
+
+        [Tooltip(
+            "The type of facial blend-shapes in the character. Select OVR for Oculus and ReallusionPlus for Reallusion Extended visemes.")]
+        public LipSyncBlendshapeType BlendshapeType = LipSyncBlendshapeType.OVR;
+        [Tooltip("Skinned Mesh Renderer Component for the head of the character.")]
+        public SkinnedMeshRenderer HeadSkinnedMeshRenderer;
+        [Tooltip("Skinned Mesh Renderer Component for the teeth of the character, if available. Leave empty if not.")]
+        public SkinnedMeshRenderer TeethSkinnedMeshRenderer;
+        [Tooltip("Skinned Mesh Renderer Component for the tongue of the character, if available. Leave empty if not.")]
+        public SkinnedMeshRenderer TongueSkinnedMeshRenderer;
+        [Tooltip("Game object with the bone of the jaw for the character, if available. Leave empty if not.")]
+        public GameObject jawBone;
+        [Tooltip("Game object with the bone of the tongue for the character, if available. Leave empty if not.")]
+        public GameObject tongueBone; // even though actually tongue doesn't have a bone
+
+        [HideInInspector]
+        public FaceModel faceModel = FaceModel.OvrModelName;
+        [Tooltip("The index of the first blendshape that will be manipulated.")]
+        public int firstIndex;
+        [Tooltip("This will multiply the weights of the incoming frames to the lipsync")]
+        [field: SerializeField] public float WeightMultiplier { get; private set; } = 1f;
+
+        private ConvaiNPC _convaiNPC;
+        public event Action<bool> OnCharacterLipSyncing;
+        private ConvaiLipSyncApplicationBase convaiLipSyncApplicationBase;
+        public ConvaiLipSyncApplicationBase ConvaiLipSyncApplicationBase { get => convaiLipSyncApplicationBase; private set => convaiLipSyncApplicationBase = value; }
+
+        private void Awake()
+        {
+            switch (BlendshapeType)
+            {
+                case LipSyncBlendshapeType.None:
+                    break;
+                case LipSyncBlendshapeType.OVR:
+                    ConvaiLipSyncApplicationBase = gameObject.GetOrAddComponent<ConvaiOVRLipsync>();
+                    break;
+                case LipSyncBlendshapeType.ReallusionPlus:
+                    ConvaiLipSyncApplicationBase = gameObject.GetOrAddComponent<ConvaiReallusionLipSync>();
+                    break;
+                case LipSyncBlendshapeType.ARKit:
+                    ConvaiLipSyncApplicationBase = gameObject.GetOrAddComponent<ConvaiARKitLipSync>();
+                    break;
+            }
+        }
+
+        /// <summary>
+        ///     This function will automatically set any of the unassigned skinned mesh renderers
+        ///     to appropriate values using regex based functions.
+        ///     It also invokes the LipSyncCharacter() function every one hundredth of a second.
+        /// </summary>
+        private void Start()
+        {
+            // regex search for SkinnedMeshRenderers: head, teeth, tongue
+            if (HeadSkinnedMeshRenderer == null)
+                HeadSkinnedMeshRenderer = GetHeadSkinnedMeshRendererWithRegex(transform);
+            if (TeethSkinnedMeshRenderer == null)
+                TeethSkinnedMeshRenderer = GetTeethSkinnedMeshRendererWithRegex(transform);
+            if (TongueSkinnedMeshRenderer == null)
+                TongueSkinnedMeshRenderer = GetTongueSkinnedMeshRendererWithRegex(transform);
+
+            _convaiNPC = GetComponent<ConvaiNPC>();
+            ConvaiLipSyncApplicationBase.Initialize(this, _convaiNPC);
+            SetCharacterLipSyncing(true);
+        }
+        /// <summary>
+        /// Fires an event with update the Character Lip Syncing State
+        /// </summary>
+        /// <param name="value"></param>
+        private void SetCharacterLipSyncing(bool value)
+        {
+            OnCharacterLipSyncing?.Invoke(value);
+        }
+
+
+        private void OnApplicationQuit()
+        {
+            StopLipSync();
+        }
+
+        /// <summary>
+        ///     This function finds the Head skinned mesh renderer components, if present,
+        ///     in the children of the parentTransform using regex.
+        /// </summary>
+        /// <param name="parentTransform">The parent transform whose children are searched.</param>
+        /// <returns>The SkinnedMeshRenderer component of the Head, if found; otherwise, null.</returns>
+        private SkinnedMeshRenderer GetHeadSkinnedMeshRendererWithRegex(Transform parentTransform)
+        {
+            // Initialize a variable to store the found SkinnedMeshRenderer.
+            SkinnedMeshRenderer findFaceSkinnedMeshRenderer = null;
+
+            // Define a regular expression pattern for matching child object names.
+            Regex regexPattern = new("(.*_Head|CC_Base_Body)");
+
+            // Iterate through each child of the parentTransform.
+            foreach (Transform child in parentTransform)
+                // Check if the child's name matches the regex pattern.
+                if (regexPattern.IsMatch(child.name))
+                {
+                    // If a match is found, get the SkinnedMeshRenderer component of the child.
+                    findFaceSkinnedMeshRenderer = child.GetComponent<SkinnedMeshRenderer>();
+
+                    // If a SkinnedMeshRenderer is found, break out of the loop.
+                    if (findFaceSkinnedMeshRenderer != null) break;
+                }
+
+            // Return the found SkinnedMeshRenderer (or null if none is found).
+            return findFaceSkinnedMeshRenderer;
+        }
+
+
+        /// <summary>
+        ///     This function finds the Teeth skinned mesh renderer components, if present,
+        ///     in the children of the parentTransform using regex.
+        /// </summary>
+        /// <param name="parentTransform">The parent transform whose children are searched.</param>
+        /// <returns>The SkinnedMeshRenderer component of the Teeth, if found; otherwise, null.</returns>
+        private SkinnedMeshRenderer GetTeethSkinnedMeshRendererWithRegex(Transform parentTransform)
+        {
+            // Initialize a variable to store the found SkinnedMeshRenderer for teeth.
+            SkinnedMeshRenderer findTeethSkinnedMeshRenderer = null;
+
+            // Define a regular expression pattern for matching child object names.
+            Regex regexPattern = new("(.*_Teeth|CC_Base_Body)");
+
+            // Iterate through each child of the parentTransform.
+            foreach (Transform child in parentTransform)
+                // Check if the child's name matches the regex pattern.
+                if (regexPattern.IsMatch(child.name))
+                {
+                    // If a match is found, get the SkinnedMeshRenderer component of the child.
+                    findTeethSkinnedMeshRenderer = child.GetComponent<SkinnedMeshRenderer>();
+
+                    // If a SkinnedMeshRenderer is found, break out of the loop.
+                    if (findTeethSkinnedMeshRenderer != null) break;
+                }
+
+            // Return the found SkinnedMeshRenderer for teeth (or null if none is found).
+            return findTeethSkinnedMeshRenderer;
+        }
+
+
+        /// <summary>
+        ///     This function finds the Tongue skinned mesh renderer components, if present,
+        ///     in the children of the parentTransform using regex.
+        /// </summary>
+        /// <param name="parentTransform">The parent transform whose children are searched.</param>
+        /// <returns>The SkinnedMeshRenderer component of the Tongue, if found; otherwise, null.</returns>
+        private SkinnedMeshRenderer GetTongueSkinnedMeshRendererWithRegex(Transform parentTransform)
+        {
+            // Initialize a variable to store the found SkinnedMeshRenderer for the tongue.
+            SkinnedMeshRenderer findTongueSkinnedMeshRenderer = null;
+
+            // Define a regular expression pattern for matching child object names.
+            Regex regexPattern = new("(.*_Tongue|CC_Base_Body)");
+
+            // Iterate through each child of the parentTransform.
+            foreach (Transform child in parentTransform)
+                // Check if the child's name matches the regex pattern.
+                if (regexPattern.IsMatch(child.name))
+                {
+                    // If a match is found, get the SkinnedMeshRenderer component of the child.
+                    findTongueSkinnedMeshRenderer = child.GetComponent<SkinnedMeshRenderer>();
+
+                    // If a SkinnedMeshRenderer is found, break out of the loop.
+                    if (findTongueSkinnedMeshRenderer != null) break;
+                }
+
+            // Return the found SkinnedMeshRenderer for the tongue (or null if none is found).
+            return findTongueSkinnedMeshRenderer;
+        }
+        /// <summary>
+        /// Purges the latest chuck of lipsync frames
+        /// </summary>
+        public void PurgeExcessFrames()
+        {
+            ConvaiLipSyncApplicationBase?.PurgeExcessBlendShapeFrames();
+        }
+        /// <summary>
+        /// Stops the Lipsync by clearing the frames queue
+        /// </summary>
+        public void StopLipSync()
+        {
+            ConvaiLipSyncApplicationBase?.ClearQueue();
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/LipSync/ConvaiLipSync.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 88bce56f6985ef84f8835a0152628fa1
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/LipSync/ConvaiLipSyncApplicationBase.cs

@@ -0,0 +1,159 @@
+using System;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+using Service;
+using UnityEngine;
+
+
+namespace Convai.Scripts.Utils.LipSync
+{
+
+    /// <summary>
+    /// This Class will serve as a base for any method of Lipsync that Convai will develop or use
+    /// </summary>
+    public abstract class ConvaiLipSyncApplicationBase : MonoBehaviour
+    {
+        /// <summary>
+        /// This stores a dictionary of blendshape name and index of the Blendweight it will affect
+        /// </summary>
+        protected Dictionary<string, int> _headMapping;
+        /// <summary>
+        /// Reference to the Head Skin Mesh Renderer used for lipsync
+        /// </summary>
+        protected SkinnedMeshRenderer _headSkinMeshRenderer;
+        /// <summary>
+        /// Reference to the Teeth Skin Mesh Renderer used for lipsync
+        /// </summary>
+        protected SkinnedMeshRenderer _teethSkinMeshRenderer;
+        /// <summary>
+        /// Reference to the Jaw bone gameobject used for lipsync
+        /// </summary>
+        private GameObject _jawBone;
+        /// <summary>
+        /// Reference to the Tongue bone gameobject used for lipsync
+        /// </summary>
+        private GameObject _tongueBone;
+        /// <summary>
+        /// Reference to the NPC on which lipsync will be applied
+        /// </summary>
+        protected ConvaiNPC _convaiNPC;
+        protected float _weightMultiplier { get; private set; }
+        #region Null States of References
+        protected bool HasHeadSkinnedMeshRenderer { get; private set; }
+        protected bool HasTeethSkinnedMeshRenderer { get; private set; }
+        protected bool HasJawBone { get; private set; }
+        protected bool HasTongueBone { get; private set; }
+        #endregion
+        /// <summary>
+        /// Initializes and setup up of the things necessary for lipsync to work
+        /// </summary>
+        /// <param name="convaiLipSync"></param>
+        /// <param name="convaiNPC"></param>
+        public virtual void Initialize(ConvaiLipSync convaiLipSync, ConvaiNPC convaiNPC)
+        {
+            _headSkinMeshRenderer = convaiLipSync.HeadSkinnedMeshRenderer;
+            HasHeadSkinnedMeshRenderer = _headSkinMeshRenderer != null;
+
+            _teethSkinMeshRenderer = convaiLipSync.TeethSkinnedMeshRenderer;
+            HasTeethSkinnedMeshRenderer = _teethSkinMeshRenderer != null;
+
+            _jawBone = convaiLipSync.jawBone;
+            HasJawBone = _jawBone != null;
+
+            _tongueBone = convaiLipSync.tongueBone;
+            HasTongueBone = _tongueBone != null;
+
+            _convaiNPC = convaiNPC;
+            _weightMultiplier = convaiLipSync != null ? convaiLipSync.WeightMultiplier : 1;
+
+            if (HasHeadSkinnedMeshRenderer)
+                _headMapping = SetupMapping(GetHeadRegexMapping, _headSkinMeshRenderer);
+        }
+        /// <summary>
+        /// Creates the mapping of blendshape and index it affects during lipsync
+        /// </summary>
+        protected Dictionary<string, int> SetupMapping(Func<Dictionary<string, string>> finder, SkinnedMeshRenderer skinnedMeshRenderer)
+        {
+            Dictionary<string, int> mapping = new Dictionary<string, int>();
+            Dictionary<string, string> regexMapping = finder();
+
+            foreach (KeyValuePair<string, string> pair in regexMapping)
+            {
+                for (int i = 0; i < skinnedMeshRenderer.sharedMesh.blendShapeCount; i++)
+                {
+                    string blendShapeName = skinnedMeshRenderer.sharedMesh.GetBlendShapeName(i);
+                    Regex regex = new(pair.Value);
+                    if (regex.IsMatch(blendShapeName))
+                    {
+                        mapping.TryAdd(pair.Key, i);
+                    }
+                }
+            }
+            return mapping;
+        }
+        /// <summary>
+        /// Returns a dictionary of blendshape name and regex string used to find the index
+        /// TODO Modify the override to fit your version of the mapping
+        /// </summary>
+        /// <returns></returns>
+        protected virtual Dictionary<string, string> GetHeadRegexMapping()
+        {
+            return new Dictionary<string, string>();
+        }
+        /// <summary>
+        /// Updates the tongue bone rotation to the new rotation
+        /// </summary>
+        /// <param name="newRotation"></param>
+        protected void UpdateTongueBoneRotation(Vector3 newRotation)
+        {
+            if (!HasTongueBone) return;
+            _tongueBone.transform.localEulerAngles = newRotation;
+        }
+        /// <summary>
+        /// Updates the jaw bone rotation to the new rotation
+        /// </summary>
+        /// <param name="newRotation"></param>
+        protected void UpdateJawBoneRotation(Vector3 newRotation)
+        {
+            if (!HasJawBone) return;
+            _jawBone.transform.localEulerAngles = newRotation;
+        }
+        /// <summary>
+        /// Updates the current blendshape or visemes frame
+        /// </summary>
+        protected abstract void UpdateBlendShape();
+        /// <summary>
+        /// This removes the excess frames in the queue
+        /// </summary>
+        public abstract void PurgeExcessBlendShapeFrames();
+        /// <summary>
+        /// This resets the whole queue of the frames
+        /// </summary>
+        protected bool CanPurge<T>(Queue<T> queue)
+        {
+            // ? Should I hardcode the limiter for this check
+            return queue.Count < 10;
+        }
+        public abstract void ClearQueue();
+        /// <summary>
+        /// Adds blendshape frames in the queue
+        /// </summary>
+        /// <param name="blendshapeFrames"></param>
+        public virtual void EnqueueQueue(Queue<BlendshapeFrame> blendshapeFrames) { }
+        /// <summary>
+        /// Adds Visemes frames in the list
+        /// </summary>
+        /// <param name="visemesFrames"></param>
+        public virtual void EnqueueQueue(Queue<VisemesData> visemesFrames) { }
+        /// <summary>
+        /// Adds a blendshape frame in the last queue
+        /// </summary>
+        /// <param name="blendshapeFrame"></param>
+        public virtual void EnqueueFrame(BlendshapeFrame blendshapeFrame) { }
+        /// <summary>
+        /// Adds a viseme frame to the last element of the list
+        /// </summary>
+        /// <param name="viseme"></param>
+        public virtual void EnqueueFrame(VisemesData viseme) { }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/LipSync/ConvaiLipSyncApplicationBase.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: bb231034f5b2dee4494498fe9117bda1
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/LipSync/LipSyncBlendFrameData.cs

@@ -0,0 +1,96 @@
+using System.Collections.Generic;
+using Service;
+using UnityEngine;
+
+namespace Convai.Scripts.Utils.LipSync
+{
+    public class LipSyncBlendFrameData
+    {
+        public enum FrameType
+        {
+            Visemes,
+            Blendshape
+        }
+        private readonly int _totalFrames;
+        private readonly Queue<BlendshapeFrame> _blendShapeFrames = new Queue<BlendshapeFrame>();
+        private readonly Queue<VisemesData> _visemesFrames = new Queue<VisemesData>();
+        private readonly GetResponseResponse _getResponseResponse;
+        private readonly FrameType _frameType;
+
+        private int _framesCaptured;
+        private bool _partiallyProcessed;
+
+        public LipSyncBlendFrameData(int totalFrames, GetResponseResponse response, FrameType frameType)
+        {
+            _totalFrames = totalFrames;
+            _framesCaptured = 0;
+            _getResponseResponse = response;
+            _frameType = frameType;
+            //Logger.DebugLog($"Total Frames: {_totalFrames} | {response.AudioResponse.TextData}", Logger.LogCategory.LipSync);
+        }
+
+        public void Enqueue(BlendshapeFrame blendShapeFrame)
+        {
+            _blendShapeFrames.Enqueue(blendShapeFrame);
+            _framesCaptured++;
+        }
+
+        public void Enqueue(VisemesData visemesData)
+        {
+            _visemesFrames.Enqueue(visemesData);
+        }
+
+        public void Process(ConvaiNPC npc)
+        {
+            if (!_partiallyProcessed)
+                npc.EnqueueResponse(_getResponseResponse);
+            switch (_frameType)
+            {
+                case FrameType.Visemes:
+                    npc.convaiLipSync.ConvaiLipSyncApplicationBase.EnqueueQueue(new Queue<VisemesData>(_visemesFrames));
+                    break;
+                case FrameType.Blendshape:
+                    npc.convaiLipSync.ConvaiLipSyncApplicationBase.EnqueueQueue(new Queue<BlendshapeFrame>(_blendShapeFrames));
+                    break;
+            }
+
+            npc.AudioManager.SetWaitForCharacterLipSync(false);
+        }
+
+        public void ProcessPartially(ConvaiNPC npc)
+        {
+            if (!_partiallyProcessed)
+            {
+                _partiallyProcessed = true;
+                npc.EnqueueResponse(_getResponseResponse);
+                npc.AudioManager.SetWaitForCharacterLipSync(false);
+            }
+
+            switch (_frameType)
+            {
+                case FrameType.Visemes:
+                    while (_visemesFrames.Count != 0)
+                    {
+                        npc.convaiLipSync.ConvaiLipSyncApplicationBase.EnqueueFrame(_visemesFrames.Dequeue());
+                    }
+                    break;
+                case FrameType.Blendshape:
+                    while (_blendShapeFrames.Count != 0)
+                    {
+                        npc.convaiLipSync.ConvaiLipSyncApplicationBase.EnqueueFrame(_blendShapeFrames.Dequeue());
+                    }
+                    break;
+            }
+        }
+
+        public bool CanPartiallyProcess()
+        {
+            return _framesCaptured > Mathf.Min(21, _totalFrames * 0.7f);
+        }
+
+        public bool CanProcess()
+        {
+            return _framesCaptured == _totalFrames;
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/LipSync/LipSyncBlendFrameData.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 31db1a9457d64f3d936ff7f5aabfb193
+timeCreated: 1708491067

Assets/Convai/Scripts/Runtime/Features/LipSync/Types.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 6158a9323c720f5408c5b7caa77405cc
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiARKitLipSync.cs

@@ -0,0 +1,112 @@
+using System.Collections.Generic;
+using System.Reflection;
+using Service;
+
+
+namespace Convai.Scripts.Utils.LipSync.Types
+{
+    public class ConvaiARKitLipSync : ConvaiVisemesLipSync
+    {
+        private Dictionary<string, int> _teethMapping;
+        public override void Initialize(ConvaiLipSync convaiLipSync, ConvaiNPC convaiNPC)
+        {
+            base.Initialize(convaiLipSync, convaiNPC);
+            if (HasTeethSkinnedMeshRenderer)
+                _teethMapping = SetupMapping(GetTeethRegexMapping, _teethSkinMeshRenderer);
+        }
+        private Dictionary<string, string> GetTeethRegexMapping()
+        {
+            string prefix = "(?:[A-Z]\\d{1,2}_)?";
+            string spacer = "[\\s_]*";
+            string open = "[Oo]pen";
+            return new Dictionary<string, string>()
+            {
+                {"KK", $"{prefix}[Jj]aw{spacer}{open}"},
+                {"AA", $"{prefix}[Jj]aw{spacer}[Ff]orward"}
+            };
+        }
+        protected override Dictionary<string, string> GetHeadRegexMapping()
+        {
+            string mouth = "[Mm]outh";
+            string spacer = "[\\s_]*";
+            string left = "[Ll]eft";
+            string right = "[Rr]ight";
+            string lower = "[Ll]ower";
+            string upper = "[Uu]pper";
+            string open = "[Oo]pen";
+            string funnel = "[Ff]unnel";
+            string pucker = "[Pp]ucker";
+            string prefix = "(?:[A-Z]\\d{1,2}_)?";
+
+            return new Dictionary<string, string>()
+            {
+                {"PP", $"{prefix}{mouth}{spacer}{pucker}"},
+                {"FF", $"{prefix}{mouth}{spacer}{funnel}"},
+                {"THL", $"{prefix}{mouth}{spacer}{lower}{spacer}[Dd]own{spacer}{left}"},
+                {"THR", $"{prefix}{mouth}{spacer}{lower}{spacer}[Dd]own{spacer}{right}"},
+                {"DDL", $"{prefix}{mouth}{spacer}[Pp]ress{spacer}{left}"},
+                {"DDR", $"{prefix}{mouth}{spacer}[Pp]ress{spacer}{right}"},
+                {"KK", $"{prefix}[Jj]aw{spacer}{open}"},
+                {"CHL",$"{prefix}{mouth}{spacer}[Ss]tretch{spacer}{left}"},
+                {"CHR",$"{prefix}{mouth}{spacer}[Ss]tretch{spacer}{right}"},
+                {"SSL", $"{prefix}{mouth}{spacer}[Ss]mile{spacer}{left}"},
+                {"SSR", $"{prefix}{mouth}{spacer}[Ss]mile{spacer}{right}"},
+                {"NNL", $"{prefix}[Nn]ose{spacer}[Ss]neer{spacer}{left}"},
+                {"NNR", $"{prefix}[Nn]ose{spacer}[Ss]neer{spacer}{right}"},
+                {"RRU",$"{prefix}{mouth}{spacer}[Rr]oll{spacer}{upper}"},
+                {"RRL", $"{prefix}{mouth}{spacer}[Rr]oll{spacer}{lower}"},
+                {"AA", $"{prefix}[Jj]aw{spacer}{open}"},
+                {"EL", $"{prefix}{mouth}{spacer}{upper}{spacer}[Uu]p{spacer}{left}"},
+                {"ER", $"{prefix}{mouth}{spacer}{upper}{spacer}[Uu]p{spacer}{right}"},
+                {"IHL", $"{prefix}{mouth}{spacer}[Ff]rown{spacer}{left}"},
+                {"IHR",$"{prefix}{mouth}{spacer}[Ff]rown{spacer}{right}"},
+                {"OU", $"{prefix}{mouth}{spacer}{pucker}"},
+                {"OH", $"{prefix}{mouth}{spacer}{funnel}"},
+            };
+        }
+        private void Update()
+        {
+            // Check if the dequeued frame is not null.
+            if (_currentViseme == null) return;
+            // Check if the frame represents silence (-2 is a placeholder for silence).
+            if (_currentViseme.Sil == -2) return;
+
+            float weight;
+            List<int> knownHeadIndexs = new List<int>();
+            List<int> knownTeethIndexs = new List<int>();
+            foreach (PropertyInfo propertyInfo in typeof(Viseme).GetProperties())
+            {
+                if (propertyInfo.PropertyType != typeof(float)) continue;
+                string fieldName = propertyInfo.Name.ToUpper();
+                float value = (float)propertyInfo.GetValue(_currentViseme);
+                weight = fieldName switch
+                {
+                    "KK" => 1.0f / 1.5f,
+                    "DD" => 1.0f / 0.7f,
+                    "CH" => 1.0f / 2.7f,
+                    "SS" => 1.0f / 1.5f,
+                    "NN" => 1.0f / 2.0f,
+                    "RR" => 1.0f / 0.9f,
+                    "AA" => 1.0f / 2.0f,
+                    "II" => 1.0f / 1.2f,
+                    "OH" => 1.2f,
+                    _ => 1.0f
+                };
+
+                foreach (string s in _possibleCombinations)
+                {
+                    float weightThisFrame = value * weight * _weightMultiplier;
+                    string modifiedFieldName = fieldName + s;
+                    if (HasHeadSkinnedMeshRenderer)
+                    {
+                        FindAndUpdateBlendWeight(_headSkinMeshRenderer, modifiedFieldName, weightThisFrame, knownHeadIndexs, _headMapping);
+                    }
+                    if (HasTeethSkinnedMeshRenderer)
+                    {
+                        FindAndUpdateBlendWeight(_teethSkinMeshRenderer, modifiedFieldName, weightThisFrame, knownTeethIndexs, _teethMapping);
+                    }
+                }
+            }
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiARKitLipSync.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: a78929d391d407d46ab3288e15a0700d
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiBlendShapeLipSync.cs

@@ -0,0 +1,202 @@
+using System.Collections.Generic;
+using System.Reflection;
+using Service;
+using UnityEngine;
+
+/*
+! This class is a Work in progress and can produce un expected results, Convai does not advise to use this class in production, please use this with extreme caution
+*/
+
+namespace Convai.Scripts.Utils.LipSync.Types
+{
+    public class ConvaiBlendShapeLipSync : ConvaiLipSyncApplicationBase
+    {
+        private const float A2XFRAMERATE = 1f / 30f;
+        private Queue<Queue<BlendshapeFrame>> _blendShapesQueue = new();
+        private ARKitBlendShapes _currentBlendshape;
+
+        protected override Dictionary<string, string> GetHeadRegexMapping()
+        {
+            #region Regex Finders
+            string prefix = "(?:[A-Z]\\d{1,2}_)?";
+            string spacer = "[\\s_]*";
+            string mouth = "[Mm]outh";
+            string nose = "[Nn]ose";
+            string left = "[Ll]eft";
+            string right = "[Rr]ight";
+            string up = "[Uu]p";
+            string down = "[Dd]own";
+            string lower = "[Ll]ower";
+            string upper = "[Uu]pper";
+            string open = "[Oo]pen";
+            string funnel = "[Ff]unnel";
+            string pucker = "[Pp]ucker";
+            string sneer = "[Ss]neer";
+            string cheek = "[Cc]heek";
+            string squint = "[Ss]quint";
+            string brow = "[Bb]row";
+            string outer = "[Oo]uter";
+            string inner = "[Ii]nner";
+            string eye = "[Ee]ye";
+            string blink = "[Bb]link";
+            string look = "[Ll]ook";
+            string In = "[Ii]n";
+            string Out = "[Oo]ut";
+            string wide = "[Ww]ide";
+            string forward = "[Ff]orward";
+            string jaw = "[Jj]aw";
+            string close = "[Cc]lose";
+            string smile = "[Ss]mile";
+            string frown = "[Ff]rown";
+            string dimple = "[Dd]imple";
+            string stretch = "[Ss]tretch";
+            string roll = "[Rr]oll";
+            string shrug = "[Ss]hrug";
+            string press = "[Pp]ress";
+            #endregion
+
+            return new Dictionary<string, string>(){
+                {"TougueOut", $"{prefix}[Tt]ougue{spacer}[Oo]ut"},
+
+                {"NoseSneerRight", $"{prefix}{nose}{spacer}{sneer}{spacer}{right}"},
+                {"NoseSneerLeft", $"{prefix}{nose}{spacer}{sneer}{spacer}{left}"},
+
+                {"CheekSquintRight", $"{prefix}{cheek}{spacer}{squint}{spacer}{right}"},
+                {"CheekSquintLeft", $"{prefix}{cheek}{spacer}{squint}{spacer}{left}"},
+                {"CheekPuff", $"{prefix}{cheek}{spacer}[Pp]uff"},
+
+                {"BrowDownLeft", $"{prefix}{brow}{spacer}{down}{spacer}{left}"},
+                {"BrowDownRight", $"{prefix}{brow}{spacer}{down}{spacer}{right}"},
+                {"BrowInnerUp", $"{prefix}{brow}{spacer}{inner}{spacer}{up}"},
+                {"BrowOuterUpLeft", $"{prefix}{brow}{spacer}{outer}{spacer}{up}{spacer}{left}"},
+                {"BrowOuterUpRight", $"{prefix}{brow}{spacer}{outer}{spacer}{up}{spacer}{right}"},
+
+                {"EyeBlinkLeft", $"{prefix}{eye}{spacer}{blink}{spacer}{left}"},
+                {"EyeLookDownLeft",$"{prefix}{eye}{spacer}{look}{spacer}{In}{left}"},
+                {"EyeLookInLeft", $"{prefix}{eye}{spacer}{look}{spacer}{In}{spacer}{left}"},
+                {"EyeLookOutLeft", $"{prefix}{eye}{spacer}{look}{spacer}{Out}{spacer}{left}"},
+                {"EyeLookUpLeft", $"{prefix}{eye}{spacer}{look}{spacer}{up}{spacer}{left}"},
+                {"EyeSquintLeft", $"{prefix}{eye}{spacer}{squint}{spacer}{left}"},
+                {"EyeWideLeft", $"{prefix}{eye}{spacer}{wide}{spacer}{left}"},
+
+                {"EyeBlinkRight", $"{prefix}{eye}{spacer}{blink}{spacer}{right}"},
+                {"EyeLookDownRight",$"{prefix}{eye}{spacer}{look}{spacer}{In}{right}"},
+                {"EyeLookInRight", $"{prefix}{eye}{spacer}{look}{spacer}{In}{spacer}{right}"},
+                {"EyeLookOutRight", $"{prefix}{eye}{spacer}{look}{spacer}{Out}{spacer}{right}"},
+                {"EyeLookUpRight", $"{prefix}{eye}{spacer}{look}{spacer}{up}{spacer}{right}"},
+                {"EyeSquintRight", $"{prefix}{eye}{spacer}{squint}{spacer}{right}"},
+                {"EyeWideRight", $"{prefix}{eye}{spacer}{wide}{spacer}{right}"},
+
+                {"JawForward", $"{prefix}{jaw}{spacer}{forward}"},
+                {"JawLeft", $"{prefix}{jaw}{spacer}{left}"},
+                {"JawRight", $"{prefix}{jaw}{spacer}{right}"},
+                {"JawOpen", $"{prefix}{jaw}{spacer}{open}"},
+
+                {"MouthClose", $"{prefix}{mouth}{spacer}{close}"},
+                {"MouthFunnel", $"{prefix}{mouth}{spacer}{funnel}"},
+                {"MouthPucker", $"{prefix}{mouth}{spacer}{pucker}"},
+
+                {"Mouthleft", $"{prefix}{mouth}{spacer}{left}"},
+                {"MouthRight", $"{prefix}{mouth}{spacer}{right}"},
+
+                {"MouthSmileLeft", $"{prefix}{mouth}{spacer}{smile}{spacer}{left}"},
+                {"MouthSmileRight", $"{prefix}{mouth}{spacer}{smile}{spacer}{right}"},
+
+                {"MouthFrownLeft", $"{prefix}{mouth}{spacer}{frown}{spacer}{left}"},
+                {"MouthFrownRight", $"{prefix}{mouth}{spacer}{frown}{spacer}{right}"},
+
+                {"MouthDimpleLeft", $"{prefix}{mouth}{spacer}{dimple}{spacer}{left}"},
+                {"MouthDimpleRight", $"{prefix}{mouth}{spacer}{dimple}{spacer}{right}"},
+
+                {"MouthStretchLeft", $"{prefix}{mouth}{spacer}{stretch}{spacer}{left}"},
+                {"MouthStretchRight", $"{prefix}{mouth}{spacer}{stretch}{spacer}{right}"},
+
+                {"MouthRollLower", $"{prefix}{mouth}{spacer}{roll}{spacer}{lower}"},
+                {"MouthRollUpper", $"{prefix}{mouth}{spacer}{roll}{spacer}{upper}"},
+
+                {"MouthShrugLower", $"{prefix}{mouth}{spacer}{shrug}{spacer}{lower}"},
+                {"MouthShrugUpper", $"{prefix}{mouth}{spacer}{shrug}{spacer}{upper}"},
+
+                {"MouthPressLeft", $"{prefix}{mouth}{spacer}{press}{spacer}{left}"},
+                {"MouthPressRight", $"{prefix}{mouth}{spacer}{press}{spacer}{right}"},
+
+                {"MouthLowerDownLeft", $"{prefix}{mouth}{spacer}{lower}{spacer}{down}{spacer}{left}"},
+                {"MouthLowerDownRight", $"{prefix}{mouth}{spacer}{lower}{spacer}{down}{spacer}{right}"},
+
+                {"MouthUpperUpLeft", $"{prefix}{mouth}{spacer}{upper}{spacer}{up}{spacer}{left}"},
+                {"MouthUpperUpRight", $"{prefix}{mouth}{spacer}{upper}{spacer}{up}{spacer}{right}"},
+            };
+        }
+
+        public override void Initialize(ConvaiLipSync convaiLipSync, ConvaiNPC convaiNPC)
+        {
+            base.Initialize(convaiLipSync, convaiNPC);
+            InvokeRepeating(nameof(UpdateBlendShape), 0, A2XFRAMERATE);
+        }
+
+        protected override void UpdateBlendShape()
+        {
+            if (_blendShapesQueue == null || _blendShapesQueue.Count <= 0)
+            {
+                _currentBlendshape = new ARKitBlendShapes();
+                return;
+            }
+            if (_blendShapesQueue.Peek().Count == 0)
+            {
+                _blendShapesQueue.Dequeue();
+                return;
+            }
+            if (!_convaiNPC.IsCharacterTalking) return;
+            _currentBlendshape = _blendShapesQueue.Peek().Dequeue().Blendshapes;
+        }
+        private void Update()
+        {
+            if (_currentBlendshape == null) return;
+            UpdateJawBoneRotation(new Vector3(0.0f, 0.0f, -90.0f - _currentBlendshape.JawOpen * 30f));
+            UpdateTongueBoneRotation(new Vector3(0.0f, 0.0f, -5.0f * _currentBlendshape.TongueOut));
+            if (!HasHeadSkinnedMeshRenderer) return;
+            foreach (PropertyInfo propertyInfo in typeof(ARKitBlendShapes).GetProperties())
+            {
+                if (propertyInfo.PropertyType != typeof(float)) continue;
+                string fieldName = propertyInfo.Name;
+                float value = (float)propertyInfo.GetValue(_currentBlendshape);
+                if (_headMapping.TryGetValue(fieldName, out int index))
+                {
+                    _headSkinMeshRenderer.SetBlendShapeWeightInterpolate(
+                        index,
+                        value * _weightMultiplier,
+                        Time.deltaTime
+                    );
+                }
+            }
+        }
+
+        public override void PurgeExcessBlendShapeFrames()
+        {
+            if (_blendShapesQueue.Count <= 0) return;
+            if (!CanPurge<BlendshapeFrame>(_blendShapesQueue.Peek())) return;
+            Logger.Info($"Purging {_blendShapesQueue.Peek().Count} frames", Logger.LogCategory.LipSync);
+            _blendShapesQueue.Dequeue();
+        }
+
+        public override void ClearQueue()
+        {
+            _blendShapesQueue = new Queue<Queue<BlendshapeFrame>>();
+            _currentBlendshape = new ARKitBlendShapes();
+        }
+
+        public override void EnqueueQueue(Queue<BlendshapeFrame> blendshapeFrames)
+        {
+            _blendShapesQueue.Enqueue(blendshapeFrames);
+        }
+
+        public override void EnqueueFrame(BlendshapeFrame blendshapeFrame)
+        {
+            if (_blendShapesQueue.Count == 0)
+            {
+                EnqueueQueue(new Queue<BlendshapeFrame>());
+            }
+            _blendShapesQueue.Peek().Enqueue(blendshapeFrame);
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiBlendShapeLipSync.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: ca3ca8129b12656449558f306c86f70d
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiOVRLipsync.cs

@@ -0,0 +1,162 @@
+using System.Collections.Generic;
+using System.Reflection;
+using Service;
+using UnityEngine;
+
+namespace Convai.Scripts.Utils.LipSync.Types
+{
+    public class ConvaiOVRLipsync : ConvaiVisemesLipSync
+    {
+        private int _firstIndex;
+
+        public override void Initialize(ConvaiLipSync convaiLipSync, ConvaiNPC convaiNPC)
+        {
+            base.Initialize(convaiLipSync, convaiNPC);
+            _firstIndex = convaiLipSync.firstIndex;
+        }
+
+        protected override Dictionary<string, string> GetHeadRegexMapping()
+        {
+            const string mouth = "[Mm]outh";
+            const string spacer = "[\\s_]*";
+            const string left = "[Ll]eft";
+            const string right = "[Rr]ight";
+            const string lower = "[Ll]ower";
+            const string upper = "[Uu]pper";
+            const string open = "[Oo]pen";
+            const string funnel = "[Ff]unnel";
+            const string pucker = "[Pp]ucker";
+            const string prefix = "(?:[A-Z]\\d{1,2}_)?";
+
+            return new Dictionary<string, string>
+            {
+                {"PP", $"{prefix}{mouth}{spacer}{pucker}"},
+                {"FF", $"{prefix}{mouth}{spacer}{funnel}"},
+                {"THL", $"{prefix}{mouth}{spacer}{lower}{spacer}[Dd]own{spacer}{left}"},
+                {"THR", $"{prefix}{mouth}{spacer}{lower}{spacer}[Dd]own{spacer}{right}"},
+                {"DDL", $"{prefix}{mouth}{spacer}[Pp]ress{spacer}{left}"},
+                {"DDR", $"{prefix}{mouth}{spacer}[Pp]ress{spacer}{right}"},
+                {"KK", $"{prefix}[Jj]aw{spacer}{open}"},
+                {"CHL", $"{prefix}{mouth}{spacer}[Ss]tretch{spacer}{left}"},
+                {"CHR", $"{prefix}{mouth}{spacer}[Ss]tretch{spacer}{right}"},
+                {"SSL", $"{prefix}{mouth}{spacer}[Ss]mile{spacer}{left}"},
+                {"SSR", $"{prefix}{mouth}{spacer}[Ss]mile{spacer}{right}"},
+                {"NNL", $"{prefix}[Nn]ose{spacer}[Ss]neer{spacer}{left}"},
+                {"NNR", $"{prefix}[Nn]ose{spacer}[Ss]neer{spacer}{right}"},
+                {"RRU", $"{prefix}{mouth}{spacer}[Rr]oll{spacer}{upper}"},
+                {"RRL", $"{prefix}{mouth}{spacer}[Rr]oll{spacer}{lower}"},
+                {"AA", $"{prefix}[Jj]aw{spacer}[Oo]pen"},
+                {"EL", $"{prefix}{mouth}{spacer}{upper}{spacer}[Uu]p{spacer}{left}"},
+                {"ER", $"{prefix}{mouth}{spacer}{upper}{spacer}[Uu]p{spacer}{right}"},
+                {"IHL", $"{prefix}{mouth}{spacer}[Ff]rown{spacer}{left}"},
+                {"IHR", $"{prefix}{mouth}{spacer}[Ff]rown{spacer}{right}"},
+                {"OU", $"{prefix}{mouth}{spacer}{pucker}"},
+                {"OH", $"{prefix}{mouth}{spacer}{funnel}"},
+            };
+        }
+
+        private void Update()
+        {
+            if (_currentViseme == null || _currentViseme.Sil == -2) return;
+
+            float weight;
+            List<int> knownIndexes = new List<int>();
+
+            UpdateJawBoneRotation(new Vector3(0.0f, 0.0f, -90.0f));
+            UpdateTongueBoneRotation(new Vector3(0.0f, 0.0f, -5.0f));
+
+            if (HasHeadSkinnedMeshRenderer)
+            {
+                foreach (PropertyInfo propertyInfo in typeof(Viseme).GetProperties())
+                {
+                    if (propertyInfo.PropertyType != typeof(float)) continue;
+
+                    string fieldName = propertyInfo.Name.ToUpper();
+                    float value = (float)propertyInfo.GetValue(_currentViseme);
+
+                    weight = fieldName switch
+                    {
+                        "KK" => 1.0f / 1.5f,
+                        "DD" => 1.0f / 0.7f,
+                        "CH" => 1.0f / 2.7f,
+                        "SS" => 1.0f / 1.5f,
+                        "NN" => 1.0f / 2.0f,
+                        "RR" => 1.0f / 0.9f,
+                        "AA" => 1.0f / 2.0f,
+                        "II" => 1.0f / 1.2f,
+                        "OH" => 1.2f,
+                        _ => 1.0f
+                    };
+
+                    foreach (string s in _possibleCombinations)
+                    {
+                        float weightThisFrame = value * weight * _weightMultiplier;
+                        string modifiedFieldName = fieldName + s;
+                        FindAndUpdateBlendWeight(_headSkinMeshRenderer, modifiedFieldName, weightThisFrame, knownIndexes, _headMapping);
+                    }
+                }
+            }
+
+            UpdateJawBoneRotation(new Vector3(0.0f, 0.0f, CalculateJawRotation()));
+            UpdateTongueBoneRotation(new Vector3(0.0f, 0.0f, CalculateTongueRotation()));
+
+            if (_teethSkinMeshRenderer.sharedMesh.blendShapeCount < (_firstIndex + 15)) return;
+
+            for (int i = 0; i < 15; i++)
+            {
+                float visemeValue = GetVisemeValueByIndex(i);
+                _teethSkinMeshRenderer.SetBlendShapeWeightInterpolate(_firstIndex + i, visemeValue * _weightMultiplier, Time.deltaTime);
+            }
+        }
+
+        private float CalculateJawRotation()
+        {
+            float totalWeight = 0.2f + 0.1f + 0.5f + 0.2f + 0.2f + 1.0f + 0.2f + 0.3f + 0.8f + 0.3f;
+            float rotation = (0.2f * _currentViseme.Th
+                             + 0.1f * _currentViseme.Dd
+                             + 0.5f * _currentViseme.Kk
+                             + 0.2f * _currentViseme.Nn
+                             + 0.2f * _currentViseme.Rr
+                             + 1.0f * _currentViseme.Aa
+                             + 0.2f * _currentViseme.E
+                             + 0.3f * _currentViseme.Ih
+                             + 0.8f * _currentViseme.Oh
+                             + 0.3f * _currentViseme.Ou) / totalWeight;
+
+            return -90.0f - rotation * 30f;
+        }
+
+        private float CalculateTongueRotation()
+        {
+            float totalWeight = 0.1f + 0.2f + 0.15f;
+            float rotation = (0.1f * _currentViseme.Th
+                             + 0.2f * _currentViseme.Nn
+                             + 0.15f * _currentViseme.Rr) / totalWeight;
+
+            return rotation * 80f - 5f;
+        }
+
+        private float GetVisemeValueByIndex(int index)
+        {
+            return index switch
+            {
+                0 => _currentViseme.Sil,
+                1 => _currentViseme.Pp,
+                2 => _currentViseme.Ff,
+                3 => _currentViseme.Th,
+                4 => _currentViseme.Dd,
+                5 => _currentViseme.Kk,
+                6 => _currentViseme.Ch,
+                7 => _currentViseme.Ss,
+                8 => _currentViseme.Nn,
+                9 => _currentViseme.Rr,
+                10 => _currentViseme.Aa,
+                11 => _currentViseme.E,
+                12 => _currentViseme.Ih,
+                13 => _currentViseme.Oh,
+                14 => _currentViseme.Ou,
+                _ => 0.0f
+            };
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiOVRLipsync.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 769391e6890ecb0459ada7f3c4fb1400
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiReallusionLipSync.cs

@@ -0,0 +1,117 @@
+using System.Collections.Generic;
+using System.Reflection;
+using Service;
+using UnityEngine;
+namespace Convai.Scripts.Utils.LipSync.Types
+{
+    public class ConvaiReallusionLipSync : ConvaiVisemesLipSync
+    {
+        protected override Dictionary<string, string> GetHeadRegexMapping()
+        {
+            string mouth = "[Mm]outh";
+            string lower = "[Ll]ower";
+            string spacer = "[\\s_]*";
+            string prefix = "(?:[A-Z]\\d{1,2}_)?";
+
+            return new Dictionary<string, string>()
+            {
+                {"PP", $"{prefix}[Vv]{spacer}[Ee]xplosive"},
+                {"FF", $"{prefix}[Vv]{spacer}[Dd]ental{spacer}[Ll]ip"},
+                {"TH", $"{prefix}{mouth}{spacer}[Dd]rop{spacer}{lower}"},
+                {"DDL", $"{prefix}{mouth}{spacer}[Dd]rop{spacer}[Ll]ower"},
+                {"DDU", $"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Uu]pper"},
+                {"KKL", $"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Ll]ower"},
+                {"KKU", $"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Uu]pper"},
+                {"CHL",$"{prefix}{mouth}[Dd]rop{spacer}[Ll]ower"},
+                {"CHU",$"{prefix}{mouth}[Dd]rop{spacer}[Uu]pper"},
+                {"CHO",$"{prefix}[Vv]{spacer}[Ll]ip{spacer}[Oo]pen"},
+                {"SSL", $"{prefix}{mouth}{spacer}[Dd]rop{spacer}[Ll]ower"},
+                {"SSU", $"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Uu]pper"},
+                {"NNL", $"{prefix}{mouth}{spacer}[Dd]rop{spacer}[Ll]ower"},
+                {"NNU", $"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Uu]pper"},
+                {"RR",$"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Uu]pper"},
+                {"AA", $"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Uu]pper"},
+                {"EL", $"{prefix}{mouth}{spacer}[Dd]rop{spacer}[Ll]ower"},
+                {"EU", $"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Uu]pper"},
+                {"IHL", $"{prefix}{mouth}{spacer}[Dd]rop{spacer}[Ll]ower"},
+                {"IHU",$"{prefix}{mouth}{spacer}[Ss]hrug{spacer}[Uu]pper"},
+                {"OH", $"{prefix}[Vv]{spacer}[Tt]ight{spacer}[Oo]"},
+                {"OU", $"{prefix}[Vv]{spacer}[Tt]ight{spacer}[Oo]"},
+            };
+        }
+
+        private void Update()
+        {
+            // Check if the dequeued frame is not null.
+            if (_currentViseme == null) return;
+            // Check if the frame represents silence (-2 is a placeholder for silence).
+            if (_currentViseme.Sil == -2) return;
+
+            float weight;
+            List<int> knownIndexs = new List<int>();
+            UpdateJawBoneRotation(new Vector3(0.0f, 0.0f, -90.0f));
+            UpdateTongueBoneRotation(new Vector3(0.0f, 0.0f, -5.0f));
+            if (HasHeadSkinnedMeshRenderer)
+            {
+                foreach (PropertyInfo propertyInfo in typeof(Viseme).GetProperties())
+                {
+                    if (propertyInfo.PropertyType != typeof(float)) continue;
+                    string fieldName = propertyInfo.Name.ToUpper();
+                    float value = (float)propertyInfo.GetValue(_currentViseme);
+                    weight = fieldName switch
+                    {
+                        "TH" => 0.5f,
+                        "DDL" => 0.2f / 0.7f,
+                        "DDU" => 0.5f / 2.7f,
+                        "KKL" => 0.5f / 1.5f,
+                        "KKU" => 1.0f / 1.5f,
+                        "CHL" => 0.7f / 2.7f,
+                        "CHU" => 1.0f / 2.7f,
+                        "CHO" => 1.0f / 2.7f,
+                        "SSL" => 0.5f / 1.5f,
+                        "SSU" => 1.0f / 1.5f,
+                        "NNL" => 0.5f / 2.0f,
+                        "NNU" => 1.0f / 2.0f,
+                        "RR" => 0.5f / 0.9f,
+                        "AA" => 1.0f / 2.0f,
+                        "EL" => 0.7f,
+                        "EU" => 0.3f,
+                        "IHL" => 0.7f / 1.2f,
+                        "IHU" => 0.5f / 1.2f,
+                        "OH" => 1.2f,
+                        _ => 1.0f
+                    };
+                    foreach (string s in _possibleCombinations)
+                    {
+                        float weightThisFrame = value * weight * _weightMultiplier;
+                        string modifiedFieldName = fieldName + s;
+                        FindAndUpdateBlendWeight(_headSkinMeshRenderer, modifiedFieldName, weightThisFrame, knownIndexs, _headMapping);
+                    }
+                }
+            }
+
+
+            UpdateJawBoneRotation(new Vector3(0.0f, 0.0f, -90.0f - (
+                        0.2f * _currentViseme.Th
+                        + 0.1f * _currentViseme.Dd
+                        + 0.5f * _currentViseme.Kk
+                        + 0.2f * _currentViseme.Nn
+                        + 0.2f * _currentViseme.Rr
+                        + 1.0f * _currentViseme.Aa
+                        + 0.2f * _currentViseme.E
+                        + 0.3f * _currentViseme.Ih
+                        + 0.8f * _currentViseme.Oh
+                        + 0.3f * _currentViseme.Ou
+                    )
+                    / (0.2f + 0.1f + 0.5f + 0.2f + 0.2f + 1.0f + 0.2f + 0.3f + 0.8f + 0.3f)
+                    * 30f));
+            UpdateTongueBoneRotation(new Vector3(0.0f, 0.0f, (
+                        0.1f * _currentViseme.Th
+                        + 0.2f * _currentViseme.Nn
+                        + 0.15f * _currentViseme.Rr
+                    )
+                    / (0.1f + 0.2f + 0.15f)
+                    * 80f - 5f));
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiReallusionLipSync.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 04cd2132229efcc4f9848b088c03e55b
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiVisemesLipSync.cs

@@ -0,0 +1,102 @@
+using System.Collections.Generic;
+using Service;
+using UnityEngine;
+namespace Convai.Scripts.Utils.LipSync.Types
+{
+    public abstract class ConvaiVisemesLipSync : ConvaiLipSyncApplicationBase
+    {
+        private const float FRAMERATE = 1f / 100.0f;
+        protected Queue<Queue<VisemesData>> _visemesDataQueue = new Queue<Queue<VisemesData>>();
+        protected Viseme _currentViseme;
+        protected List<string> _possibleCombinations = new List<string>();
+
+        public override void Initialize(ConvaiLipSync convaiLipSync, ConvaiNPC convaiNPC)
+        {
+            base.Initialize(convaiLipSync, convaiNPC);
+            _possibleCombinations = new List<string> { "", "R", "U", "L", "O" };
+            InvokeRepeating(nameof(UpdateBlendShape), 0, FRAMERATE);
+        }
+
+        public override void ClearQueue()
+        {
+            _visemesDataQueue = new Queue<Queue<VisemesData>>();
+            _currentViseme = new Viseme();
+        }
+
+        public override void PurgeExcessBlendShapeFrames()
+        {
+            if (_visemesDataQueue.Count == 0) return;
+            if (!CanPurge<VisemesData>(_visemesDataQueue.Peek())) return;
+            Logger.Info($"Purging {_visemesDataQueue.Peek().Count} Frames", Logger.LogCategory.LipSync);
+            _visemesDataQueue.Dequeue();
+        }
+
+
+
+        protected override void UpdateBlendShape()
+        {
+            if (_visemesDataQueue == null || _visemesDataQueue.Count <= 0)
+            {
+                _currentViseme = new Viseme();
+                return;
+            }
+            // Dequeue the next frame of visemes data from the faceDataList.
+            if (_visemesDataQueue.Peek().Count <= 0 || _visemesDataQueue.Peek() == null)
+            {
+                _visemesDataQueue.Dequeue();
+                return;
+            }
+            if (!_convaiNPC.IsCharacterTalking) return;
+
+            _currentViseme = _visemesDataQueue.Peek().Dequeue().Visemes;
+        }
+
+        protected void FindAndUpdateBlendWeight(SkinnedMeshRenderer renderer, string fieldName, float value, List<int> knownIndexs, Dictionary<string, int> mapping)
+        {
+            if (mapping.TryGetValue(fieldName, out int index))
+            {
+                if (!knownIndexs.Contains(index))
+                {
+                    knownIndexs.Add(index);
+                    UpdateWeight(renderer, index, value, true);
+                }
+                else
+                {
+                    UpdateWeight(_headSkinMeshRenderer, index, value, false);
+                }
+            }
+        }
+
+        protected virtual void UpdateWeight(SkinnedMeshRenderer renderer, int index, float value, bool firstTime)
+        {
+            if (value == 0f)
+            {
+                renderer.SetBlendShapeWeight(index, 0);
+                return;
+            }
+
+            if (FRAMERATE > Time.deltaTime)
+            {
+                renderer.SetBlendShapeWeight(index, (firstTime ? 0 : renderer.GetBlendShapeWeight(index)) + value);
+            }
+            else
+            {
+                renderer.SetBlendShapeWeightInterpolate(index, (firstTime ? 0 : renderer.GetBlendShapeWeight(index)) + value, Time.deltaTime);
+            }
+        }
+
+        public override void EnqueueQueue(Queue<VisemesData> visemesFrames)
+        {
+            _visemesDataQueue.Enqueue(visemesFrames);
+        }
+
+        public override void EnqueueFrame(VisemesData viseme)
+        {
+            if (_visemesDataQueue.Count == 0)
+            {
+                EnqueueQueue(new Queue<VisemesData>());
+            }
+            _visemesDataQueue.Peek().Enqueue(viseme);
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiVisemesLipSync.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 1917f6a14d9682440a27a434f0210496
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/NPC2NPC.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 533d7f0ec16149f2b19fb5b228e7a1d8
+timeCreated: 1705687336

Assets/Convai/Scripts/Runtime/Features/NPC2NPC/ConvaiGroupNPCController.cs

@@ -0,0 +1,226 @@
+using System;
+using System.Collections;
+using System.Threading.Tasks;
+using Convai.Scripts;
+using Convai.Scripts.Utils;
+using Convai.Scripts.Utils.LipSync;
+using Service;
+using UnityEngine;
+using Logger = Convai.Scripts.Utils.Logger;
+
+/// <summary>
+///     This class is responsible for handling out all the tasks related to NPC to NPC conversation for a NPC of a group
+/// </summary>
+public class ConvaiGroupNPCController : MonoBehaviour
+{
+    #region Serialized Fields
+    [Tooltip("The prefab for the speech bubble to display above the NPC. [Optional]")]
+    [SerializeField]
+    private NPCSpeechBubble speechBubblePrefab;
+
+    [Tooltip("Attach the Main Player Transform here so that distance check can be performed")]
+    [SerializeField]
+    private Transform playerTransform;
+
+    // The distance from the NPC to the player when the NPC will start talking
+    [Tooltip("The distance from the NPC to the player when the NPC will start talking. Set to 0 to disable this feature. [Optional]")]
+    [SerializeField] [Range(0f, 100f)]
+    private float conversationDistanceThreshold = 5.0f;
+    #endregion
+
+    #region Private Attributes
+    private const float PLAYER_MOVE_THRESHOLD = 0.5f;
+    private float CONVERSATION_DISTANCE_THRESHOLD;
+    private string _finalResponseText = "";
+    private NPC2NPCGRPCClient _npc2NPCGrpcClient;
+    private ConvaiLipSync _lipSync;
+    private Coroutine _checkPlayerVicinityCoroutine;
+    private FaceModel FaceModel => _lipSync == null ? FaceModel.OvrModelName : _lipSync.faceModel;
+
+    #endregion
+
+    #region Events
+    public event Action<string> ShowSpeechBubble;
+    public event Action HideSpeechBubble;
+    public event Action<bool, ConvaiGroupNPCController> OnPlayerVicinityChanged;
+    #endregion
+
+    #region Public Attributes
+    public bool CanRelayMessage { get; set; } = true;
+    public NPC2NPCConversationManager ConversationManager { get; set; }
+    public bool IsInConversationWithAnotherNPC { get; set; }
+    public string CharacterName => ConvaiNPC == null ? string.Empty : ConvaiNPC.characterName;
+    public string CharacterID => ConvaiNPC == null ? string.Empty : ConvaiNPC.characterID;
+    public ConvaiNPC ConvaiNPC { get; private set; }
+
+    #endregion
+
+    /// <summary>
+    ///     Used to set Player GameObject Transform and lip-sync
+    /// </summary>
+    private void Awake()
+    {
+        if (playerTransform == null) playerTransform = Camera.main.transform;
+        ConvaiNPC = GetComponent<ConvaiNPC>();
+        CONVERSATION_DISTANCE_THRESHOLD = conversationDistanceThreshold == 0 ? Mathf.Infinity : conversationDistanceThreshold;
+        TryGetComponent(out _lipSync);
+    }
+    /// <summary>
+    ///     Starts coroutine for player vicinity check and subscribe to necessary events
+    /// </summary>
+    private void Start()
+    {
+        _checkPlayerVicinityCoroutine = StartCoroutine(CheckPlayerVicinity());
+        if (TryGetComponent(out ConvaiNPCAudioManager convaiNPCAudio))
+        {
+            convaiNPCAudio.OnAudioTranscriptAvailable += HandleAudioTranscriptAvailable;
+        }
+    }
+    /// <summary>
+    ///     Unsubscribes to the events and stops the coroutine
+    /// </summary>
+    private void OnDestroy()
+    {
+        if (TryGetComponent(out ConvaiNPCAudioManager convaiNPCAudio))
+        {
+            convaiNPCAudio.OnAudioTranscriptAvailable -= HandleAudioTranscriptAvailable;
+        }
+        if (_checkPlayerVicinityCoroutine != null)
+        {
+            StopCoroutine(_checkPlayerVicinityCoroutine);
+        }
+    }
+    /// <summary>
+    ///     Shows speech bubble and adds the received text to final transcript
+    /// </summary>
+    /// <param name="transcript"></param>
+    private void HandleAudioTranscriptAvailable(string transcript)
+    {
+        if (IsInConversationWithAnotherNPC)
+            ShowSpeechBubble?.Invoke(transcript);
+        _finalResponseText += transcript;
+    }
+    /// <summary>
+    ///     Attaches the speech bubble to the NPC game-object
+    /// </summary>
+    public void AttachSpeechBubble()
+    {
+        if (TryGetComponent(out ConvaiSpeechBubbleController _)) return;
+        gameObject.AddComponent<ConvaiSpeechBubbleController>().Initialize(speechBubblePrefab, this);
+    }
+    /// <summary>
+    ///     Destroys the speech bubble game-object
+    /// </summary>
+    public void DetachSpeechBubble()
+    {
+        if (TryGetComponent(out ConvaiSpeechBubbleController convaiSpeechBubble)) Destroy(convaiSpeechBubble);
+    }
+    /// <summary>
+    ///     Store the references of the client and subscribe to the necessary events
+    /// </summary>
+    /// <param name="client"></param>
+    public void InitializeNpc2NpcGrpcClient(NPC2NPCGRPCClient client)
+    {
+        _npc2NPCGrpcClient = client;
+    }
+    /// <summary>
+    ///     Every 0.5 seconds updates if player is near or not and fire events according to the state
+    /// </summary>
+    /// <returns></returns>
+    private IEnumerator CheckPlayerVicinity()
+    {
+        bool previousState = false;
+        Vector3 previousPlayerPosition = Vector3.zero;
+        yield return new WaitForSeconds(0.1f);
+        while (true)
+        {
+            Vector3 currentPlayerPosition = playerTransform.transform.position;
+
+            // Check if the player has moved more than a certain threshold distance
+            if (Vector3.Distance(previousPlayerPosition, currentPlayerPosition) > PLAYER_MOVE_THRESHOLD)
+            {
+                // Calculate the distance between the NPC and the player
+                float distanceToPlayer = Vector3.Distance(transform.position, currentPlayerPosition);
+
+                // Check if the player is within the threshold distance
+                bool isPlayerCurrentlyNear = distanceToPlayer <= CONVERSATION_DISTANCE_THRESHOLD;
+
+                // If the player's current vicinity state is different from the previous state, raise the event
+                if (isPlayerCurrentlyNear != previousState && !ConvaiNPC.isCharacterActive)
+                {
+                    OnPlayerVicinityChanged?.Invoke(isPlayerCurrentlyNear, this);
+                    previousState = isPlayerCurrentlyNear; // Update the previous state
+                    Debug.Log($"Player is currently near {ConvaiNPC.characterName}: {isPlayerCurrentlyNear}");
+                }
+
+                previousPlayerPosition = currentPlayerPosition; // Update the player's previous position
+                // Check every half second
+            }
+            yield return new WaitForSeconds(0.5f);
+        }
+    }
+    /// <summary>
+    /// Sends the text to the other NPC in the group
+    /// </summary>
+    /// <param name="message"></param>
+    public async void SendTextDataNPC2NPC(string message)
+    {
+        if (_npc2NPCGrpcClient == null)
+        {
+            Logger.Warn("No GRPC client initialized for this NPC.", Logger.LogCategory.Character);
+            return;
+        }
+
+        try
+        {
+            CanRelayMessage = false;
+            await Task.Delay(500);
+            await _npc2NPCGrpcClient.SendTextData(
+                userText: message,
+                characterID: ConvaiNPC.characterID,
+                sessionID: ConvaiNPC.sessionID,
+                isLipSyncActive: _lipSync != null,
+                faceModel: FaceModel);
+        }
+        catch (Exception ex)
+        {
+            Logger.Warn($"Error sending message data for NPC2NPC: {ex.Message}", Logger.LogCategory.Character);
+        }
+    }
+
+    public void EndOfResponseReceived()
+    {
+        if (TryGetComponent(out ConvaiNPCAudioManager convaiNPCAudio))
+        {
+            convaiNPCAudio.OnCharacterTalkingChanged += SendFinalTranscriptToOtherNPC;
+        }
+    }
+
+    private void SendFinalTranscriptToOtherNPC(bool isTalking)
+    {
+        if (IsInConversationWithAnotherNPC)
+        {
+            if (!isTalking)
+            {
+                if (TryGetComponent(out ConvaiNPCAudioManager convaiNPCAudio))
+                {
+                    convaiNPCAudio.OnCharacterTalkingChanged -= SendFinalTranscriptToOtherNPC;
+                }
+                ConversationManager.RelayMessage(_finalResponseText, this);
+                _finalResponseText = "";
+                HideSpeechBubble?.Invoke();
+            }
+            else
+            {
+                Logger.DebugLog($"{ConvaiNPC.characterName} is currently still talking. ", Logger.LogCategory.Character);
+            }
+        }
+    }
+
+    public bool IsPlayerNearMe()
+    {
+        bool result = Vector3.Distance(transform.position, playerTransform.position) < CONVERSATION_DISTANCE_THRESHOLD;
+        Logger.Info($"Player is near {CharacterName}: {result}", Logger.LogCategory.Character);
+        return result;
+    }
+}

Assets/Convai/Scripts/Runtime/Features/NPC2NPC/ConvaiGroupNPCController.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 74d3f2dd15d074a429eeea8c31f80b13
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPC2NPCConversationManager.cs

@@ -0,0 +1,297 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using Convai.Scripts.Runtime.Utils;
+using Convai.Scripts.Utils;
+using Grpc.Core;
+using Service;
+using UnityEngine;
+using Logger = Convai.Scripts.Utils.Logger;
+using Random = UnityEngine.Random;
+
+/// <summary>
+///     Manages the conversation between two Convai powered NPC groups
+/// </summary>
+public class NPC2NPCConversationManager : MonoBehaviour
+{
+    private const string GRPC_API_ENDPOINT = "stream.convai.com";
+    public static NPC2NPCConversationManager Instance;
+    public List<NPCGroup> npcGroups;
+    private readonly List<NPCGroup> _groupsWhereConversationEnded = new();
+    private string _apiKey = string.Empty;
+
+    private void Awake()
+    {
+        if (Instance == null)
+            Instance = this;
+        else
+            Destroy(gameObject);
+        LoadApiKey();
+    }
+
+    private void Start()
+    {
+        foreach (NPCGroup group in npcGroups)
+            group.Initialize(HandlePlayerVicinityChanged);
+
+        StartConversationWithAllNPCs();
+    }
+
+    /// <summary>
+    ///     Handles the event of player vicinity change.
+    /// </summary>
+    /// <param name="isPlayerNear">Indicates if the player is near.</param>
+    /// <param name="npc">The NPC for which the vicinity changed.</param>
+    private void HandlePlayerVicinityChanged(bool isPlayerNear, ConvaiGroupNPCController npc)
+    {
+        if (isPlayerNear)
+            ResumeConversation(npc);
+    }
+
+
+    /// <summary>
+    ///     Processes the message from the sender NPC.
+    /// </summary>
+    /// <param name="sender">The NPC sending the message.</param>
+    /// <param name="topic">The topic of the conversation.</param>
+    /// <param name="message">The message to be processed.</param>
+    /// <returns>The processed message.</returns>
+    private string ProcessMessage(ConvaiGroupNPCController sender, string topic, string message)
+    {
+        string processedMessage = $"{sender.CharacterName} said \"{message}\" to you. Reply to it. ";
+
+        processedMessage += Random.Range(0, 2) == 0
+            ? $"Talk about something related to {message}. "
+            : $"Talk about something other than \"{message}\" but related to {topic}. Gently change the conversation topic. ";
+
+        return processedMessage + "Definitely, reply to the message. Dont address speaker. Keep the reply short. Do not repeat the same message, or keep asking same question.";
+    }
+
+    /// <summary>
+    ///     Relays the message from the sender NPC.
+    /// </summary>
+    /// <param name="message">The message to be relayed.</param>
+    /// <param name="sender">The NPC sending the message.</param>
+    /// <param name="performSwitch"></param>
+    public void RelayMessage(string message, ConvaiGroupNPCController sender, bool performSwitch = true)
+    {
+        NPCGroup npcGroup = npcGroups.Find(c => c.BelongToGroup(sender));
+        if (npcGroup == null)
+        {
+            Debug.LogError("Conversation not found for the sender.");
+            return;
+        }
+
+        npcGroup.messageToRelay = message;
+        if (performSwitch) SwitchSpeaker(npcGroup.CurrentSpeaker);
+        if (!npcGroup.CurrentSpeaker.IsPlayerNearMe()) return;
+        StartCoroutine(RelayMessageCoroutine(message, npcGroup));
+    }
+
+    /// <summary>
+    ///     Coroutine to relay the message from the sender NPC.
+    /// </summary>
+    /// <param name="message">The message to be relayed.</param>
+    /// <param name="npcGroup"> The NPC group to relay the message to. </param>
+    /// <returns>An IEnumerator to be used in a coroutine.</returns>
+    private IEnumerator RelayMessageCoroutine(string message, NPCGroup npcGroup)
+    {
+        yield return new WaitForSeconds(0.5f);
+
+        try
+        {
+            ConvaiGroupNPCController receiver = npcGroup.CurrentSpeaker;
+            if (!receiver.CanRelayMessage) yield break;
+
+            Logger.DebugLog($"Relaying message from {npcGroup.CurrentListener.CharacterName} to {receiver.CharacterName}: {message}", Logger.LogCategory.Character);
+
+            string processedMessage = ProcessMessage(npcGroup.CurrentListener, npcGroup.topic, message);
+            receiver.SendTextDataNPC2NPC(processedMessage);
+        }
+        catch (Exception e)
+        {
+            Logger.Warn($"Failed to relay message: {e.Message}", Logger.LogCategory.Character);
+        }
+    }
+
+    /// <summary>
+    ///     Switches the speaker in the conversation.
+    /// </summary>
+    /// <param name="currentSpeaker">The current speaker NPC.</param>
+    /// <returns>The new speaker NPC.</returns>
+    private void SwitchSpeaker(ConvaiGroupNPCController currentSpeaker)
+    {
+        NPCGroup group = npcGroups.Find(g => g.CurrentSpeaker == currentSpeaker);
+        if (group != null)
+        {
+            group.CurrentSpeaker = currentSpeaker == group.GroupNPC1 ? group.GroupNPC2 : group.GroupNPC1;
+            Logger.DebugLog($"Switching NPC2NPC Speaker to {group.CurrentSpeaker}", Logger.LogCategory.Character);
+            return;
+        }
+
+        Logger.Warn("Failed to switch speaker. Current speaker not found in any group.", Logger.LogCategory.Character);
+    }
+
+    private void LoadApiKey()
+    {
+        ConvaiAPIKeySetup.GetAPIKey(out _apiKey);
+    }
+
+    /// <summary>
+    ///     Initializes a single NPC for conversation.
+    /// </summary>
+    /// <param name="npc">The NPC to initialize.</param>
+    /// <param name="grpcClient">The GRPC client to use for the NPC.</param>
+    private void InitializeNPC(ConvaiGroupNPCController npc, NPC2NPCGRPCClient grpcClient)
+    {
+        if (npc == null)
+        {
+            Debug.LogWarning("The given NPC is null.");
+            return;
+        }
+
+        npc.ConversationManager = this;
+        npc.InitializeNpc2NpcGrpcClient(grpcClient);
+        npc.AttachSpeechBubble();
+        npc.IsInConversationWithAnotherNPC = true;
+        npc.ConvaiNPC.isCharacterActive = false;
+    }
+
+    /// <summary>
+    ///     Starts the conversation for the given NPC.
+    /// </summary>
+    /// <param name="npcGroup">The NPC to start the conversation for.</param>
+    private void InitializeNPCGroup(NPCGroup npcGroup)
+    {
+        if (npcGroup == null)
+        {
+            Debug.LogWarning("The given NPC is not part of any group.");
+            return;
+        }
+
+        ConvaiGroupNPCController npc1 = npcGroup.GroupNPC1;
+        ConvaiGroupNPCController npc2 = npcGroup.GroupNPC2;
+
+        if (npc1.IsInConversationWithAnotherNPC || npc2.IsInConversationWithAnotherNPC)
+        {
+            Debug.LogWarning($"{npc1.CharacterName} or {npc2.CharacterName} is already in a conversation.");
+            return;
+        }
+
+        NPC2NPCGRPCClient grpcClient = CreateAndInitializeGRPCClient(npcGroup);
+
+        // Initialize both NPCs
+        InitializeNPC(npc1, grpcClient);
+        InitializeNPC(npc2, grpcClient);
+
+        npcGroup.CurrentSpeaker = Random.Range(0, 10) % 2 == 0 ? npc1 : npc2;
+    }
+
+    /// <summary>
+    ///     Creates and initializes a new GRPC client for the given NPC group.
+    /// </summary>
+    /// <param name="group">The NPC group to create the GRPC client for.</param>
+    /// <returns>The initialized GRPC client.</returns>
+    private NPC2NPCGRPCClient CreateAndInitializeGRPCClient(NPCGroup group)
+    {
+        GameObject grpcClientGameObject = new($"GRPCClient_{group.GroupNPC1.CharacterID}_{group.GroupNPC2.CharacterID}")
+        {
+            transform = { parent = transform }
+        };
+
+        NPC2NPCGRPCClient grpcClient = grpcClientGameObject.AddComponent<NPC2NPCGRPCClient>();
+
+        ConvaiService.ConvaiServiceClient serviceClient = CreateNewConvaiServiceClient();
+        grpcClient.Initialize(_apiKey, serviceClient, group);
+
+        return grpcClient;
+    }
+
+    /// <summary>
+    ///     Creates a new ConvaiServiceClient.
+    /// </summary>
+    /// <returns> The new ConvaiServiceClient. </returns>
+    private ConvaiService.ConvaiServiceClient CreateNewConvaiServiceClient()
+    {
+        try
+        {
+            SslCredentials credentials = new();
+            Channel channel = new(GRPC_API_ENDPOINT, credentials);
+            return new ConvaiService.ConvaiServiceClient(channel);
+        }
+        catch (Exception ex)
+        {
+            Logger.Error($"Failed to create ConvaiServiceClient: {ex.Message}", Logger.LogCategory.Character);
+            throw;
+        }
+    }
+
+    /// <summary>
+    ///     Resumes the conversation for the given NPC.
+    /// </summary>
+    /// <param name="sender"> The NPC to resume the conversation for. </param>
+    private void ResumeConversation(ConvaiGroupNPCController sender)
+    {
+        NPCGroup npcGroup = npcGroups.Find(g => g.BelongToGroup(sender));
+        if (npcGroup.IsAnyoneTalking()) return;
+
+        if (_groupsWhereConversationEnded.Contains(npcGroup))
+        {
+            InitializeNPCGroup(npcGroup);
+            _groupsWhereConversationEnded.Remove(npcGroup);
+        }
+
+        if (string.IsNullOrEmpty(npcGroup.messageToRelay))
+        {
+            string message = $"Talk about {npcGroup.topic}.";
+            npcGroup.CurrentSpeaker.SendTextDataNPC2NPC(message);
+            npcGroup.messageToRelay = message;
+
+            Debug.Log($"Starting conversation for the first time between {npcGroup.GroupNPC1.CharacterName} and {npcGroup.GroupNPC2.CharacterName}");
+        }
+        else
+        {
+            RelayMessage(npcGroup.messageToRelay, npcGroup.CurrentSpeaker, false);
+            Debug.Log($"Resuming conversation between {npcGroup.GroupNPC1.CharacterName} and {npcGroup.GroupNPC2.CharacterName}");
+        }
+    }
+
+    /// <summary>
+    ///     Ends the conversation for the given NPC.
+    /// </summary>
+    /// <param name="npc"> The NPC to end the conversation for. </param>
+    public void EndConversation(ConvaiGroupNPCController npc)
+    {
+        NPCGroup group = npcGroups.Find(g => g.BelongToGroup(npc));
+        Debug.Log($"Ending conversation between {group.GroupNPC1.CharacterName} and {group.GroupNPC2.CharacterName}");
+
+        void EndConversationForNPC(ConvaiGroupNPCController groupNPC)
+        {
+            groupNPC.IsInConversationWithAnotherNPC = false;
+            groupNPC.ConvaiNPC.InterruptCharacterSpeech();
+            groupNPC.GetComponent<ConvaiGroupNPCController>().DetachSpeechBubble();
+        }
+
+        EndConversationForNPC(group.GroupNPC1);
+        EndConversationForNPC(group.GroupNPC2);
+        _groupsWhereConversationEnded.Add(group);
+        ConvaiNPCManager.Instance.SetActiveConvaiNPC(npc.ConvaiNPC);
+
+        Destroy(transform.Find($"GRPCClient_{group.GroupNPC1.CharacterID}_{group.GroupNPC2.CharacterID}").gameObject);
+    }
+
+
+    /// <summary>
+    ///     Starts the conversation with all NPCs.
+    /// </summary>
+    private void StartConversationWithAllNPCs()
+    {
+        IEnumerable filteredList = npcGroups
+            .Where(npcGroup => npcGroup.BothNPCAreNotNull())
+            .Where(npcGroup => npcGroup.BothNPCAreNotActiveNPC());
+
+        foreach (NPCGroup npcGroup in filteredList)
+            InitializeNPCGroup(npcGroup);
+    }
+}

Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPC2NPCConversationManager.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: e5c3956535ee405983974a5434b6f4a4
+timeCreated: 1705687346

Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPC2NPCGRPCClient.cs

@@ -0,0 +1,240 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Convai.Scripts.Utils.LipSync;
+using Grpc.Core;
+using Service;
+using UnityEngine;
+
+namespace Convai.Scripts.Utils
+{
+    /// <summary>
+    ///     Represents an NPC2NPCGRPCClient that can be used to communicate with the ConvaiService using gRPC.
+    /// </summary>
+    public class NPC2NPCGRPCClient : MonoBehaviour
+    {
+        private readonly CancellationTokenSource _cancellationTokenSource = new();
+        private string _apiKey;
+        private ConvaiService.ConvaiServiceClient _client;
+        private NPCGroup _npcGroup;
+
+        private void OnDestroy()
+        {
+            _cancellationTokenSource?.Cancel();
+            _cancellationTokenSource?.Dispose();
+        }
+
+        /// <summary>
+        ///     Initializes the NPC2NPCGRPCClient with the given API key and ConvaiService client.
+        /// </summary>
+        /// <param name="apiKey">The API key to use for authentication.</param>
+        /// <param name="client">The ConvaiService client to use for communication.</param>
+        public void Initialize(string apiKey, ConvaiService.ConvaiServiceClient client, NPCGroup group)
+        {
+            _apiKey = apiKey;
+            _client = client;
+            _npcGroup = group;
+        }
+
+        /// <summary>
+        ///     Creates an AsyncDuplexStreamingCall with the specified headers.
+        /// </summary>
+        /// <returns>An AsyncDuplexStreamingCall with the specified headers.</returns>
+        private AsyncDuplexStreamingCall<GetResponseRequest, GetResponseResponse> GetAsyncDuplexStreamingCallOptions()
+        {
+            Metadata headers = new()
+            {
+                { "source", "Unity" },
+                { "version", "3.0.0" }
+            };
+
+            CallOptions options = new(headers);
+            return _client.GetResponse(options);
+        }
+
+        /// <summary>
+        ///     Sends the specified user text to the server and receives a response.
+        /// </summary>
+        /// <param name="userText">The user text to send to the server.</param>
+        /// <param name="characterID">The ID of the character to use for the request.</param>
+        /// <param name="sessionID">The ID of the session to use for the request.</param>
+        /// <param name="isLipSyncActive">Whether lip sync is active for the request.</param>
+        /// <param name="faceModel">The face model to use for the request.</param>
+        /// <returns>A task that represents the asynchronous operation.</returns>
+        public async Task SendTextData(string userText, string characterID, string sessionID, bool isLipSyncActive, FaceModel faceModel)
+        {
+            AsyncDuplexStreamingCall<GetResponseRequest, GetResponseResponse> call = GetAsyncDuplexStreamingCallOptions();
+
+            GetResponseRequest getResponseConfigRequest = CreateGetResponseRequest(characterID, sessionID, isLipSyncActive, faceModel, false, null);
+
+            try
+            {
+                await call.RequestStream.WriteAsync(getResponseConfigRequest);
+                await call.RequestStream.WriteAsync(new GetResponseRequest
+                {
+                    GetResponseData = new GetResponseRequest.Types.GetResponseData
+                    {
+                        TextData = userText
+                    }
+                });
+                await call.RequestStream.CompleteAsync();
+
+                Task receiveResultsTask = Task.Run(
+                    async () => { await ReceiveResultFromServer(call, _cancellationTokenSource.Token); },
+                    _cancellationTokenSource.Token);
+                await receiveResultsTask.ConfigureAwait(false);
+            }
+            catch (Exception)
+            {
+                // ignored
+            }
+        }
+
+        /// <summary>
+        ///     Creates a GetResponseRequest with the specified parameters.
+        /// </summary>
+        /// <param name="characterID">The ID of the character to use for the request.</param>
+        /// <param name="sessionID">The ID of the session to use for the request.</param>
+        /// <param name="isLipSyncActive">Whether lip sync is active for the request.</param>
+        /// <param name="faceModel">The face model to use for the request.</param>
+        /// <param name="isActionActive">Whether action is active for the request.</param>
+        /// <param name="actionConfig">The action configuration to use for the request.</param>
+        /// <returns>A GetResponseRequest with the specified parameters.</returns>
+        private GetResponseRequest CreateGetResponseRequest(string characterID, string sessionID, bool isLipSyncActive, FaceModel faceModel, bool isActionActive,
+            ActionConfig actionConfig)
+        {
+            GetResponseRequest getResponseConfigRequest = new()
+            {
+                GetResponseConfig = new GetResponseRequest.Types.GetResponseConfig
+                {
+                    CharacterId = characterID,
+                    ApiKey = _apiKey,
+                    SessionId = sessionID,
+                    AudioConfig = new AudioConfig
+                    {
+                        EnableFacialData = isLipSyncActive,
+                        FaceModel = faceModel
+                    }
+                }
+            };
+
+            if (isActionActive)
+                getResponseConfigRequest.GetResponseConfig.ActionConfig = actionConfig;
+
+            return getResponseConfigRequest;
+        }
+
+        /// <summary>
+        ///     Receives a response from the server asynchronously.
+        /// </summary>
+        /// <param name="call">The AsyncDuplexStreamingCall to use for receiving the response.</param>
+        /// <param name="cancellationToken">The cancellation token to use for cancelling the operation.</param>
+        /// <returns>A task that represents the asynchronous operation.</returns>
+        private async Task ReceiveResultFromServer(AsyncDuplexStreamingCall<GetResponseRequest, GetResponseResponse> call, CancellationToken cancellationToken)
+        {
+            Logger.Info("Receiving response from server", Logger.LogCategory.Character);
+            ConvaiNPC convaiNPC = _npcGroup.CurrentSpeaker.ConvaiNPC;
+            _npcGroup.CurrentSpeaker.CanRelayMessage = true;
+            Queue<LipSyncBlendFrameData> lipSyncBlendFrameQueue = new Queue<LipSyncBlendFrameData>();
+            bool firstSilFound = false;
+            while (!cancellationToken.IsCancellationRequested && await call.ResponseStream.MoveNext(cancellationToken).ConfigureAwait(false))
+                try
+                {
+                    GetResponseResponse result = call.ResponseStream.Current;
+                    // Process the received response here
+                    if (result.AudioResponse != null)
+                    {
+                        if (result.AudioResponse.AudioData != null)
+                        // Add response to the list in the active NPC
+                        {
+                            if (result.AudioResponse.AudioData.ToByteArray().Length > 46)
+                            {
+                                byte[] wavBytes = result.AudioResponse.AudioData.ToByteArray();
+
+                                // will only work for wav files
+                                WavHeaderParser parser = new(wavBytes);
+                                if (convaiNPC.convaiLipSync == null)
+                                {
+                                    Logger.DebugLog($"Enqueuing responses: {result.AudioResponse.TextData}", Logger.LogCategory.LipSync);
+                                    convaiNPC.EnqueueResponse(result);
+                                }
+                                else
+                                {
+                                    LipSyncBlendFrameData.FrameType frameType =
+                                        convaiNPC.convaiLipSync.faceModel == FaceModel.OvrModelName
+                                            ? LipSyncBlendFrameData.FrameType.Visemes
+                                            : LipSyncBlendFrameData.FrameType.Blendshape;
+                                    lipSyncBlendFrameQueue.Enqueue(
+                                        new LipSyncBlendFrameData(
+                                            (int)(parser.CalculateDurationSeconds() * 30),
+                                            result,
+                                            frameType
+                                        )
+                                    );
+                                }
+                            }
+
+                            if (result.AudioResponse.VisemesData != null)
+                            {
+                                if (convaiNPC.convaiLipSync != null)
+                                {
+                                    //Logger.Info(result.AudioResponse.VisemesData, Logger.LogCategory.LipSync);
+
+                                    if (result.AudioResponse.VisemesData.Visemes.Sil == -2 || result.AudioResponse.EndOfResponse)
+                                    {
+                                        if (firstSilFound)
+                                        {
+                                            lipSyncBlendFrameQueue.Dequeue().Process(convaiNPC);
+                                        }
+                                        firstSilFound = true;
+                                    }
+                                    else
+                                        lipSyncBlendFrameQueue.Peek().Enqueue(result.AudioResponse.VisemesData);
+                                }
+                            }
+
+                            if (result.AudioResponse.BlendshapesFrame != null)
+                            {
+                                if (convaiNPC.convaiLipSync != null)
+                                {
+                                    if (lipSyncBlendFrameQueue.Peek().CanProcess() || result.AudioResponse.EndOfResponse)
+                                    {
+                                        lipSyncBlendFrameQueue.Dequeue().Process(convaiNPC);
+                                    }
+                                    else
+                                    {
+                                        lipSyncBlendFrameQueue.Peek().Enqueue(result.AudioResponse.BlendshapesFrame);
+
+                                        if (lipSyncBlendFrameQueue.Peek().CanPartiallyProcess())
+                                        {
+                                            lipSyncBlendFrameQueue.Peek().ProcessPartially(convaiNPC);
+                                        }
+                                    }
+                                }
+                            }
+
+                            if (result.AudioResponse.EndOfResponse)
+                            {
+                                MainThreadDispatcher.Instance.RunOnMainThread(() =>
+                                {
+                                    _npcGroup.CurrentSpeaker.EndOfResponseReceived();
+                                });
+                            }
+                        }
+                    }
+                }
+                catch (RpcException rpcException)
+                {
+                    if (rpcException.StatusCode == StatusCode.Cancelled)
+                        Logger.Error(rpcException, Logger.LogCategory.Character);
+                    else
+                        throw;
+                }
+                catch (Exception ex)
+                {
+                    Logger.DebugLog(ex, Logger.LogCategory.Character);
+                }
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPC2NPCGRPCClient.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: dc8dba94e7504aab878f30a19a55fa5a
+timeCreated: 1708531595

Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPCGroup.cs

@@ -0,0 +1,71 @@
+using System;
+using Convai.Scripts;
+using Convai.Scripts.Utils;
+using UnityEngine;
+
+/// <summary>
+///     A group of NPCs that are currently conversing with each other.
+/// </summary>
+[Serializable]
+public class NPCGroup
+{
+    [field: SerializeField] public ConvaiGroupNPCController GroupNPC1 { get; private set; }
+    [field: SerializeField] public ConvaiGroupNPCController GroupNPC2 { get; private set; }
+    public string topic;
+    [ReadOnly] public string messageToRelay;
+
+    public ConvaiGroupNPCController CurrentSpeaker { get; set; }
+    public ConvaiGroupNPCController CurrentListener => CurrentSpeaker == GroupNPC1 ? GroupNPC2 : GroupNPC1;
+
+    private bool _isPlayerNearGroup;
+    private Action<bool, ConvaiGroupNPCController> _vicinityChangedCallback;
+
+    public void Initialize(Action<bool, ConvaiGroupNPCController> vicinityChangedCallback)
+    {
+        _vicinityChangedCallback = vicinityChangedCallback;
+        GroupNPC1.OnPlayerVicinityChanged += HandleVicinity;
+        GroupNPC1.OnPlayerVicinityChanged += HandleVicinity;
+    }
+
+    ~NPCGroup()
+    {
+        GroupNPC1.OnPlayerVicinityChanged -= HandleVicinity;
+        GroupNPC1.OnPlayerVicinityChanged -= HandleVicinity;
+    }
+
+    private void HandleVicinity(bool isPlayerNear, ConvaiGroupNPCController npc)
+    {
+        if (isPlayerNear && !_isPlayerNearGroup)
+        {
+            _isPlayerNearGroup = true;
+            _vicinityChangedCallback?.Invoke(true, npc);
+        }
+
+        if (!isPlayerNear && _isPlayerNearGroup)
+        {
+            _isPlayerNearGroup = false;
+            _vicinityChangedCallback?.Invoke(false, npc);
+        }
+    }
+
+    public bool IsAnyoneTalking()
+    {
+        return GroupNPC1.ConvaiNPC.IsCharacterTalking || GroupNPC2.ConvaiNPC.IsCharacterTalking;
+    }
+
+    public bool BelongToGroup(ConvaiGroupNPCController controller)
+    {
+        return controller.CharacterID == GroupNPC1.CharacterID || controller.CharacterID == GroupNPC2.CharacterID;
+    }
+
+    public bool BothNPCAreNotNull()
+    {
+        return GroupNPC1 != null && GroupNPC2 != null;
+    }
+    public bool BothNPCAreNotActiveNPC()
+    {
+        ConvaiNPC activeNPC = ConvaiNPCManager.Instance.activeConvaiNPC;
+        string activeNPCId = activeNPC != null ? activeNPC.characterID : string.Empty;
+        return !GroupNPC1.CharacterID.Equals(activeNPCId) && !GroupNPC2.CharacterID.Equals(activeNPCId);
+    }
+}

Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPCGroup.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 6c1f1924c9c44d79a1c9abfaeb720f83
+timeCreated: 1713324738

Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 0c6246458db44b939c322f228563602e
+timeCreated: 1712719472

Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/ConvaiSpeechBubbleController.cs

@@ -0,0 +1,33 @@
+using UnityEngine;
+
+public class ConvaiSpeechBubbleController : MonoBehaviour
+{
+    ConvaiGroupNPCController _convaiGroupNPC;
+    NPCSpeechBubble _speechBubble;
+    public void Initialize(NPCSpeechBubble speechBubbleDisplay, ConvaiGroupNPCController convaiGroupNPC)
+    {
+        if (_speechBubble != null) return;
+        _speechBubble = Instantiate(speechBubbleDisplay, transform);
+        _convaiGroupNPC = convaiGroupNPC;
+        _convaiGroupNPC.ShowSpeechBubble += ConvaiNPC_ShowSpeechBubble;
+        _convaiGroupNPC.HideSpeechBubble += ConvaiNPC_HideSpeechBubble;
+    }
+
+    private void ConvaiNPC_HideSpeechBubble()
+    {
+        _speechBubble.HideSpeechBubble();
+    }
+
+    private void ConvaiNPC_ShowSpeechBubble(string text)
+    {
+        _speechBubble.ShowSpeechBubble(text);
+    }
+
+    void OnDestroy()
+    {
+        _convaiGroupNPC.ShowSpeechBubble -= ConvaiNPC_ShowSpeechBubble;
+        _convaiGroupNPC.HideSpeechBubble -= ConvaiNPC_HideSpeechBubble;
+        Destroy(_speechBubble.gameObject);
+        _speechBubble = null;
+    }
+}

Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/ConvaiSpeechBubbleController.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: e4c8a9e9735c471493ad93d4f677e7de
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/ISpeechBubbleDisplay.cs

@@ -0,0 +1,8 @@
+/// <summary>
+///     Interface for displaying speech bubbles.
+/// </summary>
+public interface ISpeechBubbleDisplay
+{
+    void ShowSpeechBubble(string text);
+    void HideSpeechBubble();
+}

Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/ISpeechBubbleDisplay.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 6c1403fe2d584d4abbb9e6588b8035f7
+timeCreated: 1712719388

Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/NPCSpeechBubble.cs

@@ -0,0 +1,26 @@
+using TMPro;
+using UnityEngine;
+
+public class NPCSpeechBubble : MonoBehaviour, ISpeechBubbleDisplay
+{
+    [SerializeField] private TMP_Text speechBubbleText;
+    [SerializeField] private Canvas speechBubbleCanvas;
+
+    /// <summary>
+    ///     Show the speech bubble with the given text.
+    /// </summary>
+    /// <param name="text"> The text to display in the speech bubble. </param>
+    public void ShowSpeechBubble(string text)
+    {
+        speechBubbleText.text = text;
+        speechBubbleCanvas.enabled = true;
+    }
+
+    /// <summary>
+    ///     Hide the speech bubble.
+    /// </summary>
+    public void HideSpeechBubble()
+    {
+        speechBubbleCanvas.enabled = false;
+    }
+}

Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/NPCSpeechBubble.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 8a26c5b33822429bb443f94d041a3374
+timeCreated: 1712719408

Assets/Convai/Scripts/Runtime/Features/NarrativeDesign.meta

@@ -0,0 +1,12 @@
+fileFormatVersion: 2
+<<<<<<<< HEAD:Assets/Convai/Scripts/Narrative Design.meta
+guid: 0e7719b16b26ad348a9cd33d522adf6e
+folderAsset: yes
+========
+guid: 3e8de7ddca083724bb9b286f52bbbc65
+>>>>>>>> january-24-nightly:Assets/Convai/Convai Custom Unity Packages/ConvaiiOSBuild.unitypackage.meta
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Models.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: b390e5b7c8d51a44fa4f560d93e8067f
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Models/SectionChangeEventsData.cs

@@ -0,0 +1,53 @@
+using System;
+using UnityEngine;
+using UnityEngine.Events;
+
+namespace Convai.Scripts.Narrative_Design.Models
+{
+    /// <summary>
+    ///     Data class for Section Change Events
+    /// </summary>
+    [Serializable]
+    public class SectionChangeEventsData
+    {
+        [SerializeField] public string id;
+        [SerializeField] public UnityEvent onSectionStart;
+        [SerializeField] public UnityEvent onSectionEnd;
+        private NarrativeDesignManager _manager;
+
+        private string SectionName
+        {
+            get
+            {
+                if (_manager == null) return string.Empty;
+                SectionData sectionData = _manager.sectionDataList.Find(s => s.sectionId == id);
+                return sectionData?.sectionName ?? "Unknown Section";
+            }
+        }
+
+        /// <summary>
+        ///     Initialize the Section Change Events
+        /// </summary>
+        /// <param name="manager"> The Narrative Design Manager </param>
+        public void Initialize(NarrativeDesignManager manager)
+        {
+            _manager = manager;
+
+            onSectionStart.RemoveListener(LogSectionStart);
+            onSectionStart.AddListener(LogSectionStart);
+
+            onSectionEnd.RemoveListener(LogSectionEnd);
+            onSectionEnd.AddListener(LogSectionEnd);
+        }
+
+        private void LogSectionStart()
+        {
+            Debug.Log($"Section {SectionName} started");
+        }
+
+        private void LogSectionEnd()
+        {
+            Debug.Log($"Section {SectionName} ended");
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Models/SectionChangeEventsData.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 32b4f9b2242945c19917f76c7ceb0a71
+timeCreated: 1707390195

Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Models/SectionData.cs

@@ -0,0 +1,37 @@
+using System;
+using Newtonsoft.Json;
+using UnityEngine;
+
+namespace Convai.Scripts.Narrative_Design.Models
+{
+    /// <summary>
+    ///     Data class for Section Data
+    /// </summary>
+    [Serializable]
+    public class SectionData
+    {
+        [JsonProperty("section_id")] [ReadOnly] [SerializeField]
+        public string sectionId;
+
+        [JsonProperty("section_name")] [ReadOnly] [SerializeField]
+        public string sectionName;
+
+        [JsonProperty("bt_constants")] [HideInInspector] [SerializeField]
+        public string behaviorTreeConstants;
+
+        [JsonProperty("objective")] [ReadOnly] [SerializeField]
+        public string objective;
+
+        [JsonProperty("character_id")] [ReadOnly] [HideInInspector] [SerializeField]
+        public string characterId;
+
+        [JsonProperty("decisions")] [ReadOnly] public object Decisions;
+
+        [JsonProperty("parents")] [ReadOnly] public object Parents;
+
+        [JsonProperty("triggers")] [ReadOnly] public object Triggers;
+
+        [JsonProperty("updated_character_data")] [ReadOnly]
+        public object UpdatedCharacterData;
+    }
+}

Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Models/SectionData.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 98ee0e1dea8bb8c4ba9fe2daf30e9960
+timeCreated: 1706764030

Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Models/TriggerData.cs

@@ -0,0 +1,25 @@
+using System;
+using Newtonsoft.Json;
+using UnityEngine;
+
+namespace Convai.Scripts.Narrative_Design.Models
+{
+    [Serializable]
+    public class TriggerData
+    {
+        [JsonProperty("trigger_id")] [ReadOnly] [SerializeField]
+        public string triggerId;
+
+        [JsonProperty("trigger_name")] [ReadOnly] [SerializeField]
+        public string triggerName;
+
+        [JsonProperty("trigger_message")] [ReadOnly] [SerializeField]
+        public string triggerMessage;
+
+        [JsonProperty("destination_section")] [ReadOnly] [SerializeField]
+        public string destinationSection;
+
+        [JsonProperty("character_id")] [HideInInspector] [SerializeField]
+        public string characterId;
+    }
+}

Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Models/TriggerData.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: ed95a107485d00746a551222f53d0950
+timeCreated: 1706764014

Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Runtime.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: ecfb3b2eea944946a1d2336205b5eaad
+timeCreated: 1706853590

Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Runtime/NarrativeDesignAPI.cs

@@ -0,0 +1,150 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Text;
+using System.Threading.Tasks;
+using Convai.Scripts.Runtime.Utils;
+using Newtonsoft.Json;
+using UnityEngine;
+
+/// <summary>
+///     API client for the Narrative Design API.
+/// </summary>
+public class NarrativeDesignAPI
+{
+    private const string BASE_URL = "https://api.convai.com/character/narrative/";
+    private readonly HttpClient _httpClient;
+
+    /// <summary>
+    ///     Initializes a new instance of the <see cref="NarrativeDesignAPI" /> class.
+    /// </summary>
+    public NarrativeDesignAPI()
+    {
+        _httpClient = new HttpClient
+        {
+            // Set a default request timeout if needed
+            Timeout = TimeSpan.FromSeconds(30) // Example: 30 seconds
+        };
+
+        // Get the API key from the ConvaiAPIKeySetup object
+        if (ConvaiAPIKeySetup.GetAPIKey(out string apiKey))
+        {
+            // Set default request headers here
+            _httpClient.DefaultRequestHeaders.Add("CONVAI-API-KEY", apiKey);
+
+            // Set default headers like Accept to expect a JSON response
+            _httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+        }
+
+    }
+
+    public async Task<string> CreateSectionAsync(string characterId, string objective, string sectionName, string behaviorTreeCode = null, string btConstants = null)
+    {
+        string endpoint = "create-section";
+        HttpContent content = CreateHttpContent(new Dictionary<string, object>
+        {
+            { "character_id", characterId },
+            { "objective", objective },
+            { "section_name", sectionName },
+            { "behavior_tree_code", behaviorTreeCode },
+            { "bt_constants", btConstants }
+        });
+        return await SendPostRequestAsync(endpoint, content);
+    }
+
+    public async Task<string> GetSectionAsync(string characterId, string sectionId)
+    {
+        string endpoint = "get-section";
+        HttpContent content = CreateHttpContent(new Dictionary<string, object>
+        {
+            { "character_id", characterId },
+            { "section_id", sectionId }
+        });
+        return await SendPostRequestAsync(endpoint, content);
+    }
+
+    /// <summary>
+    ///     Get a list of sections for a character.
+    /// </summary>
+    /// <param name="characterId"> The character ID. </param>
+    /// <returns> A JSON string containing the list of sections. </returns>
+    public async Task<string> ListSectionsAsync(string characterId)
+    {
+        string endpoint = "list-sections";
+        HttpContent content = CreateHttpContent(new Dictionary<string, object>
+        {
+            { "character_id", characterId }
+        });
+        return await SendPostRequestAsync(endpoint, content);
+    }
+
+    public async Task<string> CreateTriggerAsync(string characterId, string triggerName, string triggerMessage = null, string destinationSection = null)
+    {
+        string endpoint = "create-trigger";
+
+        HttpContent content = CreateHttpContent(new Dictionary<string, object>
+        {
+            { "character_id", characterId },
+            { "trigger_message", triggerMessage },
+            { "destination_section", destinationSection }
+        });
+
+        return await SendPostRequestAsync(endpoint, content);
+    }
+
+    public async Task<string> GetTriggerAsync(string characterId, string triggerId)
+    {
+        string endpoint = "get-trigger";
+        HttpContent content = CreateHttpContent(new Dictionary<string, object>
+        {
+            { "character_id", characterId },
+            { "trigger_id", triggerId }
+        });
+        return await SendPostRequestAsync(endpoint, content);
+    }
+
+    /// <summary>
+    ///     Get a list of triggers for a character.
+    /// </summary>
+    /// <param name="characterId"> The character ID. </param>
+    /// <returns> A JSON string containing the list of triggers. </returns>
+    public async Task<string> GetTriggerListAsync(string characterId)
+    {
+        string endpoint = "list-triggers";
+        HttpContent content = CreateHttpContent(new Dictionary<string, object>
+        {
+            { "character_id", characterId }
+        });
+        return await SendPostRequestAsync(endpoint, content);
+    }
+
+    private static HttpContent CreateHttpContent(Dictionary<string, object> data)
+    {
+        //Dictionary where all values are not null
+        Dictionary<string, object> dataToSend =
+            data.Where(keyValuePair => keyValuePair.Value != null).ToDictionary(keyValuePair => keyValuePair.Key, keyValuePair => keyValuePair.Value);
+
+        // Serialize the dictionary to JSON
+        string json = JsonConvert.SerializeObject(dataToSend);
+
+        // Convert JSON to HttpContent
+        return new StringContent(json, Encoding.UTF8, "application/json");
+    }
+
+    private async Task<string> SendPostRequestAsync(string endpoint, HttpContent content)
+    {
+        try
+        {
+            HttpResponseMessage response = await _httpClient.PostAsync(BASE_URL + endpoint, content);
+            response.EnsureSuccessStatusCode();
+            return await response.Content.ReadAsStringAsync();
+        }
+        catch (HttpRequestException e)
+        {
+            Debug.LogError($"Request to {endpoint} failed: {e.Message}");
+            return null;
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Runtime/NarrativeDesignAPI.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 13850aaec3d742c2867a15ab2273a1e0
+timeCreated: 1706546525

Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Runtime/NarrativeDesignManager.cs

@@ -0,0 +1,140 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Convai.Scripts;
+using Convai.Scripts.Narrative_Design.Models;
+using Newtonsoft.Json;
+using UnityEngine;
+using Logger = Convai.Scripts.Utils.Logger;
+
+/// <summary>
+///     Manages the narrative design for a ConvaiNPC.
+/// </summary>
+[RequireComponent(typeof(ConvaiNPC))]
+public class NarrativeDesignManager : MonoBehaviour
+{
+    public List<SectionChangeEventsData> sectionChangeEventsDataList = new();
+    public List<SectionData> sectionDataList = new();
+    public List<TriggerData> triggerDataList = new();
+    private ConvaiNPC _convaiNpc;
+    private string _currentSectionID;
+    private NarrativeDesignAPI _narrativeDesignAPI;
+
+    private NarrativeDesignAPI NarrativeDesignAPI => _narrativeDesignAPI ??= new NarrativeDesignAPI();
+    private ConvaiNPC ConvaiNpc => _convaiNpc ??= GetComponent<ConvaiNPC>();
+    private string CharacterID => ConvaiNpc.characterID;
+
+    private async void Awake()
+    {
+        _convaiNpc = GetComponent<ConvaiNPC>();
+        await Task.WhenAll(UpdateSectionListAsync(), UpdateTriggerListAsync());
+    }
+
+    private async void Reset() => await Task.WhenAll(UpdateSectionListAsync(), UpdateTriggerListAsync());
+
+    /// <summary>
+    ///     Updates the section list from the server.
+    /// </summary>
+    public async Task UpdateSectionListAsync()
+    {
+        List<SectionData> updatedSectionList = await GetSectionListFromServerAsync();
+        UpdateSectionDataList(updatedSectionList);
+    }
+
+    /// <summary>
+    ///     Updates the trigger list from the server.
+    /// </summary>
+    public async Task UpdateTriggerListAsync() => await ListTriggersAsync(CharacterID);
+
+    /// <summary>
+    ///     Invoked when the section event list changes.
+    /// </summary>
+    public void OnSectionEventListChange()
+    {
+        foreach (SectionChangeEventsData sectionChangeEventsData in sectionChangeEventsDataList) sectionChangeEventsData.Initialize(this);
+    }
+
+    private async Task<List<SectionData>> GetSectionListFromServerAsync()
+    {
+        try
+        {
+            string sections = await NarrativeDesignAPI.ListSectionsAsync(CharacterID);
+            return JsonConvert.DeserializeObject<List<SectionData>>(sections);
+        }
+        catch (Exception e)
+        {
+            Logger.Error($"Please setup API Key properly. FormatException occurred: {e.Message}", Logger.LogCategory.Character);
+            throw;
+        }
+    }
+
+    private async Task ListTriggersAsync(string characterId)
+    {
+        try
+        {
+            string triggers = await NarrativeDesignAPI.GetTriggerListAsync(characterId);
+            triggerDataList = JsonConvert.DeserializeObject<List<TriggerData>>(triggers);
+        }
+        catch (FormatException e)
+        {
+            Debug.LogError($"Format Exception occurred: {e.Message}");
+            throw;
+        }
+    }
+
+
+    /// <summary>
+    ///     Updates the current section.
+    /// </summary>
+    /// <param name="sectionID"> The section ID to update to. </param>
+    public void UpdateCurrentSection(string sectionID)
+    {
+        if (string.IsNullOrEmpty(_currentSectionID))
+        {
+            _currentSectionID = sectionID;
+            InvokeSectionEvent(_currentSectionID, true);
+            return;
+        }
+
+        if (_currentSectionID.Equals(sectionID))
+            return;
+
+        InvokeSectionEvent(_currentSectionID, false);
+        _currentSectionID = sectionID;
+        InvokeSectionEvent(_currentSectionID, true);
+    }
+
+    private void InvokeSectionEvent(string id, bool isStarting)
+    {
+        SectionChangeEventsData sectionChangeEventsData = sectionChangeEventsDataList.Find(x => x.id == id);
+
+        if (sectionChangeEventsData == null)
+        {
+            Logger.Info($"No Section Change Events have been created for sectionID: {id}", Logger.LogCategory.Actions);
+            return;
+        }
+
+        if (isStarting)
+            sectionChangeEventsData.onSectionStart?.Invoke();
+        else
+            sectionChangeEventsData.onSectionEnd?.Invoke();
+    }
+
+    private void UpdateSectionDataList(List<SectionData> updatedSectionList)
+    {
+        Dictionary<string, SectionData> updatedSectionDictionary = updatedSectionList.ToDictionary(s => s.sectionId);
+
+        foreach (SectionData currentSection in sectionDataList)
+            if (updatedSectionDictionary.TryGetValue(currentSection.sectionId, out SectionData updatedSection))
+            {
+                currentSection.sectionName = updatedSection.sectionName;
+                currentSection.objective = updatedSection.objective;
+                updatedSectionDictionary.Remove(currentSection.sectionId);
+            }
+
+        foreach (SectionData newSection in updatedSectionDictionary.Values) sectionDataList.Add(newSection);
+
+        foreach (SectionChangeEventsData sectionChangeEvent in sectionChangeEventsDataList) sectionChangeEvent.Initialize(this);
+    }
+}

Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Runtime/NarrativeDesignManager.cs.meta

@@ -0,0 +1,3 @@
+fileFormatVersion: 2
+guid: 3d56bc33052944e0aba5a0fe7073c3ab
+timeCreated: 1706546743

Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Runtime/NarrativeDesignTrigger.cs

@@ -0,0 +1,51 @@
+using System.Collections.Generic;
+using System.Linq;
+using Convai.Scripts.Utils;
+using UnityEngine;
+using UnityEngine.Events;
+
+namespace Convai.Scripts.Narrative_Design
+{
+    public class NarrativeDesignTrigger : MonoBehaviour
+    {
+        public ConvaiNPC convaiNPC;
+        [HideInInspector] public int selectedTriggerIndex;
+        [HideInInspector] public List<string> availableTriggers;
+        public UnityEvent onTriggerEvent;
+
+        private void Awake()
+        {
+            onTriggerEvent.AddListener(InvokeSelectedTrigger);
+        }
+
+        private void OnTriggerEnter(Collider other)
+        {
+            if (other.CompareTag("Player")) InvokeSelectedTrigger();
+        }
+
+
+        private void OnValidate()
+        {
+            availableTriggers = null;
+
+            if (convaiNPC != null)
+            {
+                NarrativeDesignManager narrativeDesignManager = convaiNPC.GetComponent<NarrativeDesignManager>();
+                if (narrativeDesignManager != null) availableTriggers = narrativeDesignManager.triggerDataList.Select(trigger => trigger.triggerName).ToList();
+            }
+        }
+
+        /// <summary>
+        ///   Invokes the selected trigger.
+        /// </summary>
+        public void InvokeSelectedTrigger()
+        {
+            if (convaiNPC != null && availableTriggers != null && selectedTriggerIndex >= 0 && selectedTriggerIndex < availableTriggers.Count)
+            {
+                string selectedTriggerName = availableTriggers[selectedTriggerIndex];
+                ConvaiNPCManager.Instance.SetActiveConvaiNPC(convaiNPC);
+                convaiNPC.TriggerEvent(selectedTriggerName);
+            }
+        }
+    }
+}

Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Runtime/NarrativeDesignTrigger.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 1c783ed73e64a204ebf38b93c054b566
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: