tomh104/Virtual-Tutor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial upload

Browse Source
This commit is contained in:
tom.hempel
2025-09-30 17:58:33 +02:00
commit 69b0c79692
4818 changed files with 229318 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
Assets/Convai/Scripts/Runtime/Features/Actions.meta Normal file
View File

@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: f7ee91cfe70e2af439a2ed9b7557855d
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

787
Assets/Convai/Scripts/Runtime/Features/Actions/ConvaiActionsHandler.cs Normal file
View File

@ -0,0 +1,787 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using Convai.Scripts.Runtime.Core;
using Convai.Scripts.Runtime.LoggerSystem;
using Service;
using UnityEngine;
using UnityEngine.AI;
using UnityEngine.Serialization;
namespace Convai.Scripts.Runtime.Features
{
// STEP 1: Add the enum for your custom action here.
public enum ActionChoice
{
None,
Jump,
Crouch,
MoveTo,
PickUp,
Drop
}
/// <summary>
/// DISCLAIMER: The action API is in experimental stages and can misbehave. Meanwhile, feel free to try it out and play
/// around with it.
/// </summary>
[DisallowMultipleComponent]
[AddComponentMenu("Convai/Convai Actions Handler")]
public class ConvaiActionsHandler : MonoBehaviour
{
[SerializeField] public ActionMethod[] actionMethods;
public List<string> actionResponseList = new();
private readonly List<ConvaiAction> _actionList = new();
public readonly ActionConfig ActionConfig = new();
private List<string> _actions = new();
private ConvaiNPC _currentNPC;
private ConvaiInteractablesData _interactablesData;
private Coroutine _playActionListCoroutine;
// Awake is called when the script instance is being loaded
private void Awake()
{
// Find the global action settings object in the scene
_interactablesData = FindObjectOfType<ConvaiInteractablesData>();
// Check if the global action settings object is missing
if (_interactablesData == null)
// Log an error message to indicate missing Convai Action Settings
ConvaiLogger.Error("Convai Action Settings missing. Please create a game object that handles actions.",
ConvaiLogger.LogCategory.Character);
// Check if this GameObject has a ConvaiNPC component attached
if (TryGetComponent(out ConvaiNPC npc))
// If it does, set the current NPC to this GameObject
_currentNPC = npc;
// Iterate through each action method and add its name to the action configuration
foreach (ActionMethod actionMethod in actionMethods) ActionConfig.Actions.Add(actionMethod.action);
if (_interactablesData != null)
{
// Iterate through each character in global action settings and add them to the action configuration
foreach (ConvaiInteractablesData.Character character in _interactablesData.Characters)
{
ActionConfig.Types.Character rpcCharacter = new()
{
Name = character.Name,
Bio = character.Bio
};
ActionConfig.Characters.Add(rpcCharacter);
}
// Iterate through each object in global action settings and add them to the action configuration
foreach (ConvaiInteractablesData.Object eachObject in _interactablesData.Objects)
{
ActionConfig.Types.Object rpcObject = new()
{
Name = eachObject.Name,
Description = eachObject.Description
};
ActionConfig.Objects.Add(rpcObject);
}
}
}
private void Reset()
{
actionMethods = new ActionMethod[]
{
new() { action = "Move To", actionChoice = ActionChoice.MoveTo },
new() { action = "Pick Up", actionChoice = ActionChoice.PickUp },
new() { action = "Dance", animationName = "Dance", actionChoice = ActionChoice.None },
new() { action = "Drop", actionChoice = ActionChoice.Drop },
new() { action = "Jump", actionChoice = ActionChoice.Jump }
};
}
// Start is called before the first frame update
private void Start()
{
// Set up the action configuration
#region Actions Setup
// Set the classification of the action configuration to "multistep"
ActionConfig.Classification = "multistep";
// Log the configured action information
ConvaiLogger.DebugLog(ActionConfig, ConvaiLogger.LogCategory.Actions);
#endregion
// Start playing the action list using a coroutine
_playActionListCoroutine = StartCoroutine(PlayActionList());
}
private void OnEnable() {
if ( _playActionListCoroutine != null ) {
_playActionListCoroutine = StartCoroutine(PlayActionList());
}
}
private void OnDisable() {
if ( _playActionListCoroutine != null ) {
StopCoroutine(_playActionListCoroutine);
}
}
private void Update()
{
if (actionResponseList.Count > 0)
{
ParseActions(actionResponseList[0]);
actionResponseList.RemoveAt(0);
}
}
private void ParseActions(string actionsString)
{
actionsString = actionsString.Trim();
ConvaiLogger.DebugLog($"Parsing actions from: {actionsString}", ConvaiLogger.LogCategory.Actions);
_actions = actionsString.Split(", ").ToList();
_actionList.Clear();
foreach (string action in _actions)
{
List<string> actionWords = action.Split(' ').ToList();
ConvaiLogger.Info($"Processing action: {action}", ConvaiLogger.LogCategory.Actions);
ParseSingleAction(actionWords);
}
}
/// <summary>
/// Parses a single action from a list of action words.
/// </summary>
/// <param name="actionWords">The list of words representing the action.</param>
private void ParseSingleAction(List<string> actionWords)
{
for (int j = 0; j < actionWords.Count; j++)
{
// Split the action into verb and object parts
string[] verbPart = actionWords.Take(j + 1).ToArray();
string[] objectPart = actionWords.Skip(j + 1).ToArray();
// Remove trailing 's' from verb words
verbPart = verbPart.Select(word => word.TrimEnd('s')).ToArray();
string actionString = string.Join(" ", verbPart);
// Find the best matching action using Levenshtein distance
ActionMethod matchingAction = actionMethods
.OrderBy(a => LevenshteinDistance(a.action.ToLower(), actionString.ToLower()))
.FirstOrDefault();
// If no close match is found, continue to the next iteration
if (matchingAction == null || LevenshteinDistance(matchingAction.action.ToLower(), actionString.ToLower()) > 2) continue;
// Find the target object for the action
GameObject targetObject = FindTargetObject(objectPart);
LogActionResult(verbPart, objectPart, targetObject);
// Add the parsed action to the action list
_actionList.Add(new ConvaiAction(matchingAction.actionChoice, targetObject, matchingAction.animationName));
break;
}
}
/// <summary>
/// Finds the target object based on the object part of the action.
/// </summary>
/// <param name="objectPart">The array of words representing the object.</param>
/// <returns>The GameObject that best matches the object description, or null if no match is found.</returns>
private GameObject FindTargetObject(string[] objectPart)
{
string targetName = string.Join(" ", objectPart);
// Try to find a matching object
ConvaiInteractablesData.Object obj = _interactablesData.Objects
.OrderBy(o => LevenshteinDistance(o.Name.ToLower(), targetName.ToLower()))
.FirstOrDefault();
if (obj != null && LevenshteinDistance(obj.Name.ToLower(), targetName.ToLower()) <= 2)
return obj.gameObject;
// If no object is found, try to find a matching character
ConvaiInteractablesData.Character character = _interactablesData.Characters
.OrderBy(c => LevenshteinDistance(c.Name.ToLower(), targetName.ToLower()))
.FirstOrDefault();
if (character != null && LevenshteinDistance(character.Name.ToLower(), targetName.ToLower()) <= 2)
return character.gameObject;
return null;
}
/// <summary>
/// Calculates the Levenshtein distance between two strings.
/// </summary>
/// <param name="s">The first string.</param>
/// <param name="t">The second string.</param>
/// <returns>The Levenshtein distance between the two strings.</returns>
private int LevenshteinDistance(string s, string t)
{
int[][] d = new int[s.Length + 1][];
for (int index = 0; index < s.Length + 1; index++) d[index] = new int[t.Length + 1];
// Initialize the first row and column
for (int i = 0; i <= s.Length; i++)
d[i][0] = i;
for (int j = 0; j <= t.Length; j++)
d[0][j] = j;
// Calculate the distance
for (int j = 1; j <= t.Length; j++)
for (int i = 1; i <= s.Length; i++)
{
int cost = s[i - 1] == t[j - 1] ? 0 : 1;
d[i][j] = Math.Min(Math.Min(d[i - 1][j] + 1, d[i][j - 1] + 1), d[i - 1][j - 1] + cost);
}
return d[s.Length][t.Length];
}
private void LogActionResult(string[] verbPart, string[] objectPart, GameObject targetObject)
{
string verb = string.Join(" ", verbPart).ToLower();
string obj = string.Join(" ", objectPart).ToLower();
if (targetObject != null)
{
ConvaiLogger.DebugLog($"Active Target: {obj}", ConvaiLogger.LogCategory.Actions);
ConvaiLogger.DebugLog($"Found matching target: {targetObject.name} for action: {verb}", ConvaiLogger.LogCategory.Actions);
}
else
{
ConvaiLogger.Warn($"No matching target found for action: {verb}", ConvaiLogger.LogCategory.Actions);
}
}
/// <summary>
/// Event that is triggered when an action starts.
/// </summary>
/// <remarks>
/// This event can be subscribed to in order to perform custom logic when an action starts.
/// The event provides the name of the action and the GameObject that the action is targeting.
/// </remarks>
public event Action<string, GameObject> ActionStarted;
/// <summary>
/// Event that is triggered when an action ends.
/// </summary>
/// <remarks>
/// This event can be subscribed to in order to perform custom logic when an action ends.
/// The event provides the name of the action and the GameObject that the action was targeting.
/// </remarks>
public event Action<string, GameObject> ActionEnded;
/// <summary>
/// This coroutine handles playing the actions in the action list.
/// </summary>
/// <returns></returns>
private IEnumerator PlayActionList()
{
while (true)
// Check if there are actions in the action list
if (_actionList.Count > 0)
{
// Call the DoAction function for the first action in the list and wait until it's done
yield return DoAction(_actionList[0]);
// Remove the completed action from the list
_actionList.RemoveAt(0);
}
else
{
// If there are no actions in the list, yield to wait for the next frame
yield return null;
}
}
private IEnumerator DoAction(ConvaiAction action)
{
// STEP 2: Add the function call for your action here corresponding to your enum.
// Remember to yield until its return if it is an Enumerator function.
// Use a switch statement to handle different action choices based on the ActionChoice enum
switch (action.Verb)
{
case ActionChoice.MoveTo:
// Call the MoveTo function and yield until it's completed
yield return MoveTo(action.Target);
break;
case ActionChoice.PickUp:
// Call the PickUp function and yield until it's completed
yield return PickUp(action.Target);
break;
case ActionChoice.Drop:
// Call the Drop function
Drop(action.Target);
break;
case ActionChoice.Jump:
// Call the Jump function
Jump();
break;
case ActionChoice.Crouch:
// Call the Crouch function and yield until it's completed
yield return Crouch();
break;
case ActionChoice.None:
// Call the AnimationActions function and yield until it's completed
yield return AnimationActions(action.Animation);
break;
default:
throw new ArgumentOutOfRangeException();
}
// Yield once to ensure the coroutine advances to the next frame
yield return null;
}
/// <summary>
/// This method is a coroutine that handles playing an animation for Convai NPC.
/// The method takes in the name of the animation to be played as a string parameter.
/// </summary>
/// <param name="animationName"> The name of the animation to be played. </param>
/// <returns> A coroutine that plays the animation. </returns>
private IEnumerator AnimationActions(string animationName)
{
// Logging the action of initiating the animation with the provided animation name.
ConvaiLogger.DebugLog("Doing animation: " + animationName, ConvaiLogger.LogCategory.Actions);
// Attempting to get the Animator component attached to the current NPC object.
// The Animator component is responsible for controlling animations on the GameObject.
Animator animator = _currentNPC.GetComponent<Animator>();
// Converting the provided animation name to its corresponding hash code.
// This is a more efficient way to refer to animations and Animator states.
int animationHash = Animator.StringToHash(animationName);
// Check if the Animator component has a state with the provided hash code.
// This is a safety check to prevent runtime errors if the animation is not found.
if (!animator.HasState(0, animationHash))
{
// Logging a message to indicate that the animation was not found.
ConvaiLogger.DebugLog("Could not find an animator state named: " + animationName, ConvaiLogger.LogCategory.Actions);
// Exiting the coroutine early since the animation is not available.
yield break;
}
// Playing the animation with a cross-fade transition.
// The second parameter '0.1f' specifies the duration of the cross-fade.
animator.CrossFadeInFixedTime(animationHash, 0.1f);
// Waiting for a short duration (just over the cross-fade time) to allow the animation transition to start.
// This ensures that subsequent code runs after the animation has started playing.
yield return new WaitForSeconds(0.11f);
// Getting information about the current animation clip that is playing.
AnimatorClipInfo[] clipInfo = animator.GetCurrentAnimatorClipInfo(0);
// Checking if there is no animation clip information available.
if (clipInfo == null || clipInfo.Length == 0)
{
// Logging a message to indicate that there are no animation clips associated with the state.
ConvaiLogger.DebugLog("Animator state named: " + animationName + " has no associated animation clips",
ConvaiLogger.LogCategory.Actions);
// Exiting the coroutine as there is no animation to play.
yield break;
}
// Defining variables to store the length and name of the animation clip.
float length = 0;
string animationClipName = "";
// Iterating through the array of animation clips to find the one that is currently playing.
foreach (AnimatorClipInfo clipInf in clipInfo)
{
// Logging the name of the animation clip for debugging purposes.
ConvaiLogger.DebugLog("Clip name: " + clipInf.clip.name, ConvaiLogger.LogCategory.Actions);
// Storing the current animation clip in a local variable for easier access.
AnimationClip clip = clipInf.clip;
// Checking if the animation clip is valid.
if (clip != null)
{
// Storing the length and name of the animation clip.
length = clip.length;
animationClipName = clip.name;
// Exiting the loop as we've found the information we need.
break;
}
}
// Checking if a valid animation clip was found.
if (length > 0.0f)
{
// Logging a message indicating that the animation is now playing.
ConvaiLogger.DebugLog(
"Playing the animation " + animationClipName + " from the Animator State " + animationName +
" for " + length + " seconds", ConvaiLogger.LogCategory.Actions);
// Waiting for the duration of the animation to allow it to play out.
yield return new WaitForSeconds(length);
}
else
{
// Logging a message to indicate that no valid animation clips were found or their length was zero.
ConvaiLogger.DebugLog(
"Animator state named: " + animationName +
" has no valid animation clips or they have a length of 0", ConvaiLogger.LogCategory.Actions);
// Exiting the coroutine early.
yield break;
}
// Transitioning back to the idle animation.
// It is assumed that an "Idle" animation exists and is set up in your Animator Controller.
animator.CrossFadeInFixedTime(Animator.StringToHash("Idle"), 0.1f);
// Yielding to wait for one frame to ensure that the coroutine progresses to the next frame.
// This is often done at the end of a coroutine to prevent issues with Unity's execution order.
yield return null;
}
/// <summary>
/// Registers the provided methods to the ActionStarted and ActionEnded events.
/// This allows external code to subscribe to these events and react when they are triggered.
/// </summary>
/// <param name="onActionStarted">
/// The method to be called when an action starts. It should accept a string (the action
/// name) and a GameObject (the target of the action).
/// </param>
/// <param name="onActionEnded">
/// The method to be called when an action ends. It should accept a string (the action name)
/// and a GameObject (the target of the action).
/// </param>
public void RegisterForActionEvents(Action<string, GameObject> onActionStarted,
Action<string, GameObject> onActionEnded)
{
ActionStarted += onActionStarted;
ActionEnded += onActionEnded;
}
/// <summary>
/// Unregisters the provided methods from the ActionStarted and ActionEnded events.
/// This allows external code to unsubscribe from these events when they are no longer interested in them.
/// </summary>
/// <param name="onActionStarted">
/// The method to be removed from the ActionStarted event. It should be the same method that
/// was previously registered.
/// </param>
/// <param name="onActionEnded">
/// The method to be removed from the ActionEnded event. It should be the same method that was
/// previously registered.
/// </param>
public void UnregisterForActionEvents(Action<string, GameObject> onActionStarted,
Action<string, GameObject> onActionEnded)
{
ActionStarted -= onActionStarted;
ActionEnded -= onActionEnded;
}
[Serializable]
public class ActionMethod
{
[FormerlySerializedAs("Action")] [SerializeField]
public string action;
[SerializeField] public string animationName;
[SerializeField] public ActionChoice actionChoice;
}
private class ConvaiAction
{
public ConvaiAction(ActionChoice verb, GameObject target, string animation)
{
Verb = verb;
Target = target;
Animation = animation;
}
#region 04. Public variables
public readonly string Animation;
public readonly GameObject Target;
public readonly ActionChoice Verb;
#endregion
}
// STEP 3: Add the function for your action here.
#region Action Implementation Methods
private IEnumerator Crouch()
{
ActionStarted?.Invoke("Crouch", _currentNPC.gameObject);
ConvaiLogger.DebugLog("Crouching!", ConvaiLogger.LogCategory.Actions);
Animator animator = _currentNPC.GetComponent<Animator>();
animator.CrossFadeInFixedTime(Animator.StringToHash("Crouch"), 0.1f);
// Wait for the next frame to ensure the Animator has transitioned to the new state.
yield return new WaitForSeconds(0.11f);
AnimatorClipInfo[] clipInfo = animator.GetCurrentAnimatorClipInfo(0);
if (clipInfo == null || clipInfo.Length == 0)
{
ConvaiLogger.DebugLog("No animation clips found for crouch state!", ConvaiLogger.LogCategory.Actions);
yield break;
}
float length = clipInfo[0].clip.length;
_currentNPC.GetComponents<CapsuleCollider>()[0].height = 1.2f;
_currentNPC.GetComponents<CapsuleCollider>()[0].center = new Vector3(0, 0.6f, 0);
if (_currentNPC.GetComponents<CapsuleCollider>().Length > 1)
{
_currentNPC.GetComponents<CapsuleCollider>()[1].height = 1.2f;
_currentNPC.GetComponents<CapsuleCollider>()[1].center = new Vector3(0, 0.6f, 0);
}
yield return new WaitForSeconds(length);
animator.CrossFadeInFixedTime(Animator.StringToHash("Idle"), 0.1f);
yield return null;
ActionEnded?.Invoke("Crouch", _currentNPC.gameObject);
}
private IEnumerator MoveTo(GameObject target)
{
if (!IsTargetValid(target)) yield break;
ConvaiLogger.DebugLog($"Moving to Target: {target.name}", ConvaiLogger.LogCategory.Actions);
ActionStarted?.Invoke("MoveTo", target);
Animator animator = _currentNPC.GetComponent<Animator>();
NavMeshAgent navMeshAgent = _currentNPC.GetComponent<NavMeshAgent>();
SetupAnimationAndNavigation(animator, navMeshAgent);
Vector3 targetDestination = CalculateTargetDestination(target);
navMeshAgent.SetDestination(targetDestination);
yield return null;
yield return MoveTowardsTarget(target, navMeshAgent);
FinishMovement(animator, target);
}
private bool IsTargetValid(GameObject target)
{
if (target == null || !target.activeInHierarchy)
{
ConvaiLogger.DebugLog("MoveTo target is null or inactive.", ConvaiLogger.LogCategory.Actions);
return false;
}
return true;
}
private void SetupAnimationAndNavigation(Animator animator, NavMeshAgent navMeshAgent)
{
animator.CrossFade(Animator.StringToHash("Walking"), 0.01f);
animator.applyRootMotion = false;
navMeshAgent.updateRotation = false;
}
private Vector3 CalculateTargetDestination(GameObject target)
{
Vector3 targetDestination = target.transform.position;
if (target.TryGetComponent(out Renderer rendererComponent))
{
float zOffset = rendererComponent.bounds.size.z;
targetDestination += zOffset * target.transform.forward;
}
else
{
targetDestination += 0.5f * target.transform.forward;
}
return targetDestination;
}
private IEnumerator MoveTowardsTarget(GameObject target, NavMeshAgent navMeshAgent)
{
float rotationSpeed = 5;
while (navMeshAgent.remainingDistance > navMeshAgent.stoppingDistance)
{
if (!target.activeInHierarchy)
{
ConvaiLogger.DebugLog("Target deactivated during movement.", ConvaiLogger.LogCategory.Actions);
yield break;
}
if (navMeshAgent.velocity.sqrMagnitude < Mathf.Epsilon) yield return null;
RotateTowardsMovementDirection(navMeshAgent, rotationSpeed);
yield return null;
}
}
private void RotateTowardsMovementDirection(NavMeshAgent navMeshAgent, float rotationSpeed)
{
Quaternion rotation = Quaternion.LookRotation(navMeshAgent.velocity.normalized);
rotation.x = 0;
rotation.z = 0;
transform.rotation = Quaternion.Slerp(transform.rotation, rotation, rotationSpeed * Time.deltaTime);
}
private void FinishMovement(Animator animator, GameObject target)
{
animator.CrossFade(Animator.StringToHash("Idle"), 0.1f);
if (_actions.Count == 1 && Camera.main != null) StartCoroutine(RotateTowardsCamera());
animator.applyRootMotion = true;
ActionEnded?.Invoke("MoveTo", target);
}
private IEnumerator RotateTowardsCamera()
{
if (Camera.main != null)
{
Vector3 direction = (Camera.main.transform.position - transform.position).normalized;
Quaternion targetRotation = Quaternion.LookRotation(direction);
float elapsedTime = 0f;
float rotationTime = 2f;
while (elapsedTime < rotationTime)
{
targetRotation.x = 0;
targetRotation.z = 0;
transform.rotation = Quaternion.Slerp(transform.rotation, targetRotation, elapsedTime / rotationTime);
elapsedTime += Time.deltaTime;
yield return null;
}
}
}
/// <summary>
/// Coroutine to pick up a target GameObject, adjusting the NPCs' rotation and playing animations.
/// </summary>
/// <param name="target">The target GameObject to pick up.</param>
private IEnumerator PickUp(GameObject target)
{
// Invoke the ActionStarted event with the "PickUp" action and the target GameObject.
ActionStarted?.Invoke("PickUp", target);
// Check if the target GameObject is null. If it is, log an error and exit the coroutine.
if (target == null)
{
ConvaiLogger.DebugLog("Target is null! Exiting PickUp coroutine.", ConvaiLogger.LogCategory.Actions);
yield break;
}
// Check if the target GameObject is active. If not, log an error and exit the coroutine.
if (!target.activeInHierarchy)
{
ConvaiLogger.DebugLog($"Target: {target.name} is inactive! Exiting PickUp coroutine.",
ConvaiLogger.LogCategory.Actions);
yield break;
}
// Calculate the direction from the NPC to the target, ignoring the vertical (y) component.
Vector3 direction = (target.transform.position - transform.position).normalized;
direction.y = 0;
// Calculate the target rotation to face the target direction.
Quaternion targetRotation = Quaternion.LookRotation(direction);
float elapsedTime = 0f;
float rotationTime = 0.5f;
// Smoothly rotate the NPC towards the target direction over a specified time.
while (elapsedTime < rotationTime)
{
targetRotation.x = 0;
targetRotation.z = 0;
transform.rotation = Quaternion.Slerp(transform.rotation, targetRotation, elapsedTime / rotationTime);
elapsedTime += Time.deltaTime;
yield return null;
}
// Log the action of picking up the target along with its name.
ConvaiLogger.DebugLog($"Picking up Target: {target.name}", ConvaiLogger.LogCategory.Actions);
// Retrieve the Animator component from the current NPC.
Animator animator = _currentNPC.GetComponent<Animator>();
// Start the "Picking Up" animation with a cross-fade transition.
animator.CrossFade(Animator.StringToHash("Picking Up"), 0.1f);
// Wait for one second to ensure that the Animator has had time to transition to the "Picking Up" animation state.
yield return new WaitForSeconds(1);
// Define the time it takes for the hand to reach the object in the "Picking Up" animation.
// This is a specific point in time during the animation that we are interested in.
float timeToReachObject = 1f;
// Wait for the time it takes for the hand to reach the object.
yield return new WaitForSeconds(timeToReachObject);
// Check again if the target is still active before attempting to pick it up.
if (!target.activeInHierarchy)
{
ConvaiLogger.DebugLog(
$"Target: {target.name} became inactive during the pick up animation! Exiting PickUp coroutine.",
ConvaiLogger.LogCategory.Actions);
yield break;
}
// Once the hand has reached the object, set the target's parent to the NPCs' transform,
// effectively "picking up" the object, and then deactivate the object.
target.transform.parent = gameObject.transform;
target.SetActive(false);
// Transition back to the "Idle" animation.
animator.CrossFade(Animator.StringToHash("Idle"), 0.4f);
// Invoke the ActionEnded event with the "PickUp" action and the target GameObject.
ActionEnded?.Invoke("PickUp", target);
}
private void Drop(GameObject target)
{
ActionStarted?.Invoke("Drop", target);
if (target == null) return;
ConvaiLogger.DebugLog($"Dropping Target: {target.name}", ConvaiLogger.LogCategory.Actions);
target.transform.parent = null;
target.SetActive(true);
ActionEnded?.Invoke("Drop", target);
}
private void Jump()
{
ActionStarted?.Invoke("Jump", _currentNPC.gameObject);
float jumpForce = 5f;
GetComponent<Rigidbody>().AddForce(new Vector3(0f, jumpForce, 0f), ForceMode.Impulse);
_currentNPC.GetComponent<Animator>().CrossFade(Animator.StringToHash("Dance"), 1);
ActionEnded?.Invoke("Jump", _currentNPC.gameObject);
}
// STEP 3: Add the function for your action here.
#endregion
}
}

11
Assets/Convai/Scripts/Runtime/Features/Actions/ConvaiActionsHandler.cs.meta Normal file
View File

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 744cc0ebc534aa44b8b0871ca72570b9
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

39
Assets/Convai/Scripts/Runtime/Features/Actions/ConvaiInteractablesData.cs Normal file
View File

@ -0,0 +1,39 @@
using System;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
/// <summary>
/// This script defines global actions and settings for Convai.
/// </summary>
[AddComponentMenu("Convai/Convai Interactables Data")]
public class ConvaiInteractablesData : MonoBehaviour
{
[Tooltip("Array of Characters in the environment")] [SerializeField]
public Character[] Characters;
[Tooltip("Array of Objects in the environment")] [SerializeField]
public Object[] Objects;
public Transform DynamicMoveTargetIndicator;
/// <summary>
/// Represents a character in the environment.
/// </summary>
[Serializable]
public class Character
{
[SerializeField] public string Name;
[SerializeField] public string Bio;
[SerializeField] public GameObject gameObject;
}
[Serializable]
public class Object
{
[SerializeField] public string Name;
[SerializeField] public string Description;
[SerializeField] public GameObject gameObject;
}
}
}

11
Assets/Convai/Scripts/Runtime/Features/Actions/ConvaiInteractablesData.cs.meta Normal file
View File

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: b3b04a8e56d4a394585355d57021ba7a
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

3
Assets/Convai/Scripts/Runtime/Features/DynamicInfo.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 2b8ebe81f46c4ee9ba8dc4742b92ca72
timeCreated: 1729098038

30
Assets/Convai/Scripts/Runtime/Features/DynamicInfo/DynamicInfoController.cs Normal file
View File

@ -0,0 +1,30 @@
using Service;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
public class DynamicInfoController : MonoBehaviour
{
public DynamicInfoConfig DynamicInfoConfig { get; private set; }
private void Awake()
{
DynamicInfoConfig = new DynamicInfoConfig();
}
public void SetDynamicInfo(string info)
{
DynamicInfoConfig.Text = info;
}
public void AddDynamicInfo(string info)
{
DynamicInfoConfig.Text += info;
}
public void ClearDynamicInfo()
{
DynamicInfoConfig.Text = "";
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/DynamicInfo/DynamicInfoController.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 05aa3cdd3ee246a0a8e5ca86bcfba893
timeCreated: 1729098061

8
Assets/Convai/Scripts/Runtime/Features/LipSync.meta Normal file
View File

@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: 79e17fbbffadd7f47b7b345e0be9a750
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

112
Assets/Convai/Scripts/Runtime/Features/LipSync/ConvaiLipSync.cs Normal file
View File

@ -0,0 +1,112 @@
using System;
using System.Collections.Generic;
using Convai.Scripts.Runtime.Core;
using Convai.Scripts.Runtime.Extensions;
using Convai.Scripts.Runtime.Features.LipSync.Models;
using Convai.Scripts.Runtime.Features.LipSync.Types;
using Service;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features.LipSync
{
public class ConvaiLipSync : MonoBehaviour
{
[HideInInspector] public FaceModel faceModel = FaceModel.OvrModelName;
[field: SerializeField]
[field: Tooltip("Assign the skin renderers and its respective effectors, along with the bones used for Facial Expression")]
public FacialExpressionData FacialExpressionData { get; private set; } = new();
[field: SerializeField]
[field: Range(0f, 1f)]
[field: Tooltip("This decides how much blending will occur between two different blendshape frames")]
public float WeightBlendingPower { get; private set; } = 0.5f;
[SerializeField] private List<string> characterEmotions;
private ConvaiNPC _convaiNPC;
public ConvaiLipSyncApplicationBase ConvaiLipSyncApplicationBase { get; private set; }
/// <summary>
/// This function will automatically set any of the unassigned skinned mesh renderers to appropriate values using regex
/// based functions.
/// Sets the references of the required variables
/// Sets wait for lipsync to true
/// </summary>
private void Start()
{
FindSkinMeshRenderer();
_convaiNPC = GetComponent<ConvaiNPC>();
ConvaiLipSyncApplicationBase = gameObject.GetOrAddComponent<ConvaiVisemesLipSync>();
ConvaiLipSyncApplicationBase.Initialize(this, _convaiNPC);
SetCharacterLipSyncing(true);
}
private void OnDisable()
{
StopLipSync();
}
private void OnApplicationQuit()
{
StopLipSync();
}
public event Action<bool> OnCharacterLipSyncing;
private void FindSkinMeshRenderer()
{
if (FacialExpressionData.Head.Renderer == null)
FacialExpressionData.Head.Renderer = transform.GetComponentOnChildWithMatchingRegex<SkinnedMeshRenderer>("(.*_Head|CC_Base_Body)");
if (FacialExpressionData.Teeth.Renderer == null)
FacialExpressionData.Teeth.Renderer = transform.GetComponentOnChildWithMatchingRegex<SkinnedMeshRenderer>("(.*_Teeth|CC_Base_Teeth)");
if (FacialExpressionData.Tongue.Renderer == null)
FacialExpressionData.Tongue.Renderer = transform.GetComponentOnChildWithMatchingRegex<SkinnedMeshRenderer>("(.*_Tongue|CC_Base_Tongue)");
}
/// <summary>
/// Overrides the character emotions list
/// </summary>
/// <param name="newEmotions">list of new emotions</param>
public void SetCharacterEmotions(List<string> newEmotions)
{
characterEmotions = new List<string>(newEmotions);
}
/// <summary>
/// Returns Direct reference of the character emotions [Not Recommended to directly change this list]
/// </summary>
/// <returns></returns>
public List<string> GetCharacterEmotions()
{
return characterEmotions;
}
/// <summary>
/// Fires an event with update the Character Lip Syncing State
/// </summary>
/// <param name="value"></param>
private void SetCharacterLipSyncing(bool value)
{
OnCharacterLipSyncing?.Invoke(value);
}
/// <summary>
/// Purges the latest chuck of lipsync frames
/// </summary>
public void PurgeExcessFrames()
{
ConvaiLipSyncApplicationBase?.PurgeExcessBlendShapeFrames();
}
/// <summary>
/// Stops the Lipsync by clearing the frames queue
/// </summary>
public void StopLipSync()
{
ConvaiLipSyncApplicationBase?.ClearQueue();
}
}
}

11
Assets/Convai/Scripts/Runtime/Features/LipSync/ConvaiLipSync.cs.meta Normal file
View File

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 88bce56f6985ef84f8835a0152628fa1
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

126
Assets/Convai/Scripts/Runtime/Features/LipSync/ConvaiLipSyncApplicationBase.cs Normal file
View File

@ -0,0 +1,126 @@
using System.Collections.Generic;
using Convai.Scripts.Runtime.Core;
using Convai.Scripts.Runtime.Features.LipSync;
using Convai.Scripts.Runtime.Features.LipSync.Models;
using Service;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
/// <summary>
/// This Class will serve as a base for any method of Lipsync that Convai will develop or use
/// </summary>
public abstract class ConvaiLipSyncApplicationBase : MonoBehaviour
{
/// <summary>
/// Reference to the NPC on which lipsync will be applied
/// </summary>
protected ConvaiNPC ConvaiNPC;
/// <summary>
/// Cached Reference of Facial Expression Data
/// </summary>
protected FacialExpressionData FacialExpressionData;
/// <summary>
/// Cached Reference of WeightBlendingPower
/// </summary>
protected float WeightBlendingPower;
/// <summary>
/// Initializes and setup up of the things necessary for lipsync to work
/// </summary>
/// <param name="convaiLipSync"></param>
/// <param name="convaiNPC"></param>
public virtual void Initialize(ConvaiLipSync convaiLipSync, ConvaiNPC convaiNPC)
{
FacialExpressionData = convaiLipSync.FacialExpressionData;
WeightBlendingPower = convaiLipSync.WeightBlendingPower;
HasHeadSkinnedMeshRenderer = FacialExpressionData.Head.Renderer != null;
HasTeethSkinnedMeshRenderer = FacialExpressionData.Teeth.Renderer != null;
HasTongueSkinnedMeshRenderer = FacialExpressionData.Tongue.Renderer != null;
HasJawBone = FacialExpressionData.JawBone != null;
HasTongueBone = FacialExpressionData.TongueBone != null;
ConvaiNPC = convaiNPC;
}
/// <summary>
/// Updates the tongue bone rotation to the new rotation
/// </summary>
/// <param name="newRotation"></param>
protected void UpdateTongueBoneRotation(Vector3 newRotation)
{
if (!HasTongueBone) return;
FacialExpressionData.TongueBone.transform.localEulerAngles = newRotation;
}
/// <summary>
/// Updates the jaw bone rotation to the new rotation
/// </summary>
/// <param name="newRotation"></param>
protected void UpdateJawBoneRotation(Vector3 newRotation)
{
if (!HasJawBone) return;
FacialExpressionData.JawBone.transform.localEulerAngles = newRotation;
}
/// <summary>
/// This removes the excess frames in the queue
/// </summary>
public abstract void PurgeExcessBlendShapeFrames();
/// <summary>
/// This resets the whole queue of the frames
/// </summary>
protected bool CanPurge<T>(Queue<T> queue)
{
// ? Should I hardcode the limiter for this check
return queue.Count < 10;
}
public abstract void ClearQueue();
/// <summary>
/// Adds blendshape frames in the queue
/// </summary>
/// <param name="blendshapeFrames"></param>
public virtual void EnqueueQueue(Queue<ARKitBlendShapes> blendshapeFrames)
{
}
/// <summary>
/// Adds Visemes frames in the list
/// </summary>
/// <param name="visemesFrames"></param>
public virtual void EnqueueQueue(Queue<VisemesData> visemesFrames)
{
}
/// <summary>
/// Adds a blendshape frame in the last queue
/// </summary>
/// <param name="blendshapeFrame"></param>
public virtual void EnqueueFrame(ARKitBlendShapes blendshapeFrame)
{
}
/// <summary>
/// Adds a viseme frame to the last element of the list
/// </summary>
/// <param name="viseme"></param>
public virtual void EnqueueFrame(VisemesData viseme)
{
}
#region Null States of References
protected bool HasHeadSkinnedMeshRenderer { get; private set; }
protected bool HasTeethSkinnedMeshRenderer { get; private set; }
protected bool HasTongueSkinnedMeshRenderer { get; private set; }
private bool HasJawBone { get; set; }
private bool HasTongueBone { get; set; }
#endregion
}
}

11
Assets/Convai/Scripts/Runtime/Features/LipSync/ConvaiLipSyncApplicationBase.cs.meta Normal file
View File

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: bb231034f5b2dee4494498fe9117bda1
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

96
Assets/Convai/Scripts/Runtime/Features/LipSync/LipSyncBlendFrameData.cs Normal file
View File

@ -0,0 +1,96 @@
using System.Collections.Generic;
using Convai.Scripts.Runtime.Core;
using Service;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
public class LipSyncBlendFrameData
{
#region FrameType enum
public enum FrameType
{
Visemes,
Blendshape
}
#endregion
private readonly Queue<ARKitBlendShapes> _blendShapeFrames = new();
private readonly FrameType _frameType;
private readonly GetResponseResponse _getResponseResponse;
private readonly int _totalFrames;
private readonly Queue<VisemesData> _visemesFrames = new();
private int _framesCaptured;
private bool _partiallyProcessed;
public LipSyncBlendFrameData(int totalFrames, GetResponseResponse response, FrameType frameType)
{
_totalFrames = totalFrames;
_framesCaptured = 0;
_getResponseResponse = response;
_frameType = frameType;
//ConvaiLogger.DebugLog($"Total Frames: {_totalFrames} | {response.AudioResponse.TextData}", ConvaiLogger.LogCategory.LipSync);
}
public void Enqueue(ARKitBlendShapes blendShapeFrame)
{
_blendShapeFrames.Enqueue(blendShapeFrame);
_framesCaptured++;
}
public void Enqueue(VisemesData visemesData)
{
_visemesFrames.Enqueue(visemesData);
}
public void Process(ConvaiNPC npc)
{
if (!_partiallyProcessed)
npc.EnqueueResponse(_getResponseResponse);
switch (_frameType)
{
case FrameType.Visemes:
npc.convaiLipSync.ConvaiLipSyncApplicationBase.EnqueueQueue(new Queue<VisemesData>(_visemesFrames));
break;
case FrameType.Blendshape:
npc.convaiLipSync.ConvaiLipSyncApplicationBase.EnqueueQueue(new Queue<ARKitBlendShapes>(_blendShapeFrames));
break;
}
npc.AudioManager.SetWaitForCharacterLipSync(false);
}
public void ProcessPartially(ConvaiNPC npc)
{
if (!_partiallyProcessed)
{
_partiallyProcessed = true;
npc.EnqueueResponse(_getResponseResponse);
npc.AudioManager.SetWaitForCharacterLipSync(false);
}
switch (_frameType)
{
case FrameType.Visemes:
while (_visemesFrames.Count != 0) npc.convaiLipSync.ConvaiLipSyncApplicationBase.EnqueueFrame(_visemesFrames.Dequeue());
break;
case FrameType.Blendshape:
while (_blendShapeFrames.Count != 0) npc.convaiLipSync.ConvaiLipSyncApplicationBase.EnqueueFrame(_blendShapeFrames.Dequeue());
break;
}
}
public bool CanPartiallyProcess()
{
return _framesCaptured > Mathf.Min(21, _totalFrames * 0.7f);
}
public bool CanProcess()
{
return _framesCaptured == _totalFrames;
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/LipSync/LipSyncBlendFrameData.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 31db1a9457d64f3d936ff7f5aabfb193
timeCreated: 1708491067

8
Assets/Convai/Scripts/Runtime/Features/LipSync/Models.meta Normal file
View File

@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: c45a9d44677146f4eb628784af6e9461
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

13
Assets/Convai/Scripts/Runtime/Features/LipSync/Models/BlendShapesIndexEffector.cs Normal file
View File

@ -0,0 +1,13 @@
using System;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features.LipSync.Models
{
[Serializable]
public class BlendShapesIndexEffector
{
[SerializeField] public int index;
[SerializeField] public float effectPercentage;
}
}

3
Assets/Convai/Scripts/Runtime/Features/LipSync/Models/BlendShapesIndexEffector.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 2e635a84c98545eca5224853f9d59618
timeCreated: 1722024855

33
Assets/Convai/Scripts/Runtime/Features/LipSync/Models/FacialExpressionData.cs Normal file
View File

@ -0,0 +1,33 @@
using System;
using Convai.Scripts.Runtime.Features.LipSync.Visemes;
using UnityEngine;
// ReSharper disable InconsistentNaming
namespace Convai.Scripts.Runtime.Features.LipSync.Models
{
[Serializable]
public class FacialExpressionData
{
[Tooltip("Assign the Skin Renderer and Effector for Head")]
public SkinMeshRendererData Head;
[Tooltip("Assign the Skin Renderer and Effector for Teeth")]
public SkinMeshRendererData Teeth;
[Tooltip("Assign the Skin Renderer and Effector for Tongue")]
public SkinMeshRendererData Tongue;
[Tooltip("Assign the Viseme Bone Effector List for Jaw")]
public VisemeBoneEffectorList JawBoneEffector;
[Tooltip("Assign the Viseme Bone Effector List for Tongue")]
public VisemeBoneEffectorList TongueBoneEffector;
[Tooltip("Assign the bone which effects movement of jaw")]
public GameObject JawBone;
[Tooltip("Assign the bone which effects movement of tongue")]
public GameObject TongueBone;
}
}

3
Assets/Convai/Scripts/Runtime/Features/LipSync/Models/FacialExpressionData.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: eeb1390214d44f679a8f26e8e95a40e1
timeCreated: 1722028711

16
Assets/Convai/Scripts/Runtime/Features/LipSync/Models/SkinMeshRendererData.cs Normal file
View File

@ -0,0 +1,16 @@
using System;
using Convai.Scripts.Runtime.Features.LipSync.Visemes;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features.LipSync.Models
{
[Serializable]
public class SkinMeshRendererData
{
public SkinnedMeshRenderer Renderer;
public VisemeEffectorsList VisemeEffectorsList;
[Tooltip("Lower and Upper bound of the Blendshape weight, Ex: 0-1, or 0-100")]
public Vector2 WeightBounds = new(0, 1);
}
}

3
Assets/Convai/Scripts/Runtime/Features/LipSync/Models/SkinMeshRendererData.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: a23bc1b310f54ba7849b92bcb381cf4d
timeCreated: 1722028729

8
Assets/Convai/Scripts/Runtime/Features/LipSync/Types.meta Normal file
View File

@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: 6158a9323c720f5408c5b7caa77405cc
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

204
Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiBlendShapeLipSync.cs Normal file
View File

@ -0,0 +1,204 @@
using System.Collections.Generic;
using Convai.Scripts.Runtime.Core;
using Convai.Scripts.Runtime.Features.LipSync;
using Convai.Scripts.Runtime.LoggerSystem;
using Service;
/*
! This class is a Work in progress and can produce un expected results, Convai does not advise to use this class in production, please use this with extreme caution
*/
namespace Convai.Scripts.Runtime.Features
{
public class ConvaiBlendShapeLipSync : ConvaiLipSyncApplicationBase
{
private const float A2XFRAMERATE = 1f / 30f;
private Queue<Queue<ARKitBlendShapes>> _blendShapesQueue = new();
private ARKitBlendShapes _currentBlendshape;
// private void Update()
// {
// if (_currentBlendshape == null) return;
// UpdateJawBoneRotation(new Vector3(0.0f, 0.0f, -90.0f - _currentBlendshape.JawOpen * 30f));
// UpdateTongueBoneRotation(new Vector3(0.0f, 0.0f, -5.0f * _currentBlendshape.TongueOut));
// if (!HasHeadSkinnedMeshRenderer) return;
// foreach (PropertyInfo propertyInfo in typeof(ARKitBlendShapes).GetProperties())
// {
// if (propertyInfo.PropertyType != typeof(float)) continue;
// string fieldName = propertyInfo.Name;
// float value = (float)propertyInfo.GetValue(_currentBlendshape);
// if (HeadMapping.TryGetValue(fieldName, out int index))
// HeadSkinMeshRenderer.SetBlendShapeWeightInterpolate(
// index,
// value * WeightMultiplier,
// Time.deltaTime
// );
// }
// }
protected Dictionary<string, string> GetHeadRegexMapping()
{
#region Regex Finders
string prefix = "(?:[A-Z]\\d{1,2}_)?";
string spacer = "[\\s_]*";
string mouth = "[Mm]outh";
string nose = "[Nn]ose";
string left = "[Ll]eft";
string right = "[Rr]ight";
string up = "[Uu]p";
string down = "[Dd]own";
string lower = "[Ll]ower";
string upper = "[Uu]pper";
string open = "[Oo]pen";
string funnel = "[Ff]unnel";
string pucker = "[Pp]ucker";
string sneer = "[Ss]neer";
string cheek = "[Cc]heek";
string squint = "[Ss]quint";
string brow = "[Bb]row";
string outer = "[Oo]uter";
string inner = "[Ii]nner";
string eye = "[Ee]ye";
string blink = "[Bb]link";
string look = "[Ll]ook";
string In = "[Ii]n";
string Out = "[Oo]ut";
string wide = "[Ww]ide";
string forward = "[Ff]orward";
string jaw = "[Jj]aw";
string close = "[Cc]lose";
string smile = "[Ss]mile";
string frown = "[Ff]rown";
string dimple = "[Dd]imple";
string stretch = "[Ss]tretch";
string roll = "[Rr]oll";
string shrug = "[Ss]hrug";
string press = "[Pp]ress";
#endregion
return new Dictionary<string, string>
{
{ "TougueOut", $"{prefix}[Tt]ougue{spacer}[Oo]ut" },
{ "NoseSneerRight", $"{prefix}{nose}{spacer}{sneer}{spacer}{right}" },
{ "NoseSneerLeft", $"{prefix}{nose}{spacer}{sneer}{spacer}{left}" },
{ "CheekSquintRight", $"{prefix}{cheek}{spacer}{squint}{spacer}{right}" },
{ "CheekSquintLeft", $"{prefix}{cheek}{spacer}{squint}{spacer}{left}" },
{ "CheekPuff", $"{prefix}{cheek}{spacer}[Pp]uff" },
{ "BrowDownLeft", $"{prefix}{brow}{spacer}{down}{spacer}{left}" },
{ "BrowDownRight", $"{prefix}{brow}{spacer}{down}{spacer}{right}" },
{ "BrowInnerUp", $"{prefix}{brow}{spacer}{inner}{spacer}{up}" },
{ "BrowOuterUpLeft", $"{prefix}{brow}{spacer}{outer}{spacer}{up}{spacer}{left}" },
{ "BrowOuterUpRight", $"{prefix}{brow}{spacer}{outer}{spacer}{up}{spacer}{right}" },
{ "EyeBlinkLeft", $"{prefix}{eye}{spacer}{blink}{spacer}{left}" },
{ "EyeLookDownLeft", $"{prefix}{eye}{spacer}{look}{spacer}{In}{left}" },
{ "EyeLookInLeft", $"{prefix}{eye}{spacer}{look}{spacer}{In}{spacer}{left}" },
{ "EyeLookOutLeft", $"{prefix}{eye}{spacer}{look}{spacer}{Out}{spacer}{left}" },
{ "EyeLookUpLeft", $"{prefix}{eye}{spacer}{look}{spacer}{up}{spacer}{left}" },
{ "EyeSquintLeft", $"{prefix}{eye}{spacer}{squint}{spacer}{left}" },
{ "EyeWideLeft", $"{prefix}{eye}{spacer}{wide}{spacer}{left}" },
{ "EyeBlinkRight", $"{prefix}{eye}{spacer}{blink}{spacer}{right}" },
{ "EyeLookDownRight", $"{prefix}{eye}{spacer}{look}{spacer}{In}{right}" },
{ "EyeLookInRight", $"{prefix}{eye}{spacer}{look}{spacer}{In}{spacer}{right}" },
{ "EyeLookOutRight", $"{prefix}{eye}{spacer}{look}{spacer}{Out}{spacer}{right}" },
{ "EyeLookUpRight", $"{prefix}{eye}{spacer}{look}{spacer}{up}{spacer}{right}" },
{ "EyeSquintRight", $"{prefix}{eye}{spacer}{squint}{spacer}{right}" },
{ "EyeWideRight", $"{prefix}{eye}{spacer}{wide}{spacer}{right}" },
{ "JawForward", $"{prefix}{jaw}{spacer}{forward}" },
{ "JawLeft", $"{prefix}{jaw}{spacer}{left}" },
{ "JawRight", $"{prefix}{jaw}{spacer}{right}" },
{ "JawOpen", $"{prefix}{jaw}{spacer}{open}" },
{ "MouthClose", $"{prefix}{mouth}{spacer}{close}" },
{ "MouthFunnel", $"{prefix}{mouth}{spacer}{funnel}" },
{ "MouthPucker", $"{prefix}{mouth}{spacer}{pucker}" },
{ "Mouthleft", $"{prefix}{mouth}{spacer}{left}" },
{ "MouthRight", $"{prefix}{mouth}{spacer}{right}" },
{ "MouthSmileLeft", $"{prefix}{mouth}{spacer}{smile}{spacer}{left}" },
{ "MouthSmileRight", $"{prefix}{mouth}{spacer}{smile}{spacer}{right}" },
{ "MouthFrownLeft", $"{prefix}{mouth}{spacer}{frown}{spacer}{left}" },
{ "MouthFrownRight", $"{prefix}{mouth}{spacer}{frown}{spacer}{right}" },
{ "MouthDimpleLeft", $"{prefix}{mouth}{spacer}{dimple}{spacer}{left}" },
{ "MouthDimpleRight", $"{prefix}{mouth}{spacer}{dimple}{spacer}{right}" },
{ "MouthStretchLeft", $"{prefix}{mouth}{spacer}{stretch}{spacer}{left}" },
{ "MouthStretchRight", $"{prefix}{mouth}{spacer}{stretch}{spacer}{right}" },
{ "MouthRollLower", $"{prefix}{mouth}{spacer}{roll}{spacer}{lower}" },
{ "MouthRollUpper", $"{prefix}{mouth}{spacer}{roll}{spacer}{upper}" },
{ "MouthShrugLower", $"{prefix}{mouth}{spacer}{shrug}{spacer}{lower}" },
{ "MouthShrugUpper", $"{prefix}{mouth}{spacer}{shrug}{spacer}{upper}" },
{ "MouthPressLeft", $"{prefix}{mouth}{spacer}{press}{spacer}{left}" },
{ "MouthPressRight", $"{prefix}{mouth}{spacer}{press}{spacer}{right}" },
{ "MouthLowerDownLeft", $"{prefix}{mouth}{spacer}{lower}{spacer}{down}{spacer}{left}" },
{ "MouthLowerDownRight", $"{prefix}{mouth}{spacer}{lower}{spacer}{down}{spacer}{right}" },
{ "MouthUpperUpLeft", $"{prefix}{mouth}{spacer}{upper}{spacer}{up}{spacer}{left}" },
{ "MouthUpperUpRight", $"{prefix}{mouth}{spacer}{upper}{spacer}{up}{spacer}{right}" }
};
}
public override void Initialize(ConvaiLipSync convaiLipSync, ConvaiNPC convaiNPC)
{
base.Initialize(convaiLipSync, convaiNPC);
InvokeRepeating(nameof(UpdateBlendShape), 0, A2XFRAMERATE);
}
protected virtual void UpdateBlendShape()
{
if (_blendShapesQueue == null || _blendShapesQueue.Count <= 0)
{
_currentBlendshape = new ARKitBlendShapes();
return;
}
if (_blendShapesQueue.Peek().Count == 0)
{
_blendShapesQueue.Dequeue();
return;
}
if (!ConvaiNPC.IsCharacterTalking) return;
_currentBlendshape = _blendShapesQueue.Peek().Dequeue();
}
public override void PurgeExcessBlendShapeFrames()
{
if (_blendShapesQueue.Count <= 0) return;
if (!CanPurge(_blendShapesQueue.Peek())) return;
ConvaiLogger.Info($"Purging {_blendShapesQueue.Peek().Count} frames", ConvaiLogger.LogCategory.LipSync);
_blendShapesQueue.Dequeue();
}
public override void ClearQueue()
{
_blendShapesQueue = new Queue<Queue<ARKitBlendShapes>>();
_currentBlendshape = new ARKitBlendShapes();
}
public override void EnqueueQueue(Queue<ARKitBlendShapes> blendshapeFrames)
{
_blendShapesQueue.Enqueue(blendshapeFrames);
}
public override void EnqueueFrame(ARKitBlendShapes blendshapeFrame)
{
if (_blendShapesQueue.Count == 0) EnqueueQueue(new Queue<ARKitBlendShapes>());
_blendShapesQueue.Peek().Enqueue(blendshapeFrame);
}
}
}

11
Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiBlendShapeLipSync.cs.meta Normal file
View File

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: ca3ca8129b12656449558f306c86f70d
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

150
Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiVisemesLipSync.cs Normal file
View File

@ -0,0 +1,150 @@
using System.Collections.Generic;
using Convai.Scripts.Runtime.Core;
using Convai.Scripts.Runtime.Extensions;
using Convai.Scripts.Runtime.Features.LipSync.Models;
using Convai.Scripts.Runtime.Features.LipSync.Visemes;
using Convai.Scripts.Runtime.LoggerSystem;
using Service;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features.LipSync.Types
{
public class ConvaiVisemesLipSync : ConvaiLipSyncApplicationBase
{
private const float FRAMERATE = 1f / 100.0f;
private readonly Viseme _defaultViseme = new();
private Viseme _currentViseme;
private Queue<Queue<VisemesData>> _visemesDataQueue = new();
private void LateUpdate()
{
// Check if the dequeued frame is not null.
if (_currentViseme == null) return;
// Check if the frame represents silence (-2 is a placeholder for silence).
if (Mathf.Approximately(_currentViseme.Sil, -2)) return;
UpdateJawBoneRotation(new Vector3(0.0f, 0.0f, -90.0f));
UpdateTongueBoneRotation(new Vector3(0.0f, 0.0f, -5.0f));
if (HasHeadSkinnedMeshRenderer)
UpdateMeshRenderer(FacialExpressionData.Head);
if (HasTeethSkinnedMeshRenderer)
UpdateMeshRenderer(FacialExpressionData.Teeth);
if (HasTongueSkinnedMeshRenderer)
UpdateMeshRenderer(FacialExpressionData.Tongue);
UpdateJawBoneRotation(new Vector3(0.0f, 0.0f, -90.0f - CalculateBoneEffect(FacialExpressionData.JawBoneEffector) * 30f));
UpdateTongueBoneRotation(new Vector3(0.0f, 0.0f, CalculateBoneEffect(FacialExpressionData.TongueBoneEffector) * 80f - 5f));
}
public override void Initialize(ConvaiLipSync convaiLipSync, ConvaiNPC convaiNPC)
{
base.Initialize(convaiLipSync, convaiNPC);
InvokeRepeating(nameof(UpdateBlendShape), 0, FRAMERATE);
}
public override void ClearQueue()
{
_visemesDataQueue = new Queue<Queue<VisemesData>>();
_currentViseme = new Viseme();
}
public override void PurgeExcessBlendShapeFrames()
{
if (_visemesDataQueue.Count == 0) return;
if (!CanPurge(_visemesDataQueue.Peek())) return;
ConvaiLogger.Info($"Purging {_visemesDataQueue.Peek().Count} Frames", ConvaiLogger.LogCategory.LipSync);
_visemesDataQueue.Dequeue();
}
public override void EnqueueQueue(Queue<VisemesData> visemesFrames)
{
_visemesDataQueue.Enqueue(visemesFrames);
}
public override void EnqueueFrame(VisemesData viseme)
{
if (_visemesDataQueue.Count == 0) EnqueueQueue(new Queue<VisemesData>());
_visemesDataQueue.Peek().Enqueue(viseme);
}
protected void UpdateBlendShape()
{
if (_visemesDataQueue is not { Count: > 0 })
{
_currentViseme = _defaultViseme;
return;
}
// Dequeue the next frame of visemes data from the faceDataList.
if (_visemesDataQueue.Peek() == null || _visemesDataQueue.Peek().Count <= 0)
{
_visemesDataQueue.Dequeue();
return;
}
if (!ConvaiNPC.IsCharacterTalking) return;
_currentViseme = _visemesDataQueue.Peek().Dequeue().Visemes;
}
private float CalculateBoneEffect(VisemeBoneEffectorList boneEffectorList)
{
if (boneEffectorList is null) return 0;
return (
boneEffectorList.sil * _currentViseme.Sil +
boneEffectorList.pp * _currentViseme.Pp +
boneEffectorList.ff * _currentViseme.Ff +
boneEffectorList.th * _currentViseme.Th +
boneEffectorList.dd * _currentViseme.Dd +
boneEffectorList.kk * _currentViseme.Kk +
boneEffectorList.ch * _currentViseme.Ch +
boneEffectorList.ss * _currentViseme.Ss +
boneEffectorList.nn * _currentViseme.Nn +
boneEffectorList.rr * _currentViseme.Rr +
boneEffectorList.aa * _currentViseme.Aa +
boneEffectorList.e * _currentViseme.E +
boneEffectorList.ih * _currentViseme.Ih +
boneEffectorList.oh * _currentViseme.Oh +
boneEffectorList.ou * _currentViseme.Ou
)
/ boneEffectorList.Total;
}
private void UpdateMeshRenderer(SkinMeshRendererData data)
{
VisemeEffectorsList effectorsList = data.VisemeEffectorsList;
SkinnedMeshRenderer skinnedMesh = data.Renderer;
Vector2 bounds = data.WeightBounds;
if (effectorsList == null) return;
Dictionary<int, float> finalModifiedValuesDictionary = new();
CalculateBlendShapeEffect(effectorsList.pp, ref finalModifiedValuesDictionary, _currentViseme.Pp);
CalculateBlendShapeEffect(effectorsList.ff, ref finalModifiedValuesDictionary, _currentViseme.Ff);
CalculateBlendShapeEffect(effectorsList.th, ref finalModifiedValuesDictionary, _currentViseme.Th);
CalculateBlendShapeEffect(effectorsList.dd, ref finalModifiedValuesDictionary, _currentViseme.Dd);
CalculateBlendShapeEffect(effectorsList.kk, ref finalModifiedValuesDictionary, _currentViseme.Kk);
CalculateBlendShapeEffect(effectorsList.ch, ref finalModifiedValuesDictionary, _currentViseme.Ch);
CalculateBlendShapeEffect(effectorsList.ss, ref finalModifiedValuesDictionary, _currentViseme.Ss);
CalculateBlendShapeEffect(effectorsList.nn, ref finalModifiedValuesDictionary, _currentViseme.Nn);
CalculateBlendShapeEffect(effectorsList.rr, ref finalModifiedValuesDictionary, _currentViseme.Rr);
CalculateBlendShapeEffect(effectorsList.aa, ref finalModifiedValuesDictionary, _currentViseme.Aa);
CalculateBlendShapeEffect(effectorsList.e, ref finalModifiedValuesDictionary, _currentViseme.E);
CalculateBlendShapeEffect(effectorsList.ih, ref finalModifiedValuesDictionary, _currentViseme.Ih);
CalculateBlendShapeEffect(effectorsList.oh, ref finalModifiedValuesDictionary, _currentViseme.Oh);
CalculateBlendShapeEffect(effectorsList.ou, ref finalModifiedValuesDictionary, _currentViseme.Ou);
foreach (KeyValuePair<int, float> keyValuePair in finalModifiedValuesDictionary)
skinnedMesh.SetBlendShapeWeightInterpolate(keyValuePair.Key, keyValuePair.Value * bounds.y - bounds.x, WeightBlendingPower);
}
private static void CalculateBlendShapeEffect(List<BlendShapesIndexEffector> effectors, ref Dictionary<int, float> dictionary, float value)
{
foreach (BlendShapesIndexEffector blendShapesIndexEffector in effectors)
if (dictionary.ContainsKey(blendShapesIndexEffector.index))
dictionary[blendShapesIndexEffector.index] += value * blendShapesIndexEffector.effectPercentage;
else
dictionary[blendShapesIndexEffector.index] = value * blendShapesIndexEffector.effectPercentage;
}
}
}

11
Assets/Convai/Scripts/Runtime/Features/LipSync/Types/ConvaiVisemesLipSync.cs.meta Normal file
View File

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 1917f6a14d9682440a27a434f0210496
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

8
Assets/Convai/Scripts/Runtime/Features/LipSync/Visemes.meta Normal file
View File

@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: 7d67d884b024af84cb3e3a450c6a742a
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

50
Assets/Convai/Scripts/Runtime/Features/LipSync/Visemes/VisemeBoneEffectorList.cs Normal file
View File

@ -0,0 +1,50 @@
using Convai.Scripts.Runtime.Attributes;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features.LipSync.Visemes
{
[CreateAssetMenu(fileName = "Convai Viseme Bone Effector", menuName = "Convai/Expression/Visemes Bone Effector", order = 0)]
public class VisemeBoneEffectorList : ScriptableObject
{
[SerializeField] public float sil;
[SerializeField] public float pp;
[SerializeField] public float ff;
[SerializeField] public float th;
[SerializeField] public float dd;
[SerializeField] public float kk;
[SerializeField] public float ch;
[SerializeField] public float ss;
[SerializeField] public float nn;
[SerializeField] public float rr;
[SerializeField] public float aa;
[SerializeField] public float e;
[SerializeField] public float ih;
[SerializeField] public float oh;
[SerializeField] public float ou;
[field: SerializeField]
[field: ReadOnly]
public float Total { get; private set; }
private void OnValidate()
{
Total = 0;
Total += sil;
Total += pp;
Total += ff;
Total += th;
Total += dd;
Total += kk;
Total += ch;
Total += ss;
Total += nn;
Total += rr;
Total += aa;
Total += e;
Total += ih;
Total += oh;
Total += ou;
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/LipSync/Visemes/VisemeBoneEffectorList.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: df06673497ba4952ab788f87edbf7950
timeCreated: 1722030678

26
Assets/Convai/Scripts/Runtime/Features/LipSync/Visemes/VisemeEffectorsList.cs Normal file
View File

@ -0,0 +1,26 @@
using System.Collections.Generic;
using Convai.Scripts.Runtime.Features.LipSync.Models;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features.LipSync.Visemes
{
[CreateAssetMenu(fileName = "Convai Viseme Effectors", menuName = "Convai/Expression/Visemes Skin Effector", order = 0)]
public class VisemeEffectorsList : ScriptableObject
{
[SerializeField] public List<BlendShapesIndexEffector> sil;
[SerializeField] public List<BlendShapesIndexEffector> pp;
[SerializeField] public List<BlendShapesIndexEffector> ff;
[SerializeField] public List<BlendShapesIndexEffector> th;
[SerializeField] public List<BlendShapesIndexEffector> dd;
[SerializeField] public List<BlendShapesIndexEffector> kk;
[SerializeField] public List<BlendShapesIndexEffector> ch;
[SerializeField] public List<BlendShapesIndexEffector> ss;
[SerializeField] public List<BlendShapesIndexEffector> nn;
[SerializeField] public List<BlendShapesIndexEffector> rr;
[SerializeField] public List<BlendShapesIndexEffector> aa;
[SerializeField] public List<BlendShapesIndexEffector> e;
[SerializeField] public List<BlendShapesIndexEffector> ih;
[SerializeField] public List<BlendShapesIndexEffector> oh;
[SerializeField] public List<BlendShapesIndexEffector> ou;
}
}

11
Assets/Convai/Scripts/Runtime/Features/LipSync/Visemes/VisemeEffectorsList.cs.meta Normal file
View File

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 60d464fb2d5c4b959eb83e0010e7b3a8
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

3
Assets/Convai/Scripts/Runtime/Features/LongTermMemory.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 3f68b63159b9486eb827769aface233e
timeCreated: 1720520243

53
Assets/Convai/Scripts/Runtime/Features/LongTermMemory/ConvaiLTMController.cs Normal file
View File

@ -0,0 +1,53 @@
using System.Collections;
using System.Threading.Tasks;
using Convai.Scripts.Runtime.Core;
using Convai.Scripts.Runtime.PlayerStats.API;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features.LongTermMemory
{
[AddComponentMenu("Convai/Convai Long Term Memory")]
public class ConvaiLTMController : MonoBehaviour
{
[field: HideInInspector]
[field: SerializeField]
public LTMStatus LTMStatus { get; private set; } = LTMStatus.NotDefined;
private ConvaiNPC _convaiNpc;
private async void Reset()
{
await GetLTMStatus();
}
public async Task GetLTMStatus()
{
LTMStatus = LTMStatus.NotDefined;
_convaiNpc = GetComponent<ConvaiNPC>();
if (!ConvaiAPIKeySetup.GetAPIKey(out string apiKey)) return;
LTMStatus = await LongTermMemoryAPI.GetLTMStatus(apiKey, _convaiNpc.characterID, OnRequestFailed) ? LTMStatus.Enabled : LTMStatus.Disabled;
}
private void OnRequestFailed()
{
LTMStatus = LTMStatus.Failed;
}
/// <summary>
/// It starts a coroutine which can toggle the global status of the LTM for the character.
/// It should not be done at runtime since it will toggle the status of the LTM for all the users
/// </summary>
/// <param name="enable"> new status of LTM</param>
/// <returns></returns>
public IEnumerator ToggleLTM(bool enable)
{
if (!ConvaiAPIKeySetup.GetAPIKey(out string apiKey)) yield break;
LTMStatus = LTMStatus.NotDefined;
_convaiNpc = GetComponent<ConvaiNPC>();
Task<bool> resultTask = LongTermMemoryAPI.ToggleLTM(apiKey, _convaiNpc.characterID, enable, OnRequestFailed);
yield return new WaitUntil(() => resultTask.IsCompleted);
if (!resultTask.Result) yield break;
LTMStatus = enable ? LTMStatus.Enabled : LTMStatus.Disabled;
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/LongTermMemory/ConvaiLTMController.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 80eba48346a34f57b0b76a123e959940
timeCreated: 1720520265

10
Assets/Convai/Scripts/Runtime/Features/LongTermMemory/LTMStatus.cs Normal file
View File

@ -0,0 +1,10 @@
namespace Convai.Scripts.Runtime.Features.LongTermMemory
{
public enum LTMStatus
{
NotDefined,
Enabled,
Disabled,
Failed
}
}

3
Assets/Convai/Scripts/Runtime/Features/LongTermMemory/LTMStatus.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 6a89ad77f1374428bfe2252160c4e67d
timeCreated: 1720607177

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 533d7f0ec16149f2b19fb5b228e7a1d8
timeCreated: 1705687336

243
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/ConvaiGroupNPCController.cs Normal file
View File

@ -0,0 +1,243 @@
using System;
using System.Collections;
using System.Threading.Tasks;
using Convai.Scripts.Runtime.Core;
using Convai.Scripts.Runtime.Features.LipSync;
using Convai.Scripts.Runtime.LoggerSystem;
using Service;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
/// <summary>
/// This class is responsible for handling out all the tasks related to NPC to NPC conversation for a NPC of a group
/// </summary>
public class ConvaiGroupNPCController : MonoBehaviour
{
/// <summary>
/// Used to set Player GameObject Transform and lip-sync
/// </summary>
private void Awake()
{
if (playerTransform == null) playerTransform = Camera.main.transform;
ConvaiNPC = GetComponent<ConvaiNPC>();
CONVERSATION_DISTANCE_THRESHOLD = conversationDistanceThreshold == 0 ? Mathf.Infinity : conversationDistanceThreshold;
TryGetComponent(out _lipSync);
}
/// <summary>
/// Starts coroutine for player vicinity check and subscribe to necessary events
/// </summary>
private void Start()
{
_npcGroup = NPC2NPCConversationManager.Instance.npcGroups.Find(c => c.BelongToGroup(this));
otherNPC = _npcGroup.GroupNPC1 == this ? _npcGroup.GroupNPC2 : _npcGroup.GroupNPC1;
_checkPlayerVicinityCoroutine = StartCoroutine(CheckPlayerVicinity());
if (TryGetComponent(out ConvaiNPCAudioManager convaiNPCAudio)) convaiNPCAudio.OnAudioTranscriptAvailable += HandleAudioTranscriptAvailable;
}
/// <summary>
/// Unsubscribes to the events and stops the coroutine
/// </summary>
private void OnDestroy()
{
if (TryGetComponent(out ConvaiNPCAudioManager convaiNPCAudio)) convaiNPCAudio.OnAudioTranscriptAvailable -= HandleAudioTranscriptAvailable;
if(_npc2NPCGrpcClient!=null) _npc2NPCGrpcClient.OnTranscriptAvailable -= HandleTranscriptAvailable;
if (_checkPlayerVicinityCoroutine != null) StopCoroutine(_checkPlayerVicinityCoroutine);
}
/// <summary>
/// Shows speech bubble and adds the received text to final transcript
/// </summary>
/// <param name="transcript"></param>
private void HandleAudioTranscriptAvailable(string transcript)
{
if (IsInConversationWithAnotherNPC)
ShowSpeechBubble?.Invoke(transcript);
}
private void HandleTranscriptAvailable(string transcript, ConvaiGroupNPCController npcController)
{
if (npcController != this) return;
_finalResponseText += transcript;
}
/// <summary>
/// Attaches the speech bubble to the NPC game-object
/// </summary>
public void AttachSpeechBubble()
{
if (TryGetComponent(out ConvaiSpeechBubbleController _)) return;
gameObject.AddComponent<ConvaiSpeechBubbleController>().Initialize(speechBubblePrefab, this);
}
/// <summary>
/// Destroys the speech bubble game-object
/// </summary>
public void DetachSpeechBubble()
{
if (TryGetComponent(out ConvaiSpeechBubbleController convaiSpeechBubble)) Destroy(convaiSpeechBubble);
}
/// <summary>
/// Store the references of the client and subscribe to the necessary events
/// </summary>
/// <param name="client"></param>
public void InitializeNpc2NpcGrpcClient(NPC2NPCGRPCClient client)
{
_npc2NPCGrpcClient = client;
_npc2NPCGrpcClient.OnTranscriptAvailable += HandleTranscriptAvailable;
}
/// <summary>
/// Every 0.5 seconds updates if player is near or not and fire events according to the state
/// </summary>
/// <returns></returns>
private IEnumerator CheckPlayerVicinity()
{
bool previousState = false;
Vector3 previousPlayerPosition = Vector3.zero;
yield return new WaitForSeconds(0.1f);
while (true)
{
Vector3 currentPlayerPosition = playerTransform.transform.position;
// Check if the player has moved more than a certain threshold distance
if (Vector3.Distance(previousPlayerPosition, currentPlayerPosition) > PLAYER_MOVE_THRESHOLD)
{
// Calculate the distance between the NPC and the player
float distanceToPlayer = Vector3.Distance(transform.position, currentPlayerPosition);
// Check if the player is within the threshold distance
bool isPlayerCurrentlyNear = distanceToPlayer <= CONVERSATION_DISTANCE_THRESHOLD;
// If the player's current vicinity state is different from the previous state, raise the event
if (isPlayerCurrentlyNear != previousState && !ConvaiNPC.isCharacterActive)
{
OnPlayerVicinityChanged?.Invoke(isPlayerCurrentlyNear, this);
previousState = isPlayerCurrentlyNear; // Update the previous state
ConvaiLogger.Info($"Player is currently near {ConvaiNPC.characterName}: {isPlayerCurrentlyNear}", ConvaiLogger.LogCategory.Character);
}
previousPlayerPosition = currentPlayerPosition; // Update the player's previous position
// Check every half second
}
yield return new WaitForSeconds(0.5f);
}
}
/// <summary>
/// Sends the text to the other NPC in the group
/// </summary>
/// <param name="message"></param>
public async void SendTextDataNPC2NPC(string message)
{
if (_npc2NPCGrpcClient == null)
{
ConvaiLogger.Warn("No GRPC client initialized for this NPC.", ConvaiLogger.LogCategory.Character);
return;
}
try
{
CanRelayMessage = false;
await Task.Delay(500);
await _npc2NPCGrpcClient.SendTextData(
message,
ConvaiNPC.characterID,
ConvaiNPC.sessionID,
_lipSync != null,
FaceModel,
this);
}
catch (Exception ex)
{
ConvaiLogger.Warn($"Error sending message data for NPC2NPC: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
public void EndOfResponseReceived()
{
if (TryGetComponent(out ConvaiNPCAudioManager convaiNPCAudio)) convaiNPCAudio.OnCharacterTalkingChanged += SendFinalTranscriptToOtherNPC;
ConversationManager.RelayMessage(_finalResponseText, this);
_finalResponseText = "";
}
private void SendFinalTranscriptToOtherNPC(bool isTalking)
{
if (IsInConversationWithAnotherNPC)
{
if (!isTalking)
{
ConversationManager.SwitchSpeaker(this);
if (TryGetComponent(out ConvaiNPCAudioManager convaiNPCAudio)) convaiNPCAudio.OnCharacterTalkingChanged -= SendFinalTranscriptToOtherNPC;
HideSpeechBubble?.Invoke();
}
else
{
ConvaiLogger.DebugLog($"{ConvaiNPC.characterName} is currently still talking. ", ConvaiLogger.LogCategory.Character);
}
}
}
public bool IsPlayerNearMe()
{
bool result = Vector3.Distance(transform.position, playerTransform.position) < CONVERSATION_DISTANCE_THRESHOLD;
ConvaiLogger.Info($"Player is near {CharacterName}: {result}", ConvaiLogger.LogCategory.Character);
return result;
}
public bool IsOtherNPCTalking()
{
return otherNPC.ConvaiNPC.IsCharacterTalking;
}
#region Serialized Fields
[Tooltip("The prefab for the speech bubble to display above the NPC. [Optional]")] [SerializeField]
private NPCSpeechBubble speechBubblePrefab;
[Tooltip("Attach the Main Player Transform here so that distance check can be performed")] [SerializeField]
private Transform playerTransform;
// The distance from the NPC to the player when the NPC will start talking
[Tooltip("The distance from the NPC to the player when the NPC will start talking. Set to 0 to disable this feature. [Optional]")] [SerializeField] [Range(0f, 100f)]
private float conversationDistanceThreshold = 5.0f;
#endregion
#region Private Attributes
private const float PLAYER_MOVE_THRESHOLD = 0.5f;
private float CONVERSATION_DISTANCE_THRESHOLD;
private string _finalResponseText = "";
private NPC2NPCGRPCClient _npc2NPCGrpcClient;
private ConvaiLipSync _lipSync;
private Coroutine _checkPlayerVicinityCoroutine;
private FaceModel FaceModel => _lipSync == null ? FaceModel.OvrModelName : _lipSync.faceModel;
#endregion
#region Events
public event Action<string> ShowSpeechBubble;
public event Action HideSpeechBubble;
public event Action<bool, ConvaiGroupNPCController> OnPlayerVicinityChanged;
private NPCGroup _npcGroup;
private ConvaiGroupNPCController otherNPC;
#endregion
#region Public Attributes
public bool CanRelayMessage { get; set; } = true;
public NPC2NPCConversationManager ConversationManager { get; set; }
public bool IsInConversationWithAnotherNPC { get; set; }
public string CharacterName => ConvaiNPC == null ? string.Empty : ConvaiNPC.characterName;
public string CharacterID => ConvaiNPC == null ? string.Empty : ConvaiNPC.characterID;
public ConvaiNPC ConvaiNPC { get; private set; }
#endregion
}
}

11
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/ConvaiGroupNPCController.cs.meta Normal file
View File

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 74d3f2dd15d074a429eeea8c31f80b13
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

302
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPC2NPCConversationManager.cs Normal file
View File

@ -0,0 +1,302 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using Convai.Scripts.Runtime.Core;
using Convai.Scripts.Runtime.LoggerSystem;
using Grpc.Core;
using Service;
using UnityEngine;
using Random = UnityEngine.Random;
namespace Convai.Scripts.Runtime.Features
{
/// <summary>
/// Manages the conversation between two Convai powered NPC groups
/// </summary>
public class NPC2NPCConversationManager : MonoBehaviour
{
private const string GRPC_API_ENDPOINT = "stream.convai.com";
public static NPC2NPCConversationManager Instance;
public List<NPCGroup> npcGroups;
private readonly List<NPCGroup> _groupsWhereConversationEnded = new();
private string _apiKey = string.Empty;
private void Awake()
{
if (Instance == null)
Instance = this;
else
Destroy(gameObject);
LoadApiKey();
}
private void Start()
{
foreach (NPCGroup group in npcGroups)
group.Initialize(HandlePlayerVicinityChanged);
StartConversationWithAllNPCs();
}
/// <summary>
/// Handles the event of player vicinity change.
/// </summary>
/// <param name="isPlayerNear">Indicates if the player is near.</param>
/// <param name="npc">The NPC for which the vicinity changed.</param>
private void HandlePlayerVicinityChanged(bool isPlayerNear, ConvaiGroupNPCController npc)
{
if (isPlayerNear)
ResumeConversation(npc);
}
/// <summary>
/// Processes the message from the sender NPC.
/// </summary>
/// <param name="sender">The NPC sending the message.</param>
/// <param name="topic">The topic of the conversation.</param>
/// <param name="message">The message to be processed.</param>
/// <returns>The processed message.</returns>
private string ProcessMessage(ConvaiGroupNPCController sender, string topic, string message)
{
string processedMessage = $"{sender.CharacterName} said \"{message}\" to you. Reply to it. ";
processedMessage += Random.Range(0, 2) == 0
? $"Talk about something related to {message}. "
: $"Talk about something other than \"{message}\" but related to {topic}. Gently change the conversation topic. ";
return processedMessage + "Definitely, reply to the message. Dont address speaker. Keep the reply short. Do not repeat the same message, or keep asking same question.";
}
/// <summary>
/// Relays the message from the sender NPC.
/// </summary>
/// <param name="message">The message to be relayed.</param>
/// <param name="sender">The NPC sending the message.</param>
/// <param name="performSwitch"></param>
public void RelayMessage(string message, ConvaiGroupNPCController sender)
{
NPCGroup npcGroup = npcGroups.Find(c => c.BelongToGroup(sender));
if (npcGroup == null)
{
ConvaiLogger.Warn("Conversation not found for the sender.", ConvaiLogger.LogCategory.Character);
return;
}
npcGroup.messageToRelay = message;
if (!npcGroup.CurrentSpeaker.IsPlayerNearMe()) return;
StartCoroutine(RelayMessageCoroutine(message, sender, npcGroup));
}
/// <summary>
/// Coroutine to relay the message from the sender NPC.
/// </summary>
/// <param name="message">The message to be relayed.</param>
/// <param name="npcGroup"> The NPC group to relay the message to. </param>
/// <returns>An IEnumerator to be used in a coroutine.</returns>
private IEnumerator RelayMessageCoroutine(string message, ConvaiGroupNPCController sender, NPCGroup npcGroup)
{
yield return new WaitForSeconds(0.5f);
ConvaiGroupNPCController receiver = npcGroup.GroupNPC1 == sender ? npcGroup.GroupNPC2 : npcGroup.GroupNPC1;
while (receiver.ConvaiNPC.IsCharacterTalking)
{
yield return new WaitForSeconds(0.1f);
}
try
{
ConvaiLogger.DebugLog($"Relaying message from {sender.CharacterName} to {receiver.CharacterName}: {message}", ConvaiLogger.LogCategory.Character);
string processedMessage = ProcessMessage(receiver, npcGroup.topic, message);
receiver.SendTextDataNPC2NPC(processedMessage);
}
catch (Exception e)
{
ConvaiLogger.Warn($"Failed to relay message: {e.Message}", ConvaiLogger.LogCategory.Character);
}
}
/// <summary>
/// Switches the speaker in the conversation.
/// </summary>
/// <param name="currentSpeaker">The current speaker NPC.</param>
/// <returns>The new speaker NPC.</returns>
public void SwitchSpeaker(ConvaiGroupNPCController currentSpeaker)
{
NPCGroup group = npcGroups.Find(g => g.CurrentSpeaker == currentSpeaker);
if (group != null)
{
group.CurrentSpeaker = currentSpeaker == group.GroupNPC1 ? group.GroupNPC2 : group.GroupNPC1;
ConvaiLogger.DebugLog($"Switching NPC2NPC Speaker to {group.CurrentSpeaker}", ConvaiLogger.LogCategory.Character);
return;
}
ConvaiLogger.Warn("Failed to switch speaker. Current speaker not found in any group.", ConvaiLogger.LogCategory.Character);
}
private void LoadApiKey()
{
ConvaiAPIKeySetup.GetAPIKey(out _apiKey);
}
/// <summary>
/// Initializes a single NPC for conversation.
/// </summary>
/// <param name="npc">The NPC to initialize.</param>
/// <param name="grpcClient">The GRPC client to use for the NPC.</param>
private void InitializeNPC(ConvaiGroupNPCController npc, NPC2NPCGRPCClient grpcClient)
{
if (npc == null)
{
ConvaiLogger.Warn("The given NPC is null.", ConvaiLogger.LogCategory.Character);
return;
}
npc.ConversationManager = this;
npc.InitializeNpc2NpcGrpcClient(grpcClient);
npc.AttachSpeechBubble();
npc.IsInConversationWithAnotherNPC = true;
npc.ConvaiNPC.isCharacterActive = false;
}
/// <summary>
/// Starts the conversation for the given NPC.
/// </summary>
/// <param name="npcGroup">The NPC to start the conversation for.</param>
private void InitializeNPCGroup(NPCGroup npcGroup)
{
if (npcGroup == null)
{
ConvaiLogger.Warn("The given NPC is not part of any group.", ConvaiLogger.LogCategory.Character);
return;
}
ConvaiGroupNPCController npc1 = npcGroup.GroupNPC1;
ConvaiGroupNPCController npc2 = npcGroup.GroupNPC2;
if (npc1.IsInConversationWithAnotherNPC || npc2.IsInConversationWithAnotherNPC)
{
ConvaiLogger.Warn($"{npc1.CharacterName} or {npc2.CharacterName} is already in a conversation.", ConvaiLogger.LogCategory.Character);
return;
}
NPC2NPCGRPCClient grpcClient = CreateAndInitializeGRPCClient(npcGroup);
// Initialize both NPCs
InitializeNPC(npc1, grpcClient);
InitializeNPC(npc2, grpcClient);
npcGroup.CurrentSpeaker = Random.Range(0, 10) % 2 == 0 ? npc1 : npc2;
}
/// <summary>
/// Creates and initializes a new GRPC client for the given NPC group.
/// </summary>
/// <param name="group">The NPC group to create the GRPC client for.</param>
/// <returns>The initialized GRPC client.</returns>
private NPC2NPCGRPCClient CreateAndInitializeGRPCClient(NPCGroup group)
{
GameObject grpcClientGameObject = new($"GRPCClient_{group.GroupNPC1.CharacterID}_{group.GroupNPC2.CharacterID}")
{
transform = { parent = transform }
};
NPC2NPCGRPCClient grpcClient = grpcClientGameObject.AddComponent<NPC2NPCGRPCClient>();
ConvaiService.ConvaiServiceClient serviceClient = CreateNewConvaiServiceClient();
grpcClient.Initialize(_apiKey, serviceClient, group);
return grpcClient;
}
/// <summary>
/// Creates a new ConvaiServiceClient.
/// </summary>
/// <returns> The new ConvaiServiceClient. </returns>
private ConvaiService.ConvaiServiceClient CreateNewConvaiServiceClient()
{
try
{
SslCredentials credentials = new();
List<ChannelOption> options = new()
{
new ChannelOption(ChannelOptions.MaxReceiveMessageLength, 16 * 1024 * 1024)
};
Channel channel = new(GRPC_API_ENDPOINT, credentials, options);
return new ConvaiService.ConvaiServiceClient(channel);
}
catch (Exception ex)
{
ConvaiLogger.Error($"Failed to create ConvaiServiceClient: {ex.Message}", ConvaiLogger.LogCategory.Character);
throw;
}
}
/// <summary>
/// Resumes the conversation for the given NPC.
/// </summary>
/// <param name="sender"> The NPC to resume the conversation for. </param>
private void ResumeConversation(ConvaiGroupNPCController sender)
{
NPCGroup npcGroup = npcGroups.Find(g => g.BelongToGroup(sender));
if (npcGroup.IsAnyoneTalking()) return;
if (_groupsWhereConversationEnded.Contains(npcGroup))
{
InitializeNPCGroup(npcGroup);
_groupsWhereConversationEnded.Remove(npcGroup);
}
if (string.IsNullOrEmpty(npcGroup.messageToRelay))
{
string message = $"Talk about {npcGroup.topic}.";
npcGroup.CurrentSpeaker.SendTextDataNPC2NPC(message);
npcGroup.messageToRelay = message;
ConvaiLogger.DebugLog($"Starting conversation for the first time between {npcGroup.GroupNPC1.CharacterName} and {npcGroup.GroupNPC2.CharacterName}",
ConvaiLogger.LogCategory.Character);
}
else
{
RelayMessage(npcGroup.messageToRelay, npcGroup.CurrentSpeaker);
ConvaiLogger.DebugLog($"Resuming conversation between {npcGroup.GroupNPC1.CharacterName} and {npcGroup.GroupNPC2.CharacterName}",
ConvaiLogger.LogCategory.Character);
}
}
/// <summary>
/// Ends the conversation for the given NPC.
/// </summary>
/// <param name="npc"> The NPC to end the conversation for. </param>
public void EndConversation(ConvaiGroupNPCController npc)
{
NPCGroup group = npcGroups.Find(g => g.BelongToGroup(npc));
ConvaiLogger.DebugLog($"Ending conversation between {group.GroupNPC1.CharacterName} and {group.GroupNPC2.CharacterName}", ConvaiLogger.LogCategory.Character);
void EndConversationForNPC(ConvaiGroupNPCController groupNPC)
{
groupNPC.IsInConversationWithAnotherNPC = false;
groupNPC.ConvaiNPC.InterruptCharacterSpeech();
groupNPC.GetComponent<ConvaiGroupNPCController>().DetachSpeechBubble();
}
EndConversationForNPC(group.GroupNPC1);
EndConversationForNPC(group.GroupNPC2);
_groupsWhereConversationEnded.Add(group);
ConvaiNPCManager.Instance.SetActiveConvaiNPC(npc.ConvaiNPC);
Destroy(transform.Find($"GRPCClient_{group.GroupNPC1.CharacterID}_{group.GroupNPC2.CharacterID}").gameObject);
}
/// <summary>
/// Starts the conversation with all NPCs.
/// </summary>
private void StartConversationWithAllNPCs()
{
IEnumerable filteredList = npcGroups
.Where(npcGroup => npcGroup.BothNPCAreNotNull())
.Where(npcGroup => npcGroup.BothNPCAreNotActiveNPC());
foreach (NPCGroup npcGroup in filteredList)
InitializeNPCGroup(npcGroup);
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPC2NPCConversationManager.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: e5c3956535ee405983974a5434b6f4a4
timeCreated: 1705687346

226
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPC2NPCGRPCClient.cs Normal file
View File

@ -0,0 +1,226 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Convai.Scripts.Runtime.Core;
using Convai.Scripts.Runtime.LoggerSystem;
using Convai.Scripts.Runtime.Utils;
using Grpc.Core;
using Service;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
/// <summary>
/// Represents an NPC2NPCGRPCClient that can be used to communicate with the ConvaiService using gRPC.
/// </summary>
public class NPC2NPCGRPCClient : MonoBehaviour
{
private readonly CancellationTokenSource _cancellationTokenSource = new();
private string _apiKey;
private ConvaiService.ConvaiServiceClient _client;
private NPCGroup _npcGroup;
public event Action<string, ConvaiGroupNPCController> OnTranscriptAvailable;
private void OnDestroy()
{
_cancellationTokenSource?.Cancel();
_cancellationTokenSource?.Dispose();
}
/// <summary>
/// Initializes the NPC2NPCGRPCClient with the given API key and ConvaiService client.
/// </summary>
/// <param name="apiKey">The API key to use for authentication.</param>
/// <param name="client">The ConvaiService client to use for communication.</param>
public void Initialize(string apiKey, ConvaiService.ConvaiServiceClient client, NPCGroup group)
{
_apiKey = apiKey;
_client = client;
_npcGroup = group;
}
/// <summary>
/// Creates an AsyncDuplexStreamingCall with the specified headers.
/// </summary>
/// <returns>An AsyncDuplexStreamingCall with the specified headers.</returns>
private AsyncDuplexStreamingCall<GetResponseRequest, GetResponseResponse> GetAsyncDuplexStreamingCallOptions()
{
Metadata headers = new()
{
{ "source", "Unity" },
{ "version", "3.2.1" }
};
CallOptions options = new(headers);
return _client.GetResponse(options);
}
/// <summary>
/// Sends the specified user text to the server and receives a response.
/// </summary>
/// <param name="userText">The user text to send to the server.</param>
/// <param name="characterID">The ID of the character to use for the request.</param>
/// <param name="sessionID">The ID of the session to use for the request.</param>
/// <param name="isLipSyncActive">Whether lip sync is active for the request.</param>
/// <param name="faceModel">The face model to use for the request.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public async Task SendTextData(string userText, string characterID, string sessionID, bool isLipSyncActive, FaceModel faceModel, ConvaiGroupNPCController npcController)
{
AsyncDuplexStreamingCall<GetResponseRequest, GetResponseResponse> call = GetAsyncDuplexStreamingCallOptions();
GetResponseRequest getResponseConfigRequest = CreateGetResponseRequest(characterID, sessionID, isLipSyncActive, faceModel, false, null);
try
{
await call.RequestStream.WriteAsync(getResponseConfigRequest);
await call.RequestStream.WriteAsync(new GetResponseRequest
{
GetResponseData = new GetResponseRequest.Types.GetResponseData
{
TextData = userText
}
});
await call.RequestStream.CompleteAsync();
Task receiveResultsTask = Task.Run(
async () => { await ReceiveResultFromServer(call, _cancellationTokenSource.Token, npcController); },
_cancellationTokenSource.Token);
await receiveResultsTask.ConfigureAwait(false);
}
catch (Exception)
{
// ignored
}
}
/// <summary>
/// Creates a GetResponseRequest with the specified parameters.
/// </summary>
/// <param name="characterID">The ID of the character to use for the request.</param>
/// <param name="sessionID">The ID of the session to use for the request.</param>
/// <param name="isLipSyncActive">Whether lip sync is active for the request.</param>
/// <param name="faceModel">The face model to use for the request.</param>
/// <param name="isActionActive">Whether action is active for the request.</param>
/// <param name="actionConfig">The action configuration to use for the request.</param>
/// <returns>A GetResponseRequest with the specified parameters.</returns>
private GetResponseRequest CreateGetResponseRequest(string characterID, string sessionID, bool isLipSyncActive, FaceModel faceModel, bool isActionActive,
ActionConfig actionConfig)
{
GetResponseRequest getResponseConfigRequest = new()
{
GetResponseConfig = new GetResponseRequest.Types.GetResponseConfig
{
CharacterId = characterID,
ApiKey = _apiKey,
SessionId = sessionID,
AudioConfig = new AudioConfig
{
EnableFacialData = isLipSyncActive,
FaceModel = faceModel
}
}
};
if (isActionActive)
getResponseConfigRequest.GetResponseConfig.ActionConfig = actionConfig;
return getResponseConfigRequest;
}
/// <summary>
/// Receives a response from the server asynchronously.
/// </summary>
/// <param name="call">The AsyncDuplexStreamingCall to use for receiving the response.</param>
/// <param name="cancellationToken">The cancellation token to use for cancelling the operation.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
private async Task ReceiveResultFromServer(AsyncDuplexStreamingCall<GetResponseRequest, GetResponseResponse> call, CancellationToken cancellationToken, ConvaiGroupNPCController npcController)
{
ConvaiLogger.Info("Receiving response from server", ConvaiLogger.LogCategory.Character);
Queue<LipSyncBlendFrameData> lipSyncBlendFrameQueue = new();
ConvaiNPC convaiNPC = npcController.ConvaiNPC;
bool firstSilFound = false;
while (!cancellationToken.IsCancellationRequested && await call.ResponseStream.MoveNext(cancellationToken).ConfigureAwait(false))
try
{
GetResponseResponse result = call.ResponseStream.Current;
// Process the received response here
if (result.AudioResponse != null)
if (result.AudioResponse.AudioData != null)
{
MainThreadDispatcher.Instance.RunOnMainThread(() => OnTranscriptAvailable?.Invoke(result.AudioResponse.TextData, npcController));
if (result.AudioResponse.AudioData.ToByteArray().Length > 46)
{
byte[] wavBytes = result.AudioResponse.AudioData.ToByteArray();
// will only work for wav files
if (convaiNPC.convaiLipSync == null)
{
ConvaiLogger.DebugLog($"Enqueuing responses: {result.AudioResponse.TextData}", ConvaiLogger.LogCategory.LipSync);
convaiNPC.EnqueueResponse(result);
}
else
{
LipSyncBlendFrameData.FrameType frameType =
convaiNPC.convaiLipSync.faceModel == FaceModel.OvrModelName
? LipSyncBlendFrameData.FrameType.Visemes
: LipSyncBlendFrameData.FrameType.Blendshape;
lipSyncBlendFrameQueue.Enqueue(
new LipSyncBlendFrameData(
(int)(WavUtility.CalculateDurationSeconds(wavBytes) * 30),
result,
frameType
)
);
}
}
if (result.AudioResponse.VisemesData != null)
if (convaiNPC.convaiLipSync != null)
{
//ConvaiLogger.Info(result.AudioResponse.VisemesData, ConvaiLogger.LogCategory.LipSync);
if (result.AudioResponse.VisemesData.Visemes.Sil == -2 || result.AudioResponse.EndOfResponse)
{
if (firstSilFound) lipSyncBlendFrameQueue.Dequeue().Process(convaiNPC);
firstSilFound = true;
}
else
{
lipSyncBlendFrameQueue.Peek().Enqueue(result.AudioResponse.VisemesData);
}
}
if (result.AudioResponse.BlendshapesData != null)
if (convaiNPC.convaiLipSync != null)
{
if (lipSyncBlendFrameQueue.Peek().CanProcess() || result.AudioResponse.EndOfResponse)
{
lipSyncBlendFrameQueue.Dequeue().Process(convaiNPC);
}
else
{
lipSyncBlendFrameQueue.Peek().Enqueue(result.AudioResponse.FaceEmotion.ArKitBlendShapes);
if (lipSyncBlendFrameQueue.Peek().CanPartiallyProcess()) lipSyncBlendFrameQueue.Peek().ProcessPartially(convaiNPC);
}
}
if (result.AudioResponse.EndOfResponse)
MainThreadDispatcher.Instance.RunOnMainThread(npcController.EndOfResponseReceived);
}
}
catch (RpcException rpcException)
{
if (rpcException.StatusCode == StatusCode.Cancelled)
ConvaiLogger.Error(rpcException, ConvaiLogger.LogCategory.Character);
else
throw;
}
catch (Exception ex)
{
ConvaiLogger.DebugLog(ex, ConvaiLogger.LogCategory.Character);
}
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPC2NPCGRPCClient.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: dc8dba94e7504aab878f30a19a55fa5a
timeCreated: 1708531595

77
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPCGroup.cs Normal file
View File

@ -0,0 +1,77 @@
using System;
using Convai.Scripts.Runtime.Attributes;
using Convai.Scripts.Runtime.Core;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
/// <summary>
/// A group of NPCs that are currently conversing with each other.
/// </summary>
[Serializable]
public class NPCGroup
{
[field: SerializeField] public ConvaiGroupNPCController GroupNPC1 { get; private set; }
[field: SerializeField] public ConvaiGroupNPCController GroupNPC2 { get; private set; }
public string topic;
[ReadOnly] public string messageToRelay;
private bool _isPlayerNearGroup;
private Action<bool, ConvaiGroupNPCController> _vicinityChangedCallback;
public ConvaiGroupNPCController CurrentSpeaker { get; set; }
public ConvaiGroupNPCController CurrentListener => CurrentSpeaker == GroupNPC1 ? GroupNPC2 : GroupNPC1;
public void Initialize(Action<bool, ConvaiGroupNPCController> vicinityChangedCallback)
{
_vicinityChangedCallback = vicinityChangedCallback;
if (GroupNPC1 == null) return;
GroupNPC1.OnPlayerVicinityChanged += HandleVicinity;
GroupNPC1.OnPlayerVicinityChanged += HandleVicinity;
}
~NPCGroup()
{
if (GroupNPC1 == null) return;
GroupNPC1.OnPlayerVicinityChanged -= HandleVicinity;
GroupNPC1.OnPlayerVicinityChanged -= HandleVicinity;
}
private void HandleVicinity(bool isPlayerNear, ConvaiGroupNPCController npc)
{
if (isPlayerNear && !_isPlayerNearGroup)
{
_isPlayerNearGroup = true;
_vicinityChangedCallback?.Invoke(true, npc);
}
if (!isPlayerNear && _isPlayerNearGroup)
{
_isPlayerNearGroup = false;
_vicinityChangedCallback?.Invoke(false, npc);
}
}
public bool IsAnyoneTalking()
{
return GroupNPC1.ConvaiNPC.IsCharacterTalking || GroupNPC2.ConvaiNPC.IsCharacterTalking;
}
public bool BelongToGroup(ConvaiGroupNPCController controller)
{
return controller.CharacterID == GroupNPC1.CharacterID || controller.CharacterID == GroupNPC2.CharacterID;
}
public bool BothNPCAreNotNull()
{
return GroupNPC1 != null && GroupNPC2 != null;
}
public bool BothNPCAreNotActiveNPC()
{
ConvaiNPC activeNPC = ConvaiNPCManager.Instance.activeConvaiNPC;
string activeNPCId = activeNPC != null ? activeNPC.characterID : string.Empty;
return !GroupNPC1.CharacterID.Equals(activeNPCId) && !GroupNPC2.CharacterID.Equals(activeNPCId);
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPCGroup.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 6c1f1924c9c44d79a1c9abfaeb720f83
timeCreated: 1713324738

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 0c6246458db44b939c322f228563602e
timeCreated: 1712719472

37
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/ConvaiSpeechBubbleController.cs Normal file
View File

@ -0,0 +1,37 @@
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
public class ConvaiSpeechBubbleController : MonoBehaviour
{
private ConvaiGroupNPCController _convaiGroupNPC;
private NPCSpeechBubble _speechBubble;
private void OnDestroy()
{
_convaiGroupNPC.ShowSpeechBubble -= ConvaiNPC_ShowSpeechBubble;
_convaiGroupNPC.HideSpeechBubble -= ConvaiNPC_HideSpeechBubble;
Destroy(_speechBubble.gameObject);
_speechBubble = null;
}
public void Initialize(NPCSpeechBubble speechBubbleDisplay, ConvaiGroupNPCController convaiGroupNPC)
{
if (_speechBubble != null) return;
_speechBubble = Instantiate(speechBubbleDisplay, transform);
_convaiGroupNPC = convaiGroupNPC;
_convaiGroupNPC.ShowSpeechBubble += ConvaiNPC_ShowSpeechBubble;
_convaiGroupNPC.HideSpeechBubble += ConvaiNPC_HideSpeechBubble;
}
private void ConvaiNPC_HideSpeechBubble()
{
_speechBubble.HideSpeechBubble();
}
private void ConvaiNPC_ShowSpeechBubble(string text)
{
if(!string.IsNullOrEmpty(text)) _speechBubble.ShowSpeechBubble(text);
}
}
}

11
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/ConvaiSpeechBubbleController.cs.meta Normal file
View File

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: e4c8a9e9735c471493ad93d4f677e7de
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

11
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/ISpeechBubbleDisplay.cs Normal file
View File

@ -0,0 +1,11 @@
namespace Convai.Scripts.Runtime.Features
{
/// <summary>
/// Interface for displaying speech bubbles.
/// </summary>
public interface ISpeechBubbleDisplay
{
void ShowSpeechBubble(string text);
void HideSpeechBubble();
}
}

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/ISpeechBubbleDisplay.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 6c1403fe2d584d4abbb9e6588b8035f7
timeCreated: 1712719388

29
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/NPCSpeechBubble.cs Normal file
View File

@ -0,0 +1,29 @@
using TMPro;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
public class NPCSpeechBubble : MonoBehaviour, ISpeechBubbleDisplay
{
[SerializeField] private TMP_Text speechBubbleText;
[SerializeField] private Canvas speechBubbleCanvas;
/// <summary>
/// Show the speech bubble with the given text.
/// </summary>
/// <param name="text"> The text to display in the speech bubble. </param>
public void ShowSpeechBubble(string text)
{
speechBubbleText.text = text;
speechBubbleCanvas.enabled = true;
}
/// <summary>
/// Hide the speech bubble.
/// </summary>
public void HideSpeechBubble()
{
speechBubbleCanvas.enabled = false;
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/NPCSpeechBubble.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 8a26c5b33822429bb443f94d041a3374
timeCreated: 1712719408

12
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign.meta Normal file
View File

@ -0,0 +1,12 @@
fileFormatVersion: 2
<<<<<<<< HEAD:Assets/Convai/Scripts/Narrative Design.meta
guid: 0e7719b16b26ad348a9cd33d522adf6e
folderAsset: yes
========
guid: 3e8de7ddca083724bb9b286f52bbbc65
>>>>>>>> january-24-nightly:Assets/Convai/Convai Custom Unity Packages/ConvaiiOSBuild.unitypackage.meta
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

8
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Models.meta Normal file
View File

@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: b390e5b7c8d51a44fa4f560d93e8067f
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

54
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Models/SectionChangeEventsData.cs Normal file
View File

@ -0,0 +1,54 @@
using System;
using Convai.Scripts.Runtime.LoggerSystem;
using UnityEngine;
using UnityEngine.Events;
namespace Convai.Scripts.Runtime.Features
{
/// <summary>
/// Data class for Section Change Events
/// </summary>
[Serializable]
public class SectionChangeEventsData
{
[SerializeField] public string id;
[SerializeField] public UnityEvent onSectionStart;
[SerializeField] public UnityEvent onSectionEnd;
private NarrativeDesignManager _manager;
private string SectionName
{
get
{
if (_manager == null) return string.Empty;
SectionData sectionData = _manager.sectionDataList.Find(s => s.sectionId == id);
return sectionData?.sectionName ?? "Unknown Section";
}
}
/// <summary>
/// Initialize the Section Change Events
/// </summary>
/// <param name="manager"> The Narrative Design Manager </param>
public void Initialize(NarrativeDesignManager manager)
{
_manager = manager;
onSectionStart.RemoveListener(LogSectionStart);
onSectionStart.AddListener(LogSectionStart);
onSectionEnd.RemoveListener(LogSectionEnd);
onSectionEnd.AddListener(LogSectionEnd);
}
private void LogSectionStart()
{
ConvaiLogger.DebugLog($"Section {SectionName} started", ConvaiLogger.LogCategory.Character);
}
private void LogSectionEnd()
{
ConvaiLogger.DebugLog($"Section {SectionName} ended", ConvaiLogger.LogCategory.Character);
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Models/SectionChangeEventsData.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 32b4f9b2242945c19917f76c7ceb0a71
timeCreated: 1707390195

38
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Models/SectionData.cs Normal file
View File

@ -0,0 +1,38 @@
using System;
using Convai.Scripts.Runtime.Attributes;
using Newtonsoft.Json;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
/// <summary>
/// Data class for Section Data
/// </summary>
[Serializable]
public class SectionData
{
[JsonProperty("section_id")] [ReadOnly] [SerializeField]
public string sectionId;
[JsonProperty("section_name")] [ReadOnly] [SerializeField]
public string sectionName;
[JsonProperty("bt_constants")] [HideInInspector] [SerializeField]
public string behaviorTreeConstants;
[JsonProperty("objective")] [ReadOnly] [SerializeField]
public string objective;
[JsonProperty("character_id")] [ReadOnly] [HideInInspector] [SerializeField]
public string characterId;
[JsonProperty("decisions")] [ReadOnly] public object Decisions;
[JsonProperty("parents")] [ReadOnly] public object Parents;
[JsonProperty("triggers")] [ReadOnly] public object Triggers;
[JsonProperty("updated_character_data")] [ReadOnly]
public object UpdatedCharacterData;
}
}

3
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Models/SectionData.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 98ee0e1dea8bb8c4ba9fe2daf30e9960
timeCreated: 1706764030

26
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Models/TriggerData.cs Normal file
View File

@ -0,0 +1,26 @@
using System;
using Convai.Scripts.Runtime.Attributes;
using Newtonsoft.Json;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
[Serializable]
public class TriggerData
{
[JsonProperty("trigger_id")] [ReadOnly] [SerializeField]
public string triggerId;
[JsonProperty("trigger_name")] [ReadOnly] [SerializeField]
public string triggerName;
[JsonProperty("trigger_message")] [ReadOnly] [SerializeField]
public string triggerMessage;
[JsonProperty("destination_section")] [ReadOnly] [SerializeField]
public string destinationSection;
[JsonProperty("character_id")] [HideInInspector] [SerializeField]
public string characterId;
}
}

3
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Models/TriggerData.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: ed95a107485d00746a551222f53d0950
timeCreated: 1706764014

3
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Runtime.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: ecfb3b2eea944946a1d2336205b5eaad
timeCreated: 1706853590

151
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Runtime/NarrativeDesignAPI.cs Normal file
View File

@ -0,0 +1,151 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading.Tasks;
using Convai.Scripts.Runtime.LoggerSystem;
using Newtonsoft.Json;
namespace Convai.Scripts.Runtime.Features
{
/// <summary>
/// API client for the Narrative Design API.
/// </summary>
public class NarrativeDesignAPI
{
private const string BASE_URL = "https://api.convai.com/character/narrative/";
private readonly HttpClient _httpClient;
/// <summary>
/// Initializes a new instance of the <see cref="NarrativeDesignAPI" /> class.
/// </summary>
public NarrativeDesignAPI()
{
_httpClient = new HttpClient
{
// Set a default request timeout if needed
Timeout = TimeSpan.FromSeconds(30) // Example: 30 seconds
};
// Get the API key from the ConvaiAPIKeySetup object
if (ConvaiAPIKeySetup.GetAPIKey(out string apiKey))
{
// Set default request headers here
_httpClient.DefaultRequestHeaders.Add("CONVAI-API-KEY", apiKey);
// Set default headers like Accept to expect a JSON response
_httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
}
}
public async Task<string> CreateSectionAsync(string characterId, string objective, string sectionName, string behaviorTreeCode = null, string btConstants = null)
{
string endpoint = "create-section";
HttpContent content = CreateHttpContent(new Dictionary<string, object>
{
{ "character_id", characterId },
{ "objective", objective },
{ "section_name", sectionName },
{ "behavior_tree_code", behaviorTreeCode },
{ "bt_constants", btConstants }
});
return await SendPostRequestAsync(endpoint, content);
}
public async Task<string> GetSectionAsync(string characterId, string sectionId)
{
string endpoint = "get-section";
HttpContent content = CreateHttpContent(new Dictionary<string, object>
{
{ "character_id", characterId },
{ "section_id", sectionId }
});
return await SendPostRequestAsync(endpoint, content);
}
/// <summary>
/// Get a list of sections for a character.
/// </summary>
/// <param name="characterId"> The character ID. </param>
/// <returns> A JSON string containing the list of sections. </returns>
public async Task<string> ListSectionsAsync(string characterId)
{
string endpoint = "list-sections";
HttpContent content = CreateHttpContent(new Dictionary<string, object>
{
{ "character_id", characterId }
});
return await SendPostRequestAsync(endpoint, content);
}
public async Task<string> CreateTriggerAsync(string characterId, string triggerName, string triggerMessage = null, string destinationSection = null)
{
string endpoint = "create-trigger";
HttpContent content = CreateHttpContent(new Dictionary<string, object>
{
{ "character_id", characterId },
{ "trigger_message", triggerMessage },
{ "destination_section", destinationSection }
});
return await SendPostRequestAsync(endpoint, content);
}
public async Task<string> GetTriggerAsync(string characterId, string triggerId)
{
string endpoint = "get-trigger";
HttpContent content = CreateHttpContent(new Dictionary<string, object>
{
{ "character_id", characterId },
{ "trigger_id", triggerId }
});
return await SendPostRequestAsync(endpoint, content);
}
/// <summary>
/// Get a list of triggers for a character.
/// </summary>
/// <param name="characterId"> The character ID. </param>
/// <returns> A JSON string containing the list of triggers. </returns>
public async Task<string> GetTriggerListAsync(string characterId)
{
string endpoint = "list-triggers";
HttpContent content = CreateHttpContent(new Dictionary<string, object>
{
{ "character_id", characterId }
});
return await SendPostRequestAsync(endpoint, content);
}
private static HttpContent CreateHttpContent(Dictionary<string, object> data)
{
//Dictionary where all values are not null
Dictionary<string, object> dataToSend =
data.Where(keyValuePair => keyValuePair.Value != null).ToDictionary(keyValuePair => keyValuePair.Key, keyValuePair => keyValuePair.Value);
// Serialize the dictionary to JSON
string json = JsonConvert.SerializeObject(dataToSend);
// Convert JSON to HttpContent
return new StringContent(json, Encoding.UTF8, "application/json");
}
private async Task<string> SendPostRequestAsync(string endpoint, HttpContent content)
{
try
{
HttpResponseMessage response = await _httpClient.PostAsync(BASE_URL + endpoint, content);
response.EnsureSuccessStatusCode();
return await response.Content.ReadAsStringAsync();
}
catch (HttpRequestException e)
{
ConvaiLogger.Exception($"Request to {endpoint} failed: {e.Message}", ConvaiLogger.LogCategory.GRPC);
return null;
}
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Runtime/NarrativeDesignAPI.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 13850aaec3d742c2867a15ab2273a1e0
timeCreated: 1706546525

42
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Runtime/NarrativeDesignKeyController.cs Normal file
View File

@ -0,0 +1,42 @@
using System;
using System.Collections.Generic;
using System.Linq;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
public class NarrativeDesignKeyController : MonoBehaviour
{
public List<NarrativeDesignKey> narrativeDesignKeys;
[Serializable]
public class NarrativeDesignKey
{
public string name;
public string value;
}
public void SetTemplateKey(Dictionary<string, string> keyValuePairs)
{
narrativeDesignKeys.Clear();
narrativeDesignKeys.AddRange(from item in keyValuePairs
select new NarrativeDesignKey { name = item.Key, value = item.Value });
}
public void AddTemplateKey(string name, string value)
{
narrativeDesignKeys.Add(new NarrativeDesignKey { name = name, value = value });
}
public void RemoveTemplateKey(string name)
{
NarrativeDesignKey reference = narrativeDesignKeys.Find(x => x.name == name);
if(reference == null) return;
narrativeDesignKeys.Remove(reference);
}
public void UpdateTemplateKey(string name, string value)
{
NarrativeDesignKey reference = narrativeDesignKeys.Find(x => x.name == name);
if (reference == null) return;
reference.value = value;
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Runtime/NarrativeDesignKeyController.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: d526d030017744949b7412a85e6134d3
timeCreated: 1725884181

154
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Runtime/NarrativeDesignManager.cs Normal file
View File

@ -0,0 +1,154 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Convai.Scripts.Runtime.Core;
using Convai.Scripts.Runtime.LoggerSystem;
using Newtonsoft.Json;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
/// <summary>
/// Manages the narrative design for a ConvaiNPC.
/// </summary>
[RequireComponent(typeof(ConvaiNPC))]
public class NarrativeDesignManager : MonoBehaviour
{
public List<SectionChangeEventsData> sectionChangeEventsDataList = new();
public List<SectionData> sectionDataList = new();
public List<TriggerData> triggerDataList = new();
private ConvaiNPC _convaiNpc;
private string _currentSectionID;
private NarrativeDesignAPI _narrativeDesignAPI;
private NarrativeDesignAPI NarrativeDesignAPI => _narrativeDesignAPI ??= new NarrativeDesignAPI();
private ConvaiNPC ConvaiNpc => _convaiNpc ??= GetComponent<ConvaiNPC>();
private string CharacterID => ConvaiNpc.characterID;
private async void Awake()
{
_convaiNpc = GetComponent<ConvaiNPC>();
await Task.WhenAll(UpdateSectionListAsync(), UpdateTriggerListAsync());
}
private async void Reset()
{
await Task.WhenAll(UpdateSectionListAsync(), UpdateTriggerListAsync());
}
/// <summary>
/// Updates the section list from the server.
/// </summary>
public async Task UpdateSectionListAsync()
{
List<SectionData> updatedSectionList = await GetSectionListFromServerAsync();
UpdateSectionDataList(updatedSectionList);
}
/// <summary>
/// Updates the trigger list from the server.
/// </summary>
public async Task UpdateTriggerListAsync()
{
await ListTriggersAsync(CharacterID);
}
/// <summary>
/// Invoked when the section event list changes.
/// </summary>
public void OnSectionEventListChange()
{
foreach (SectionChangeEventsData sectionChangeEventsData in sectionChangeEventsDataList) sectionChangeEventsData.Initialize(this);
}
private async Task<List<SectionData>> GetSectionListFromServerAsync()
{
try
{
string sections = await NarrativeDesignAPI.ListSectionsAsync(CharacterID);
return JsonConvert.DeserializeObject<List<SectionData>>(sections);
}
catch (Exception e)
{
ConvaiLogger.Error($"Please setup API Key properly. FormatException occurred: {e.Message}", ConvaiLogger.LogCategory.Character);
throw;
}
}
public event Action OnTriggersUpdated;
private async Task ListTriggersAsync(string characterId)
{
try
{
string triggers = await NarrativeDesignAPI.GetTriggerListAsync(characterId);
triggerDataList = JsonConvert.DeserializeObject<List<TriggerData>>(triggers);
OnTriggersUpdated?.Invoke();
}
catch (FormatException e)
{
ConvaiLogger.Exception($"Format Exception occurred: {e.Message}", ConvaiLogger.LogCategory.Character);
throw;
}
}
/// <summary>
/// Updates the current section.
/// </summary>
/// <param name="sectionID"> The section ID to update to. </param>
public void UpdateCurrentSection(string sectionID)
{
if (string.IsNullOrEmpty(_currentSectionID))
{
_currentSectionID = sectionID;
InvokeSectionEvent(_currentSectionID, true);
return;
}
if (_currentSectionID.Equals(sectionID))
return;
InvokeSectionEvent(_currentSectionID, false);
_currentSectionID = sectionID;
InvokeSectionEvent(_currentSectionID, true);
}
private void InvokeSectionEvent(string id, bool isStarting)
{
SectionChangeEventsData sectionChangeEventsData = sectionChangeEventsDataList.Find(x => x.id == id);
if (sectionChangeEventsData == null)
{
ConvaiLogger.Info($"No Section Change Events have been created for sectionID: {id}", ConvaiLogger.LogCategory.Actions);
return;
}
if (isStarting)
sectionChangeEventsData.onSectionStart?.Invoke();
else
sectionChangeEventsData.onSectionEnd?.Invoke();
}
private void UpdateSectionDataList(List<SectionData> updatedSectionList)
{
Dictionary<string, SectionData> updatedSectionDictionary = updatedSectionList.ToDictionary(s => s.sectionId);
// Remove sections that no longer exist
sectionDataList.RemoveAll(currentSection => !updatedSectionDictionary.ContainsKey(currentSection.sectionId));
foreach (SectionData currentSection in sectionDataList.ToList())
if (updatedSectionDictionary.TryGetValue(currentSection.sectionId, out SectionData updatedSection))
{
currentSection.sectionName = updatedSection.sectionName;
currentSection.objective = updatedSection.objective;
updatedSectionDictionary.Remove(currentSection.sectionId);
}
foreach (SectionData newSection in updatedSectionDictionary.Values) sectionDataList.Add(newSection);
foreach (SectionChangeEventsData sectionChangeEvent in sectionChangeEventsDataList) sectionChangeEvent.Initialize(this);
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Runtime/NarrativeDesignManager.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 3d56bc33052944e0aba5a0fe7073c3ab
timeCreated: 1706546743

77
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Runtime/NarrativeDesignTrigger.cs Normal file
View File

@ -0,0 +1,77 @@
using System.Collections.Generic;
using System.Linq;
using Convai.Scripts.Runtime.Core;
using UnityEngine;
using UnityEngine.Events;
namespace Convai.Scripts.Runtime.Features
{
public class NarrativeDesignTrigger : MonoBehaviour
{
public ConvaiNPC convaiNPC;
[HideInInspector] public int selectedTriggerIndex;
[HideInInspector] public List<string> availableTriggers = new();
public UnityEvent onTriggerEvent;
private NarrativeDesignManager _narrativeDesignManager;
private void Awake()
{
UpdateNarrativeDesignManager();
}
private void OnTriggerEnter(Collider other)
{
if (other.gameObject.CompareTag("Player")) InvokeSelectedTrigger();
}
private void OnValidate()
{
UpdateNarrativeDesignManager();
}
private void UpdateNarrativeDesignManager()
{
if (convaiNPC != null)
{
_narrativeDesignManager = convaiNPC.GetComponent<NarrativeDesignManager>();
if (_narrativeDesignManager != null) UpdateAvailableTriggers();
}
else
{
availableTriggers.Clear();
selectedTriggerIndex = -1;
}
}
public void UpdateAvailableTriggers()
{
if (_narrativeDesignManager != null)
{
availableTriggers = _narrativeDesignManager.triggerDataList.Select(trigger => trigger.triggerName).ToList();
if (selectedTriggerIndex >= availableTriggers.Count) selectedTriggerIndex = availableTriggers.Count - 1;
}
}
public void InvokeSelectedTrigger()
{
if (convaiNPC != null && availableTriggers != null && selectedTriggerIndex >= 0 && selectedTriggerIndex < availableTriggers.Count)
{
string selectedTriggerName = availableTriggers[selectedTriggerIndex];
ConvaiNPCManager.Instance.SetActiveConvaiNPC(convaiNPC, false);
onTriggerEvent?.Invoke();
convaiNPC.TriggerEvent(selectedTriggerName);
}
}
public void InvokeSpeech(string message)
{
if (convaiNPC != null && !string.IsNullOrEmpty(message))
{
ConvaiNPCManager.Instance.SetActiveConvaiNPC(convaiNPC, false);
onTriggerEvent?.Invoke();
convaiNPC.TriggerSpeech(message);
}
}
}
}

11
Assets/Convai/Scripts/Runtime/Features/NarrativeDesign/Runtime/NarrativeDesignTrigger.cs.meta Normal file
View File

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 1c783ed73e64a204ebf38b93c054b566
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant: