tomh104/Virtual-Tutor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial upload

Browse Source
This commit is contained in:
tom.hempel
2025-09-30 17:58:33 +02:00
commit 69b0c79692
4818 changed files with 229318 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

243
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/ConvaiGroupNPCController.cs Normal file
View File

@ -0,0 +1,243 @@
using System;
using System.Collections;
using System.Threading.Tasks;
using Convai.Scripts.Runtime.Core;
using Convai.Scripts.Runtime.Features.LipSync;
using Convai.Scripts.Runtime.LoggerSystem;
using Service;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
/// <summary>
/// This class is responsible for handling out all the tasks related to NPC to NPC conversation for a NPC of a group
/// </summary>
public class ConvaiGroupNPCController : MonoBehaviour
{
/// <summary>
/// Used to set Player GameObject Transform and lip-sync
/// </summary>
private void Awake()
{
if (playerTransform == null) playerTransform = Camera.main.transform;
ConvaiNPC = GetComponent<ConvaiNPC>();
CONVERSATION_DISTANCE_THRESHOLD = conversationDistanceThreshold == 0 ? Mathf.Infinity : conversationDistanceThreshold;
TryGetComponent(out _lipSync);
}
/// <summary>
/// Starts coroutine for player vicinity check and subscribe to necessary events
/// </summary>
private void Start()
{
_npcGroup = NPC2NPCConversationManager.Instance.npcGroups.Find(c => c.BelongToGroup(this));
otherNPC = _npcGroup.GroupNPC1 == this ? _npcGroup.GroupNPC2 : _npcGroup.GroupNPC1;
_checkPlayerVicinityCoroutine = StartCoroutine(CheckPlayerVicinity());
if (TryGetComponent(out ConvaiNPCAudioManager convaiNPCAudio)) convaiNPCAudio.OnAudioTranscriptAvailable += HandleAudioTranscriptAvailable;
}
/// <summary>
/// Unsubscribes to the events and stops the coroutine
/// </summary>
private void OnDestroy()
{
if (TryGetComponent(out ConvaiNPCAudioManager convaiNPCAudio)) convaiNPCAudio.OnAudioTranscriptAvailable -= HandleAudioTranscriptAvailable;
if(_npc2NPCGrpcClient!=null) _npc2NPCGrpcClient.OnTranscriptAvailable -= HandleTranscriptAvailable;
if (_checkPlayerVicinityCoroutine != null) StopCoroutine(_checkPlayerVicinityCoroutine);
}
/// <summary>
/// Shows speech bubble and adds the received text to final transcript
/// </summary>
/// <param name="transcript"></param>
private void HandleAudioTranscriptAvailable(string transcript)
{
if (IsInConversationWithAnotherNPC)
ShowSpeechBubble?.Invoke(transcript);
}
private void HandleTranscriptAvailable(string transcript, ConvaiGroupNPCController npcController)
{
if (npcController != this) return;
_finalResponseText += transcript;
}
/// <summary>
/// Attaches the speech bubble to the NPC game-object
/// </summary>
public void AttachSpeechBubble()
{
if (TryGetComponent(out ConvaiSpeechBubbleController _)) return;
gameObject.AddComponent<ConvaiSpeechBubbleController>().Initialize(speechBubblePrefab, this);
}
/// <summary>
/// Destroys the speech bubble game-object
/// </summary>
public void DetachSpeechBubble()
{
if (TryGetComponent(out ConvaiSpeechBubbleController convaiSpeechBubble)) Destroy(convaiSpeechBubble);
}
/// <summary>
/// Store the references of the client and subscribe to the necessary events
/// </summary>
/// <param name="client"></param>
public void InitializeNpc2NpcGrpcClient(NPC2NPCGRPCClient client)
{
_npc2NPCGrpcClient = client;
_npc2NPCGrpcClient.OnTranscriptAvailable += HandleTranscriptAvailable;
}
/// <summary>
/// Every 0.5 seconds updates if player is near or not and fire events according to the state
/// </summary>
/// <returns></returns>
private IEnumerator CheckPlayerVicinity()
{
bool previousState = false;
Vector3 previousPlayerPosition = Vector3.zero;
yield return new WaitForSeconds(0.1f);
while (true)
{
Vector3 currentPlayerPosition = playerTransform.transform.position;
// Check if the player has moved more than a certain threshold distance
if (Vector3.Distance(previousPlayerPosition, currentPlayerPosition) > PLAYER_MOVE_THRESHOLD)
{
// Calculate the distance between the NPC and the player
float distanceToPlayer = Vector3.Distance(transform.position, currentPlayerPosition);
// Check if the player is within the threshold distance
bool isPlayerCurrentlyNear = distanceToPlayer <= CONVERSATION_DISTANCE_THRESHOLD;
// If the player's current vicinity state is different from the previous state, raise the event
if (isPlayerCurrentlyNear != previousState && !ConvaiNPC.isCharacterActive)
{
OnPlayerVicinityChanged?.Invoke(isPlayerCurrentlyNear, this);
previousState = isPlayerCurrentlyNear; // Update the previous state
ConvaiLogger.Info($"Player is currently near {ConvaiNPC.characterName}: {isPlayerCurrentlyNear}", ConvaiLogger.LogCategory.Character);
}
previousPlayerPosition = currentPlayerPosition; // Update the player's previous position
// Check every half second
}
yield return new WaitForSeconds(0.5f);
}
}
/// <summary>
/// Sends the text to the other NPC in the group
/// </summary>
/// <param name="message"></param>
public async void SendTextDataNPC2NPC(string message)
{
if (_npc2NPCGrpcClient == null)
{
ConvaiLogger.Warn("No GRPC client initialized for this NPC.", ConvaiLogger.LogCategory.Character);
return;
}
try
{
CanRelayMessage = false;
await Task.Delay(500);
await _npc2NPCGrpcClient.SendTextData(
message,
ConvaiNPC.characterID,
ConvaiNPC.sessionID,
_lipSync != null,
FaceModel,
this);
}
catch (Exception ex)
{
ConvaiLogger.Warn($"Error sending message data for NPC2NPC: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
public void EndOfResponseReceived()
{
if (TryGetComponent(out ConvaiNPCAudioManager convaiNPCAudio)) convaiNPCAudio.OnCharacterTalkingChanged += SendFinalTranscriptToOtherNPC;
ConversationManager.RelayMessage(_finalResponseText, this);
_finalResponseText = "";
}
private void SendFinalTranscriptToOtherNPC(bool isTalking)
{
if (IsInConversationWithAnotherNPC)
{
if (!isTalking)
{
ConversationManager.SwitchSpeaker(this);
if (TryGetComponent(out ConvaiNPCAudioManager convaiNPCAudio)) convaiNPCAudio.OnCharacterTalkingChanged -= SendFinalTranscriptToOtherNPC;
HideSpeechBubble?.Invoke();
}
else
{
ConvaiLogger.DebugLog($"{ConvaiNPC.characterName} is currently still talking. ", ConvaiLogger.LogCategory.Character);
}
}
}
public bool IsPlayerNearMe()
{
bool result = Vector3.Distance(transform.position, playerTransform.position) < CONVERSATION_DISTANCE_THRESHOLD;
ConvaiLogger.Info($"Player is near {CharacterName}: {result}", ConvaiLogger.LogCategory.Character);
return result;
}
public bool IsOtherNPCTalking()
{
return otherNPC.ConvaiNPC.IsCharacterTalking;
}
#region Serialized Fields
[Tooltip("The prefab for the speech bubble to display above the NPC. [Optional]")] [SerializeField]
private NPCSpeechBubble speechBubblePrefab;
[Tooltip("Attach the Main Player Transform here so that distance check can be performed")] [SerializeField]
private Transform playerTransform;
// The distance from the NPC to the player when the NPC will start talking
[Tooltip("The distance from the NPC to the player when the NPC will start talking. Set to 0 to disable this feature. [Optional]")] [SerializeField] [Range(0f, 100f)]
private float conversationDistanceThreshold = 5.0f;
#endregion
#region Private Attributes
private const float PLAYER_MOVE_THRESHOLD = 0.5f;
private float CONVERSATION_DISTANCE_THRESHOLD;
private string _finalResponseText = "";
private NPC2NPCGRPCClient _npc2NPCGrpcClient;
private ConvaiLipSync _lipSync;
private Coroutine _checkPlayerVicinityCoroutine;
private FaceModel FaceModel => _lipSync == null ? FaceModel.OvrModelName : _lipSync.faceModel;
#endregion
#region Events
public event Action<string> ShowSpeechBubble;
public event Action HideSpeechBubble;
public event Action<bool, ConvaiGroupNPCController> OnPlayerVicinityChanged;
private NPCGroup _npcGroup;
private ConvaiGroupNPCController otherNPC;
#endregion
#region Public Attributes
public bool CanRelayMessage { get; set; } = true;
public NPC2NPCConversationManager ConversationManager { get; set; }
public bool IsInConversationWithAnotherNPC { get; set; }
public string CharacterName => ConvaiNPC == null ? string.Empty : ConvaiNPC.characterName;
public string CharacterID => ConvaiNPC == null ? string.Empty : ConvaiNPC.characterID;
public ConvaiNPC ConvaiNPC { get; private set; }
#endregion
}
}

11
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/ConvaiGroupNPCController.cs.meta Normal file
View File

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 74d3f2dd15d074a429eeea8c31f80b13
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

302
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPC2NPCConversationManager.cs Normal file
View File

@ -0,0 +1,302 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using Convai.Scripts.Runtime.Core;
using Convai.Scripts.Runtime.LoggerSystem;
using Grpc.Core;
using Service;
using UnityEngine;
using Random = UnityEngine.Random;
namespace Convai.Scripts.Runtime.Features
{
/// <summary>
/// Manages the conversation between two Convai powered NPC groups
/// </summary>
public class NPC2NPCConversationManager : MonoBehaviour
{
private const string GRPC_API_ENDPOINT = "stream.convai.com";
public static NPC2NPCConversationManager Instance;
public List<NPCGroup> npcGroups;
private readonly List<NPCGroup> _groupsWhereConversationEnded = new();
private string _apiKey = string.Empty;
private void Awake()
{
if (Instance == null)
Instance = this;
else
Destroy(gameObject);
LoadApiKey();
}
private void Start()
{
foreach (NPCGroup group in npcGroups)
group.Initialize(HandlePlayerVicinityChanged);
StartConversationWithAllNPCs();
}
/// <summary>
/// Handles the event of player vicinity change.
/// </summary>
/// <param name="isPlayerNear">Indicates if the player is near.</param>
/// <param name="npc">The NPC for which the vicinity changed.</param>
private void HandlePlayerVicinityChanged(bool isPlayerNear, ConvaiGroupNPCController npc)
{
if (isPlayerNear)
ResumeConversation(npc);
}
/// <summary>
/// Processes the message from the sender NPC.
/// </summary>
/// <param name="sender">The NPC sending the message.</param>
/// <param name="topic">The topic of the conversation.</param>
/// <param name="message">The message to be processed.</param>
/// <returns>The processed message.</returns>
private string ProcessMessage(ConvaiGroupNPCController sender, string topic, string message)
{
string processedMessage = $"{sender.CharacterName} said \"{message}\" to you. Reply to it. ";
processedMessage += Random.Range(0, 2) == 0
? $"Talk about something related to {message}. "
: $"Talk about something other than \"{message}\" but related to {topic}. Gently change the conversation topic. ";
return processedMessage + "Definitely, reply to the message. Dont address speaker. Keep the reply short. Do not repeat the same message, or keep asking same question.";
}
/// <summary>
/// Relays the message from the sender NPC.
/// </summary>
/// <param name="message">The message to be relayed.</param>
/// <param name="sender">The NPC sending the message.</param>
/// <param name="performSwitch"></param>
public void RelayMessage(string message, ConvaiGroupNPCController sender)
{
NPCGroup npcGroup = npcGroups.Find(c => c.BelongToGroup(sender));
if (npcGroup == null)
{
ConvaiLogger.Warn("Conversation not found for the sender.", ConvaiLogger.LogCategory.Character);
return;
}
npcGroup.messageToRelay = message;
if (!npcGroup.CurrentSpeaker.IsPlayerNearMe()) return;
StartCoroutine(RelayMessageCoroutine(message, sender, npcGroup));
}
/// <summary>
/// Coroutine to relay the message from the sender NPC.
/// </summary>
/// <param name="message">The message to be relayed.</param>
/// <param name="npcGroup"> The NPC group to relay the message to. </param>
/// <returns>An IEnumerator to be used in a coroutine.</returns>
private IEnumerator RelayMessageCoroutine(string message, ConvaiGroupNPCController sender, NPCGroup npcGroup)
{
yield return new WaitForSeconds(0.5f);
ConvaiGroupNPCController receiver = npcGroup.GroupNPC1 == sender ? npcGroup.GroupNPC2 : npcGroup.GroupNPC1;
while (receiver.ConvaiNPC.IsCharacterTalking)
{
yield return new WaitForSeconds(0.1f);
}
try
{
ConvaiLogger.DebugLog($"Relaying message from {sender.CharacterName} to {receiver.CharacterName}: {message}", ConvaiLogger.LogCategory.Character);
string processedMessage = ProcessMessage(receiver, npcGroup.topic, message);
receiver.SendTextDataNPC2NPC(processedMessage);
}
catch (Exception e)
{
ConvaiLogger.Warn($"Failed to relay message: {e.Message}", ConvaiLogger.LogCategory.Character);
}
}
/// <summary>
/// Switches the speaker in the conversation.
/// </summary>
/// <param name="currentSpeaker">The current speaker NPC.</param>
/// <returns>The new speaker NPC.</returns>
public void SwitchSpeaker(ConvaiGroupNPCController currentSpeaker)
{
NPCGroup group = npcGroups.Find(g => g.CurrentSpeaker == currentSpeaker);
if (group != null)
{
group.CurrentSpeaker = currentSpeaker == group.GroupNPC1 ? group.GroupNPC2 : group.GroupNPC1;
ConvaiLogger.DebugLog($"Switching NPC2NPC Speaker to {group.CurrentSpeaker}", ConvaiLogger.LogCategory.Character);
return;
}
ConvaiLogger.Warn("Failed to switch speaker. Current speaker not found in any group.", ConvaiLogger.LogCategory.Character);
}
private void LoadApiKey()
{
ConvaiAPIKeySetup.GetAPIKey(out _apiKey);
}
/// <summary>
/// Initializes a single NPC for conversation.
/// </summary>
/// <param name="npc">The NPC to initialize.</param>
/// <param name="grpcClient">The GRPC client to use for the NPC.</param>
private void InitializeNPC(ConvaiGroupNPCController npc, NPC2NPCGRPCClient grpcClient)
{
if (npc == null)
{
ConvaiLogger.Warn("The given NPC is null.", ConvaiLogger.LogCategory.Character);
return;
}
npc.ConversationManager = this;
npc.InitializeNpc2NpcGrpcClient(grpcClient);
npc.AttachSpeechBubble();
npc.IsInConversationWithAnotherNPC = true;
npc.ConvaiNPC.isCharacterActive = false;
}
/// <summary>
/// Starts the conversation for the given NPC.
/// </summary>
/// <param name="npcGroup">The NPC to start the conversation for.</param>
private void InitializeNPCGroup(NPCGroup npcGroup)
{
if (npcGroup == null)
{
ConvaiLogger.Warn("The given NPC is not part of any group.", ConvaiLogger.LogCategory.Character);
return;
}
ConvaiGroupNPCController npc1 = npcGroup.GroupNPC1;
ConvaiGroupNPCController npc2 = npcGroup.GroupNPC2;
if (npc1.IsInConversationWithAnotherNPC || npc2.IsInConversationWithAnotherNPC)
{
ConvaiLogger.Warn($"{npc1.CharacterName} or {npc2.CharacterName} is already in a conversation.", ConvaiLogger.LogCategory.Character);
return;
}
NPC2NPCGRPCClient grpcClient = CreateAndInitializeGRPCClient(npcGroup);
// Initialize both NPCs
InitializeNPC(npc1, grpcClient);
InitializeNPC(npc2, grpcClient);
npcGroup.CurrentSpeaker = Random.Range(0, 10) % 2 == 0 ? npc1 : npc2;
}
/// <summary>
/// Creates and initializes a new GRPC client for the given NPC group.
/// </summary>
/// <param name="group">The NPC group to create the GRPC client for.</param>
/// <returns>The initialized GRPC client.</returns>
private NPC2NPCGRPCClient CreateAndInitializeGRPCClient(NPCGroup group)
{
GameObject grpcClientGameObject = new($"GRPCClient_{group.GroupNPC1.CharacterID}_{group.GroupNPC2.CharacterID}")
{
transform = { parent = transform }
};
NPC2NPCGRPCClient grpcClient = grpcClientGameObject.AddComponent<NPC2NPCGRPCClient>();
ConvaiService.ConvaiServiceClient serviceClient = CreateNewConvaiServiceClient();
grpcClient.Initialize(_apiKey, serviceClient, group);
return grpcClient;
}
/// <summary>
/// Creates a new ConvaiServiceClient.
/// </summary>
/// <returns> The new ConvaiServiceClient. </returns>
private ConvaiService.ConvaiServiceClient CreateNewConvaiServiceClient()
{
try
{
SslCredentials credentials = new();
List<ChannelOption> options = new()
{
new ChannelOption(ChannelOptions.MaxReceiveMessageLength, 16 * 1024 * 1024)
};
Channel channel = new(GRPC_API_ENDPOINT, credentials, options);
return new ConvaiService.ConvaiServiceClient(channel);
}
catch (Exception ex)
{
ConvaiLogger.Error($"Failed to create ConvaiServiceClient: {ex.Message}", ConvaiLogger.LogCategory.Character);
throw;
}
}
/// <summary>
/// Resumes the conversation for the given NPC.
/// </summary>
/// <param name="sender"> The NPC to resume the conversation for. </param>
private void ResumeConversation(ConvaiGroupNPCController sender)
{
NPCGroup npcGroup = npcGroups.Find(g => g.BelongToGroup(sender));
if (npcGroup.IsAnyoneTalking()) return;
if (_groupsWhereConversationEnded.Contains(npcGroup))
{
InitializeNPCGroup(npcGroup);
_groupsWhereConversationEnded.Remove(npcGroup);
}
if (string.IsNullOrEmpty(npcGroup.messageToRelay))
{
string message = $"Talk about {npcGroup.topic}.";
npcGroup.CurrentSpeaker.SendTextDataNPC2NPC(message);
npcGroup.messageToRelay = message;
ConvaiLogger.DebugLog($"Starting conversation for the first time between {npcGroup.GroupNPC1.CharacterName} and {npcGroup.GroupNPC2.CharacterName}",
ConvaiLogger.LogCategory.Character);
}
else
{
RelayMessage(npcGroup.messageToRelay, npcGroup.CurrentSpeaker);
ConvaiLogger.DebugLog($"Resuming conversation between {npcGroup.GroupNPC1.CharacterName} and {npcGroup.GroupNPC2.CharacterName}",
ConvaiLogger.LogCategory.Character);
}
}
/// <summary>
/// Ends the conversation for the given NPC.
/// </summary>
/// <param name="npc"> The NPC to end the conversation for. </param>
public void EndConversation(ConvaiGroupNPCController npc)
{
NPCGroup group = npcGroups.Find(g => g.BelongToGroup(npc));
ConvaiLogger.DebugLog($"Ending conversation between {group.GroupNPC1.CharacterName} and {group.GroupNPC2.CharacterName}", ConvaiLogger.LogCategory.Character);
void EndConversationForNPC(ConvaiGroupNPCController groupNPC)
{
groupNPC.IsInConversationWithAnotherNPC = false;
groupNPC.ConvaiNPC.InterruptCharacterSpeech();
groupNPC.GetComponent<ConvaiGroupNPCController>().DetachSpeechBubble();
}
EndConversationForNPC(group.GroupNPC1);
EndConversationForNPC(group.GroupNPC2);
_groupsWhereConversationEnded.Add(group);
ConvaiNPCManager.Instance.SetActiveConvaiNPC(npc.ConvaiNPC);
Destroy(transform.Find($"GRPCClient_{group.GroupNPC1.CharacterID}_{group.GroupNPC2.CharacterID}").gameObject);
}
/// <summary>
/// Starts the conversation with all NPCs.
/// </summary>
private void StartConversationWithAllNPCs()
{
IEnumerable filteredList = npcGroups
.Where(npcGroup => npcGroup.BothNPCAreNotNull())
.Where(npcGroup => npcGroup.BothNPCAreNotActiveNPC());
foreach (NPCGroup npcGroup in filteredList)
InitializeNPCGroup(npcGroup);
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPC2NPCConversationManager.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: e5c3956535ee405983974a5434b6f4a4
timeCreated: 1705687346

226
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPC2NPCGRPCClient.cs Normal file
View File

@ -0,0 +1,226 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Convai.Scripts.Runtime.Core;
using Convai.Scripts.Runtime.LoggerSystem;
using Convai.Scripts.Runtime.Utils;
using Grpc.Core;
using Service;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
/// <summary>
/// Represents an NPC2NPCGRPCClient that can be used to communicate with the ConvaiService using gRPC.
/// </summary>
public class NPC2NPCGRPCClient : MonoBehaviour
{
private readonly CancellationTokenSource _cancellationTokenSource = new();
private string _apiKey;
private ConvaiService.ConvaiServiceClient _client;
private NPCGroup _npcGroup;
public event Action<string, ConvaiGroupNPCController> OnTranscriptAvailable;
private void OnDestroy()
{
_cancellationTokenSource?.Cancel();
_cancellationTokenSource?.Dispose();
}
/// <summary>
/// Initializes the NPC2NPCGRPCClient with the given API key and ConvaiService client.
/// </summary>
/// <param name="apiKey">The API key to use for authentication.</param>
/// <param name="client">The ConvaiService client to use for communication.</param>
public void Initialize(string apiKey, ConvaiService.ConvaiServiceClient client, NPCGroup group)
{
_apiKey = apiKey;
_client = client;
_npcGroup = group;
}
/// <summary>
/// Creates an AsyncDuplexStreamingCall with the specified headers.
/// </summary>
/// <returns>An AsyncDuplexStreamingCall with the specified headers.</returns>
private AsyncDuplexStreamingCall<GetResponseRequest, GetResponseResponse> GetAsyncDuplexStreamingCallOptions()
{
Metadata headers = new()
{
{ "source", "Unity" },
{ "version", "3.2.1" }
};
CallOptions options = new(headers);
return _client.GetResponse(options);
}
/// <summary>
/// Sends the specified user text to the server and receives a response.
/// </summary>
/// <param name="userText">The user text to send to the server.</param>
/// <param name="characterID">The ID of the character to use for the request.</param>
/// <param name="sessionID">The ID of the session to use for the request.</param>
/// <param name="isLipSyncActive">Whether lip sync is active for the request.</param>
/// <param name="faceModel">The face model to use for the request.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public async Task SendTextData(string userText, string characterID, string sessionID, bool isLipSyncActive, FaceModel faceModel, ConvaiGroupNPCController npcController)
{
AsyncDuplexStreamingCall<GetResponseRequest, GetResponseResponse> call = GetAsyncDuplexStreamingCallOptions();
GetResponseRequest getResponseConfigRequest = CreateGetResponseRequest(characterID, sessionID, isLipSyncActive, faceModel, false, null);
try
{
await call.RequestStream.WriteAsync(getResponseConfigRequest);
await call.RequestStream.WriteAsync(new GetResponseRequest
{
GetResponseData = new GetResponseRequest.Types.GetResponseData
{
TextData = userText
}
});
await call.RequestStream.CompleteAsync();
Task receiveResultsTask = Task.Run(
async () => { await ReceiveResultFromServer(call, _cancellationTokenSource.Token, npcController); },
_cancellationTokenSource.Token);
await receiveResultsTask.ConfigureAwait(false);
}
catch (Exception)
{
// ignored
}
}
/// <summary>
/// Creates a GetResponseRequest with the specified parameters.
/// </summary>
/// <param name="characterID">The ID of the character to use for the request.</param>
/// <param name="sessionID">The ID of the session to use for the request.</param>
/// <param name="isLipSyncActive">Whether lip sync is active for the request.</param>
/// <param name="faceModel">The face model to use for the request.</param>
/// <param name="isActionActive">Whether action is active for the request.</param>
/// <param name="actionConfig">The action configuration to use for the request.</param>
/// <returns>A GetResponseRequest with the specified parameters.</returns>
private GetResponseRequest CreateGetResponseRequest(string characterID, string sessionID, bool isLipSyncActive, FaceModel faceModel, bool isActionActive,
ActionConfig actionConfig)
{
GetResponseRequest getResponseConfigRequest = new()
{
GetResponseConfig = new GetResponseRequest.Types.GetResponseConfig
{
CharacterId = characterID,
ApiKey = _apiKey,
SessionId = sessionID,
AudioConfig = new AudioConfig
{
EnableFacialData = isLipSyncActive,
FaceModel = faceModel
}
}
};
if (isActionActive)
getResponseConfigRequest.GetResponseConfig.ActionConfig = actionConfig;
return getResponseConfigRequest;
}
/// <summary>
/// Receives a response from the server asynchronously.
/// </summary>
/// <param name="call">The AsyncDuplexStreamingCall to use for receiving the response.</param>
/// <param name="cancellationToken">The cancellation token to use for cancelling the operation.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
private async Task ReceiveResultFromServer(AsyncDuplexStreamingCall<GetResponseRequest, GetResponseResponse> call, CancellationToken cancellationToken, ConvaiGroupNPCController npcController)
{
ConvaiLogger.Info("Receiving response from server", ConvaiLogger.LogCategory.Character);
Queue<LipSyncBlendFrameData> lipSyncBlendFrameQueue = new();
ConvaiNPC convaiNPC = npcController.ConvaiNPC;
bool firstSilFound = false;
while (!cancellationToken.IsCancellationRequested && await call.ResponseStream.MoveNext(cancellationToken).ConfigureAwait(false))
try
{
GetResponseResponse result = call.ResponseStream.Current;
// Process the received response here
if (result.AudioResponse != null)
if (result.AudioResponse.AudioData != null)
{
MainThreadDispatcher.Instance.RunOnMainThread(() => OnTranscriptAvailable?.Invoke(result.AudioResponse.TextData, npcController));
if (result.AudioResponse.AudioData.ToByteArray().Length > 46)
{
byte[] wavBytes = result.AudioResponse.AudioData.ToByteArray();
// will only work for wav files
if (convaiNPC.convaiLipSync == null)
{
ConvaiLogger.DebugLog($"Enqueuing responses: {result.AudioResponse.TextData}", ConvaiLogger.LogCategory.LipSync);
convaiNPC.EnqueueResponse(result);
}
else
{
LipSyncBlendFrameData.FrameType frameType =
convaiNPC.convaiLipSync.faceModel == FaceModel.OvrModelName
? LipSyncBlendFrameData.FrameType.Visemes
: LipSyncBlendFrameData.FrameType.Blendshape;
lipSyncBlendFrameQueue.Enqueue(
new LipSyncBlendFrameData(
(int)(WavUtility.CalculateDurationSeconds(wavBytes) * 30),
result,
frameType
)
);
}
}
if (result.AudioResponse.VisemesData != null)
if (convaiNPC.convaiLipSync != null)
{
//ConvaiLogger.Info(result.AudioResponse.VisemesData, ConvaiLogger.LogCategory.LipSync);
if (result.AudioResponse.VisemesData.Visemes.Sil == -2 || result.AudioResponse.EndOfResponse)
{
if (firstSilFound) lipSyncBlendFrameQueue.Dequeue().Process(convaiNPC);
firstSilFound = true;
}
else
{
lipSyncBlendFrameQueue.Peek().Enqueue(result.AudioResponse.VisemesData);
}
}
if (result.AudioResponse.BlendshapesData != null)
if (convaiNPC.convaiLipSync != null)
{
if (lipSyncBlendFrameQueue.Peek().CanProcess() || result.AudioResponse.EndOfResponse)
{
lipSyncBlendFrameQueue.Dequeue().Process(convaiNPC);
}
else
{
lipSyncBlendFrameQueue.Peek().Enqueue(result.AudioResponse.FaceEmotion.ArKitBlendShapes);
if (lipSyncBlendFrameQueue.Peek().CanPartiallyProcess()) lipSyncBlendFrameQueue.Peek().ProcessPartially(convaiNPC);
}
}
if (result.AudioResponse.EndOfResponse)
MainThreadDispatcher.Instance.RunOnMainThread(npcController.EndOfResponseReceived);
}
}
catch (RpcException rpcException)
{
if (rpcException.StatusCode == StatusCode.Cancelled)
ConvaiLogger.Error(rpcException, ConvaiLogger.LogCategory.Character);
else
throw;
}
catch (Exception ex)
{
ConvaiLogger.DebugLog(ex, ConvaiLogger.LogCategory.Character);
}
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPC2NPCGRPCClient.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: dc8dba94e7504aab878f30a19a55fa5a
timeCreated: 1708531595

77
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPCGroup.cs Normal file
View File

@ -0,0 +1,77 @@
using System;
using Convai.Scripts.Runtime.Attributes;
using Convai.Scripts.Runtime.Core;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
/// <summary>
/// A group of NPCs that are currently conversing with each other.
/// </summary>
[Serializable]
public class NPCGroup
{
[field: SerializeField] public ConvaiGroupNPCController GroupNPC1 { get; private set; }
[field: SerializeField] public ConvaiGroupNPCController GroupNPC2 { get; private set; }
public string topic;
[ReadOnly] public string messageToRelay;
private bool _isPlayerNearGroup;
private Action<bool, ConvaiGroupNPCController> _vicinityChangedCallback;
public ConvaiGroupNPCController CurrentSpeaker { get; set; }
public ConvaiGroupNPCController CurrentListener => CurrentSpeaker == GroupNPC1 ? GroupNPC2 : GroupNPC1;
public void Initialize(Action<bool, ConvaiGroupNPCController> vicinityChangedCallback)
{
_vicinityChangedCallback = vicinityChangedCallback;
if (GroupNPC1 == null) return;
GroupNPC1.OnPlayerVicinityChanged += HandleVicinity;
GroupNPC1.OnPlayerVicinityChanged += HandleVicinity;
}
~NPCGroup()
{
if (GroupNPC1 == null) return;
GroupNPC1.OnPlayerVicinityChanged -= HandleVicinity;
GroupNPC1.OnPlayerVicinityChanged -= HandleVicinity;
}
private void HandleVicinity(bool isPlayerNear, ConvaiGroupNPCController npc)
{
if (isPlayerNear && !_isPlayerNearGroup)
{
_isPlayerNearGroup = true;
_vicinityChangedCallback?.Invoke(true, npc);
}
if (!isPlayerNear && _isPlayerNearGroup)
{
_isPlayerNearGroup = false;
_vicinityChangedCallback?.Invoke(false, npc);
}
}
public bool IsAnyoneTalking()
{
return GroupNPC1.ConvaiNPC.IsCharacterTalking || GroupNPC2.ConvaiNPC.IsCharacterTalking;
}
public bool BelongToGroup(ConvaiGroupNPCController controller)
{
return controller.CharacterID == GroupNPC1.CharacterID || controller.CharacterID == GroupNPC2.CharacterID;
}
public bool BothNPCAreNotNull()
{
return GroupNPC1 != null && GroupNPC2 != null;
}
public bool BothNPCAreNotActiveNPC()
{
ConvaiNPC activeNPC = ConvaiNPCManager.Instance.activeConvaiNPC;
string activeNPCId = activeNPC != null ? activeNPC.characterID : string.Empty;
return !GroupNPC1.CharacterID.Equals(activeNPCId) && !GroupNPC2.CharacterID.Equals(activeNPCId);
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPCGroup.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 6c1f1924c9c44d79a1c9abfaeb720f83
timeCreated: 1713324738

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 0c6246458db44b939c322f228563602e
timeCreated: 1712719472

37
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/ConvaiSpeechBubbleController.cs Normal file
View File

@ -0,0 +1,37 @@
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
public class ConvaiSpeechBubbleController : MonoBehaviour
{
private ConvaiGroupNPCController _convaiGroupNPC;
private NPCSpeechBubble _speechBubble;
private void OnDestroy()
{
_convaiGroupNPC.ShowSpeechBubble -= ConvaiNPC_ShowSpeechBubble;
_convaiGroupNPC.HideSpeechBubble -= ConvaiNPC_HideSpeechBubble;
Destroy(_speechBubble.gameObject);
_speechBubble = null;
}
public void Initialize(NPCSpeechBubble speechBubbleDisplay, ConvaiGroupNPCController convaiGroupNPC)
{
if (_speechBubble != null) return;
_speechBubble = Instantiate(speechBubbleDisplay, transform);
_convaiGroupNPC = convaiGroupNPC;
_convaiGroupNPC.ShowSpeechBubble += ConvaiNPC_ShowSpeechBubble;
_convaiGroupNPC.HideSpeechBubble += ConvaiNPC_HideSpeechBubble;
}
private void ConvaiNPC_HideSpeechBubble()
{
_speechBubble.HideSpeechBubble();
}
private void ConvaiNPC_ShowSpeechBubble(string text)
{
if(!string.IsNullOrEmpty(text)) _speechBubble.ShowSpeechBubble(text);
}
}
}

11
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/ConvaiSpeechBubbleController.cs.meta Normal file
View File

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: e4c8a9e9735c471493ad93d4f677e7de
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

11
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/ISpeechBubbleDisplay.cs Normal file
View File

@ -0,0 +1,11 @@
namespace Convai.Scripts.Runtime.Features
{
/// <summary>
/// Interface for displaying speech bubbles.
/// </summary>
public interface ISpeechBubbleDisplay
{
void ShowSpeechBubble(string text);
void HideSpeechBubble();
}
}

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/ISpeechBubbleDisplay.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 6c1403fe2d584d4abbb9e6588b8035f7
timeCreated: 1712719388

29
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/NPCSpeechBubble.cs Normal file
View File

@ -0,0 +1,29 @@
using TMPro;
using UnityEngine;
namespace Convai.Scripts.Runtime.Features
{
public class NPCSpeechBubble : MonoBehaviour, ISpeechBubbleDisplay
{
[SerializeField] private TMP_Text speechBubbleText;
[SerializeField] private Canvas speechBubbleCanvas;
/// <summary>
/// Show the speech bubble with the given text.
/// </summary>
/// <param name="text"> The text to display in the speech bubble. </param>
public void ShowSpeechBubble(string text)
{
speechBubbleText.text = text;
speechBubbleCanvas.enabled = true;
}
/// <summary>
/// Hide the speech bubble.
/// </summary>
public void HideSpeechBubble()
{
speechBubbleCanvas.enabled = false;
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/NPCSpeechBubble.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 8a26c5b33822429bb443f94d041a3374
timeCreated: 1712719408