oxidiert/Bachelor-Arbeit-Luca-Haas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initialer Upload neues Unity-Projekt

Browse Source
This commit is contained in:
Daniel Ocks
2025-07-03 11:02:29 +02:00
commit 27d6b94b7c
8167 changed files with 1116569 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

226
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/ConvaiGroupNPCController.cs Normal file
View File

@ -0,0 +1,226 @@
using System;
using System.Collections;
using System.Threading.Tasks;
using Convai.Scripts;
using Convai.Scripts.Utils;
using Convai.Scripts.Utils.LipSync;
using Service;
using UnityEngine;
using Logger = Convai.Scripts.Utils.Logger;
/// <summary>
/// This class is responsible for handling out all the tasks related to NPC to NPC conversation for a NPC of a group
/// </summary>
public class ConvaiGroupNPCController : MonoBehaviour
{
#region Serialized Fields
[Tooltip("The prefab for the speech bubble to display above the NPC. [Optional]")]
[SerializeField]
private NPCSpeechBubble speechBubblePrefab;
[Tooltip("Attach the Main Player Transform here so that distance check can be performed")]
[SerializeField]
private Transform playerTransform;
// The distance from the NPC to the player when the NPC will start talking
[Tooltip("The distance from the NPC to the player when the NPC will start talking. Set to 0 to disable this feature. [Optional]")]
[SerializeField] [Range(0f, 100f)]
private float conversationDistanceThreshold = 5.0f;
#endregion
#region Private Attributes
private const float PLAYER_MOVE_THRESHOLD = 0.5f;
private float CONVERSATION_DISTANCE_THRESHOLD;
private string _finalResponseText = "";
private NPC2NPCGRPCClient _npc2NPCGrpcClient;
private ConvaiLipSync _lipSync;
private Coroutine _checkPlayerVicinityCoroutine;
private FaceModel FaceModel => _lipSync == null ? FaceModel.OvrModelName : _lipSync.faceModel;
#endregion
#region Events
public event Action<string> ShowSpeechBubble;
public event Action HideSpeechBubble;
public event Action<bool, ConvaiGroupNPCController> OnPlayerVicinityChanged;
#endregion
#region Public Attributes
public bool CanRelayMessage { get; set; } = true;
public NPC2NPCConversationManager ConversationManager { get; set; }
public bool IsInConversationWithAnotherNPC { get; set; }
public string CharacterName => ConvaiNPC == null ? string.Empty : ConvaiNPC.characterName;
public string CharacterID => ConvaiNPC == null ? string.Empty : ConvaiNPC.characterID;
public ConvaiNPC ConvaiNPC { get; private set; }
#endregion
/// <summary>
/// Used to set Player GameObject Transform and lip-sync
/// </summary>
private void Awake()
{
if (playerTransform == null) playerTransform = Camera.main.transform;
ConvaiNPC = GetComponent<ConvaiNPC>();
CONVERSATION_DISTANCE_THRESHOLD = conversationDistanceThreshold == 0 ? Mathf.Infinity : conversationDistanceThreshold;
TryGetComponent(out _lipSync);
}
/// <summary>
/// Starts coroutine for player vicinity check and subscribe to necessary events
/// </summary>
private void Start()
{
_checkPlayerVicinityCoroutine = StartCoroutine(CheckPlayerVicinity());
if (TryGetComponent(out ConvaiNPCAudioManager convaiNPCAudio))
{
convaiNPCAudio.OnAudioTranscriptAvailable += HandleAudioTranscriptAvailable;
}
}
/// <summary>
/// Unsubscribes to the events and stops the coroutine
/// </summary>
private void OnDestroy()
{
if (TryGetComponent(out ConvaiNPCAudioManager convaiNPCAudio))
{
convaiNPCAudio.OnAudioTranscriptAvailable -= HandleAudioTranscriptAvailable;
}
if (_checkPlayerVicinityCoroutine != null)
{
StopCoroutine(_checkPlayerVicinityCoroutine);
}
}
/// <summary>
/// Shows speech bubble and adds the received text to final transcript
/// </summary>
/// <param name="transcript"></param>
private void HandleAudioTranscriptAvailable(string transcript)
{
if (IsInConversationWithAnotherNPC)
ShowSpeechBubble?.Invoke(transcript);
_finalResponseText += transcript;
}
/// <summary>
/// Attaches the speech bubble to the NPC game-object
/// </summary>
public void AttachSpeechBubble()
{
if (TryGetComponent(out ConvaiSpeechBubbleController _)) return;
gameObject.AddComponent<ConvaiSpeechBubbleController>().Initialize(speechBubblePrefab, this);
}
/// <summary>
/// Destroys the speech bubble game-object
/// </summary>
public void DetachSpeechBubble()
{
if (TryGetComponent(out ConvaiSpeechBubbleController convaiSpeechBubble)) Destroy(convaiSpeechBubble);
}
/// <summary>
/// Store the references of the client and subscribe to the necessary events
/// </summary>
/// <param name="client"></param>
public void InitializeNpc2NpcGrpcClient(NPC2NPCGRPCClient client)
{
_npc2NPCGrpcClient = client;
}
/// <summary>
/// Every 0.5 seconds updates if player is near or not and fire events according to the state
/// </summary>
/// <returns></returns>
private IEnumerator CheckPlayerVicinity()
{
bool previousState = false;
Vector3 previousPlayerPosition = Vector3.zero;
yield return new WaitForSeconds(0.1f);
while (true)
{
Vector3 currentPlayerPosition = playerTransform.transform.position;
// Check if the player has moved more than a certain threshold distance
if (Vector3.Distance(previousPlayerPosition, currentPlayerPosition) > PLAYER_MOVE_THRESHOLD)
{
// Calculate the distance between the NPC and the player
float distanceToPlayer = Vector3.Distance(transform.position, currentPlayerPosition);
// Check if the player is within the threshold distance
bool isPlayerCurrentlyNear = distanceToPlayer <= CONVERSATION_DISTANCE_THRESHOLD;
// If the player's current vicinity state is different from the previous state, raise the event
if (isPlayerCurrentlyNear != previousState && !ConvaiNPC.isCharacterActive)
{
OnPlayerVicinityChanged?.Invoke(isPlayerCurrentlyNear, this);
previousState = isPlayerCurrentlyNear; // Update the previous state
Debug.Log($"Player is currently near {ConvaiNPC.characterName}: {isPlayerCurrentlyNear}");
}
previousPlayerPosition = currentPlayerPosition; // Update the player's previous position
// Check every half second
}
yield return new WaitForSeconds(0.5f);
}
}
/// <summary>
/// Sends the text to the other NPC in the group
/// </summary>
/// <param name="message"></param>
public async void SendTextDataNPC2NPC(string message)
{
if (_npc2NPCGrpcClient == null)
{
Logger.Warn("No GRPC client initialized for this NPC.", Logger.LogCategory.Character);
return;
}
try
{
CanRelayMessage = false;
await Task.Delay(500);
await _npc2NPCGrpcClient.SendTextData(
userText: message,
characterID: ConvaiNPC.characterID,
sessionID: ConvaiNPC.sessionID,
isLipSyncActive: _lipSync != null,
faceModel: FaceModel);
}
catch (Exception ex)
{
Logger.Warn($"Error sending message data for NPC2NPC: {ex.Message}", Logger.LogCategory.Character);
}
}
public void EndOfResponseReceived()
{
if (TryGetComponent(out ConvaiNPCAudioManager convaiNPCAudio))
{
convaiNPCAudio.OnCharacterTalkingChanged += SendFinalTranscriptToOtherNPC;
}
}
private void SendFinalTranscriptToOtherNPC(bool isTalking)
{
if (IsInConversationWithAnotherNPC)
{
if (!isTalking)
{
if (TryGetComponent(out ConvaiNPCAudioManager convaiNPCAudio))
{
convaiNPCAudio.OnCharacterTalkingChanged -= SendFinalTranscriptToOtherNPC;
}
ConversationManager.RelayMessage(_finalResponseText, this);
_finalResponseText = "";
HideSpeechBubble?.Invoke();
}
else
{
Logger.DebugLog($"{ConvaiNPC.characterName} is currently still talking. ", Logger.LogCategory.Character);
}
}
}
public bool IsPlayerNearMe()
{
bool result = Vector3.Distance(transform.position, playerTransform.position) < CONVERSATION_DISTANCE_THRESHOLD;
Logger.Info($"Player is near {CharacterName}: {result}", Logger.LogCategory.Character);
return result;
}
}

11
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/ConvaiGroupNPCController.cs.meta Normal file
View File

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 74d3f2dd15d074a429eeea8c31f80b13
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

297
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPC2NPCConversationManager.cs Normal file
View File

@ -0,0 +1,297 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using Convai.Scripts.Runtime.Utils;
using Convai.Scripts.Utils;
using Grpc.Core;
using Service;
using UnityEngine;
using Logger = Convai.Scripts.Utils.Logger;
using Random = UnityEngine.Random;
/// <summary>
/// Manages the conversation between two Convai powered NPC groups
/// </summary>
public class NPC2NPCConversationManager : MonoBehaviour
{
private const string GRPC_API_ENDPOINT = "stream.convai.com";
public static NPC2NPCConversationManager Instance;
public List<NPCGroup> npcGroups;
private readonly List<NPCGroup> _groupsWhereConversationEnded = new();
private string _apiKey = string.Empty;
private void Awake()
{
if (Instance == null)
Instance = this;
else
Destroy(gameObject);
LoadApiKey();
}
private void Start()
{
foreach (NPCGroup group in npcGroups)
group.Initialize(HandlePlayerVicinityChanged);
StartConversationWithAllNPCs();
}
/// <summary>
/// Handles the event of player vicinity change.
/// </summary>
/// <param name="isPlayerNear">Indicates if the player is near.</param>
/// <param name="npc">The NPC for which the vicinity changed.</param>
private void HandlePlayerVicinityChanged(bool isPlayerNear, ConvaiGroupNPCController npc)
{
if (isPlayerNear)
ResumeConversation(npc);
}
/// <summary>
/// Processes the message from the sender NPC.
/// </summary>
/// <param name="sender">The NPC sending the message.</param>
/// <param name="topic">The topic of the conversation.</param>
/// <param name="message">The message to be processed.</param>
/// <returns>The processed message.</returns>
private string ProcessMessage(ConvaiGroupNPCController sender, string topic, string message)
{
string processedMessage = $"{sender.CharacterName} said \"{message}\" to you. Reply to it. ";
processedMessage += Random.Range(0, 2) == 0
? $"Talk about something related to {message}. "
: $"Talk about something other than \"{message}\" but related to {topic}. Gently change the conversation topic. ";
return processedMessage + "Definitely, reply to the message. Dont address speaker. Keep the reply short. Do not repeat the same message, or keep asking same question.";
}
/// <summary>
/// Relays the message from the sender NPC.
/// </summary>
/// <param name="message">The message to be relayed.</param>
/// <param name="sender">The NPC sending the message.</param>
/// <param name="performSwitch"></param>
public void RelayMessage(string message, ConvaiGroupNPCController sender, bool performSwitch = true)
{
NPCGroup npcGroup = npcGroups.Find(c => c.BelongToGroup(sender));
if (npcGroup == null)
{
Debug.LogError("Conversation not found for the sender.");
return;
}
npcGroup.messageToRelay = message;
if (performSwitch) SwitchSpeaker(npcGroup.CurrentSpeaker);
if (!npcGroup.CurrentSpeaker.IsPlayerNearMe()) return;
StartCoroutine(RelayMessageCoroutine(message, npcGroup));
}
/// <summary>
/// Coroutine to relay the message from the sender NPC.
/// </summary>
/// <param name="message">The message to be relayed.</param>
/// <param name="npcGroup"> The NPC group to relay the message to. </param>
/// <returns>An IEnumerator to be used in a coroutine.</returns>
private IEnumerator RelayMessageCoroutine(string message, NPCGroup npcGroup)
{
yield return new WaitForSeconds(0.5f);
try
{
ConvaiGroupNPCController receiver = npcGroup.CurrentSpeaker;
if (!receiver.CanRelayMessage) yield break;
Logger.DebugLog($"Relaying message from {npcGroup.CurrentListener.CharacterName} to {receiver.CharacterName}: {message}", Logger.LogCategory.Character);
string processedMessage = ProcessMessage(npcGroup.CurrentListener, npcGroup.topic, message);
receiver.SendTextDataNPC2NPC(processedMessage);
}
catch (Exception e)
{
Logger.Warn($"Failed to relay message: {e.Message}", Logger.LogCategory.Character);
}
}
/// <summary>
/// Switches the speaker in the conversation.
/// </summary>
/// <param name="currentSpeaker">The current speaker NPC.</param>
/// <returns>The new speaker NPC.</returns>
private void SwitchSpeaker(ConvaiGroupNPCController currentSpeaker)
{
NPCGroup group = npcGroups.Find(g => g.CurrentSpeaker == currentSpeaker);
if (group != null)
{
group.CurrentSpeaker = currentSpeaker == group.GroupNPC1 ? group.GroupNPC2 : group.GroupNPC1;
Logger.DebugLog($"Switching NPC2NPC Speaker to {group.CurrentSpeaker}", Logger.LogCategory.Character);
return;
}
Logger.Warn("Failed to switch speaker. Current speaker not found in any group.", Logger.LogCategory.Character);
}
private void LoadApiKey()
{
ConvaiAPIKeySetup.GetAPIKey(out _apiKey);
}
/// <summary>
/// Initializes a single NPC for conversation.
/// </summary>
/// <param name="npc">The NPC to initialize.</param>
/// <param name="grpcClient">The GRPC client to use for the NPC.</param>
private void InitializeNPC(ConvaiGroupNPCController npc, NPC2NPCGRPCClient grpcClient)
{
if (npc == null)
{
Debug.LogWarning("The given NPC is null.");
return;
}
npc.ConversationManager = this;
npc.InitializeNpc2NpcGrpcClient(grpcClient);
npc.AttachSpeechBubble();
npc.IsInConversationWithAnotherNPC = true;
npc.ConvaiNPC.isCharacterActive = false;
}
/// <summary>
/// Starts the conversation for the given NPC.
/// </summary>
/// <param name="npcGroup">The NPC to start the conversation for.</param>
private void InitializeNPCGroup(NPCGroup npcGroup)
{
if (npcGroup == null)
{
Debug.LogWarning("The given NPC is not part of any group.");
return;
}
ConvaiGroupNPCController npc1 = npcGroup.GroupNPC1;
ConvaiGroupNPCController npc2 = npcGroup.GroupNPC2;
if (npc1.IsInConversationWithAnotherNPC || npc2.IsInConversationWithAnotherNPC)
{
Debug.LogWarning($"{npc1.CharacterName} or {npc2.CharacterName} is already in a conversation.");
return;
}
NPC2NPCGRPCClient grpcClient = CreateAndInitializeGRPCClient(npcGroup);
// Initialize both NPCs
InitializeNPC(npc1, grpcClient);
InitializeNPC(npc2, grpcClient);
npcGroup.CurrentSpeaker = Random.Range(0, 10) % 2 == 0 ? npc1 : npc2;
}
/// <summary>
/// Creates and initializes a new GRPC client for the given NPC group.
/// </summary>
/// <param name="group">The NPC group to create the GRPC client for.</param>
/// <returns>The initialized GRPC client.</returns>
private NPC2NPCGRPCClient CreateAndInitializeGRPCClient(NPCGroup group)
{
GameObject grpcClientGameObject = new($"GRPCClient_{group.GroupNPC1.CharacterID}_{group.GroupNPC2.CharacterID}")
{
transform = { parent = transform }
};
NPC2NPCGRPCClient grpcClient = grpcClientGameObject.AddComponent<NPC2NPCGRPCClient>();
ConvaiService.ConvaiServiceClient serviceClient = CreateNewConvaiServiceClient();
grpcClient.Initialize(_apiKey, serviceClient, group);
return grpcClient;
}
/// <summary>
/// Creates a new ConvaiServiceClient.
/// </summary>
/// <returns> The new ConvaiServiceClient. </returns>
private ConvaiService.ConvaiServiceClient CreateNewConvaiServiceClient()
{
try
{
SslCredentials credentials = new();
Channel channel = new(GRPC_API_ENDPOINT, credentials);
return new ConvaiService.ConvaiServiceClient(channel);
}
catch (Exception ex)
{
Logger.Error($"Failed to create ConvaiServiceClient: {ex.Message}", Logger.LogCategory.Character);
throw;
}
}
/// <summary>
/// Resumes the conversation for the given NPC.
/// </summary>
/// <param name="sender"> The NPC to resume the conversation for. </param>
private void ResumeConversation(ConvaiGroupNPCController sender)
{
NPCGroup npcGroup = npcGroups.Find(g => g.BelongToGroup(sender));
if (npcGroup.IsAnyoneTalking()) return;
if (_groupsWhereConversationEnded.Contains(npcGroup))
{
InitializeNPCGroup(npcGroup);
_groupsWhereConversationEnded.Remove(npcGroup);
}
if (string.IsNullOrEmpty(npcGroup.messageToRelay))
{
string message = $"Talk about {npcGroup.topic}.";
npcGroup.CurrentSpeaker.SendTextDataNPC2NPC(message);
npcGroup.messageToRelay = message;
Debug.Log($"Starting conversation for the first time between {npcGroup.GroupNPC1.CharacterName} and {npcGroup.GroupNPC2.CharacterName}");
}
else
{
RelayMessage(npcGroup.messageToRelay, npcGroup.CurrentSpeaker, false);
Debug.Log($"Resuming conversation between {npcGroup.GroupNPC1.CharacterName} and {npcGroup.GroupNPC2.CharacterName}");
}
}
/// <summary>
/// Ends the conversation for the given NPC.
/// </summary>
/// <param name="npc"> The NPC to end the conversation for. </param>
public void EndConversation(ConvaiGroupNPCController npc)
{
NPCGroup group = npcGroups.Find(g => g.BelongToGroup(npc));
Debug.Log($"Ending conversation between {group.GroupNPC1.CharacterName} and {group.GroupNPC2.CharacterName}");
void EndConversationForNPC(ConvaiGroupNPCController groupNPC)
{
groupNPC.IsInConversationWithAnotherNPC = false;
groupNPC.ConvaiNPC.InterruptCharacterSpeech();
groupNPC.GetComponent<ConvaiGroupNPCController>().DetachSpeechBubble();
}
EndConversationForNPC(group.GroupNPC1);
EndConversationForNPC(group.GroupNPC2);
_groupsWhereConversationEnded.Add(group);
ConvaiNPCManager.Instance.SetActiveConvaiNPC(npc.ConvaiNPC);
Destroy(transform.Find($"GRPCClient_{group.GroupNPC1.CharacterID}_{group.GroupNPC2.CharacterID}").gameObject);
}
/// <summary>
/// Starts the conversation with all NPCs.
/// </summary>
private void StartConversationWithAllNPCs()
{
IEnumerable filteredList = npcGroups
.Where(npcGroup => npcGroup.BothNPCAreNotNull())
.Where(npcGroup => npcGroup.BothNPCAreNotActiveNPC());
foreach (NPCGroup npcGroup in filteredList)
InitializeNPCGroup(npcGroup);
}
}

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPC2NPCConversationManager.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: e5c3956535ee405983974a5434b6f4a4
timeCreated: 1705687346

240
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPC2NPCGRPCClient.cs Normal file
View File

@ -0,0 +1,240 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Convai.Scripts.Utils.LipSync;
using Grpc.Core;
using Service;
using UnityEngine;
namespace Convai.Scripts.Utils
{
/// <summary>
/// Represents an NPC2NPCGRPCClient that can be used to communicate with the ConvaiService using gRPC.
/// </summary>
public class NPC2NPCGRPCClient : MonoBehaviour
{
private readonly CancellationTokenSource _cancellationTokenSource = new();
private string _apiKey;
private ConvaiService.ConvaiServiceClient _client;
private NPCGroup _npcGroup;
private void OnDestroy()
{
_cancellationTokenSource?.Cancel();
_cancellationTokenSource?.Dispose();
}
/// <summary>
/// Initializes the NPC2NPCGRPCClient with the given API key and ConvaiService client.
/// </summary>
/// <param name="apiKey">The API key to use for authentication.</param>
/// <param name="client">The ConvaiService client to use for communication.</param>
public void Initialize(string apiKey, ConvaiService.ConvaiServiceClient client, NPCGroup group)
{
_apiKey = apiKey;
_client = client;
_npcGroup = group;
}
/// <summary>
/// Creates an AsyncDuplexStreamingCall with the specified headers.
/// </summary>
/// <returns>An AsyncDuplexStreamingCall with the specified headers.</returns>
private AsyncDuplexStreamingCall<GetResponseRequest, GetResponseResponse> GetAsyncDuplexStreamingCallOptions()
{
Metadata headers = new()
{
{ "source", "Unity" },
{ "version", "3.0.0" }
};
CallOptions options = new(headers);
return _client.GetResponse(options);
}
/// <summary>
/// Sends the specified user text to the server and receives a response.
/// </summary>
/// <param name="userText">The user text to send to the server.</param>
/// <param name="characterID">The ID of the character to use for the request.</param>
/// <param name="sessionID">The ID of the session to use for the request.</param>
/// <param name="isLipSyncActive">Whether lip sync is active for the request.</param>
/// <param name="faceModel">The face model to use for the request.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public async Task SendTextData(string userText, string characterID, string sessionID, bool isLipSyncActive, FaceModel faceModel)
{
AsyncDuplexStreamingCall<GetResponseRequest, GetResponseResponse> call = GetAsyncDuplexStreamingCallOptions();
GetResponseRequest getResponseConfigRequest = CreateGetResponseRequest(characterID, sessionID, isLipSyncActive, faceModel, false, null);
try
{
await call.RequestStream.WriteAsync(getResponseConfigRequest);
await call.RequestStream.WriteAsync(new GetResponseRequest
{
GetResponseData = new GetResponseRequest.Types.GetResponseData
{
TextData = userText
}
});
await call.RequestStream.CompleteAsync();
Task receiveResultsTask = Task.Run(
async () => { await ReceiveResultFromServer(call, _cancellationTokenSource.Token); },
_cancellationTokenSource.Token);
await receiveResultsTask.ConfigureAwait(false);
}
catch (Exception)
{
// ignored
}
}
/// <summary>
/// Creates a GetResponseRequest with the specified parameters.
/// </summary>
/// <param name="characterID">The ID of the character to use for the request.</param>
/// <param name="sessionID">The ID of the session to use for the request.</param>
/// <param name="isLipSyncActive">Whether lip sync is active for the request.</param>
/// <param name="faceModel">The face model to use for the request.</param>
/// <param name="isActionActive">Whether action is active for the request.</param>
/// <param name="actionConfig">The action configuration to use for the request.</param>
/// <returns>A GetResponseRequest with the specified parameters.</returns>
private GetResponseRequest CreateGetResponseRequest(string characterID, string sessionID, bool isLipSyncActive, FaceModel faceModel, bool isActionActive,
ActionConfig actionConfig)
{
GetResponseRequest getResponseConfigRequest = new()
{
GetResponseConfig = new GetResponseRequest.Types.GetResponseConfig
{
CharacterId = characterID,
ApiKey = _apiKey,
SessionId = sessionID,
AudioConfig = new AudioConfig
{
EnableFacialData = isLipSyncActive,
FaceModel = faceModel
}
}
};
if (isActionActive)
getResponseConfigRequest.GetResponseConfig.ActionConfig = actionConfig;
return getResponseConfigRequest;
}
/// <summary>
/// Receives a response from the server asynchronously.
/// </summary>
/// <param name="call">The AsyncDuplexStreamingCall to use for receiving the response.</param>
/// <param name="cancellationToken">The cancellation token to use for cancelling the operation.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
private async Task ReceiveResultFromServer(AsyncDuplexStreamingCall<GetResponseRequest, GetResponseResponse> call, CancellationToken cancellationToken)
{
Logger.Info("Receiving response from server", Logger.LogCategory.Character);
ConvaiNPC convaiNPC = _npcGroup.CurrentSpeaker.ConvaiNPC;
_npcGroup.CurrentSpeaker.CanRelayMessage = true;
Queue<LipSyncBlendFrameData> lipSyncBlendFrameQueue = new Queue<LipSyncBlendFrameData>();
bool firstSilFound = false;
while (!cancellationToken.IsCancellationRequested && await call.ResponseStream.MoveNext(cancellationToken).ConfigureAwait(false))
try
{
GetResponseResponse result = call.ResponseStream.Current;
// Process the received response here
if (result.AudioResponse != null)
{
if (result.AudioResponse.AudioData != null)
// Add response to the list in the active NPC
{
if (result.AudioResponse.AudioData.ToByteArray().Length > 46)
{
byte[] wavBytes = result.AudioResponse.AudioData.ToByteArray();
// will only work for wav files
WavHeaderParser parser = new(wavBytes);
if (convaiNPC.convaiLipSync == null)
{
Logger.DebugLog($"Enqueuing responses: {result.AudioResponse.TextData}", Logger.LogCategory.LipSync);
convaiNPC.EnqueueResponse(result);
}
else
{
LipSyncBlendFrameData.FrameType frameType =
convaiNPC.convaiLipSync.faceModel == FaceModel.OvrModelName
? LipSyncBlendFrameData.FrameType.Visemes
: LipSyncBlendFrameData.FrameType.Blendshape;
lipSyncBlendFrameQueue.Enqueue(
new LipSyncBlendFrameData(
(int)(parser.CalculateDurationSeconds() * 30),
result,
frameType
)
);
}
}
if (result.AudioResponse.VisemesData != null)
{
if (convaiNPC.convaiLipSync != null)
{
//Logger.Info(result.AudioResponse.VisemesData, Logger.LogCategory.LipSync);
if (result.AudioResponse.VisemesData.Visemes.Sil == -2 || result.AudioResponse.EndOfResponse)
{
if (firstSilFound)
{
lipSyncBlendFrameQueue.Dequeue().Process(convaiNPC);
}
firstSilFound = true;
}
else
lipSyncBlendFrameQueue.Peek().Enqueue(result.AudioResponse.VisemesData);
}
}
if (result.AudioResponse.BlendshapesFrame != null)
{
if (convaiNPC.convaiLipSync != null)
{
if (lipSyncBlendFrameQueue.Peek().CanProcess() || result.AudioResponse.EndOfResponse)
{
lipSyncBlendFrameQueue.Dequeue().Process(convaiNPC);
}
else
{
lipSyncBlendFrameQueue.Peek().Enqueue(result.AudioResponse.BlendshapesFrame);
if (lipSyncBlendFrameQueue.Peek().CanPartiallyProcess())
{
lipSyncBlendFrameQueue.Peek().ProcessPartially(convaiNPC);
}
}
}
}
if (result.AudioResponse.EndOfResponse)
{
MainThreadDispatcher.Instance.RunOnMainThread(() =>
{
_npcGroup.CurrentSpeaker.EndOfResponseReceived();
});
}
}
}
}
catch (RpcException rpcException)
{
if (rpcException.StatusCode == StatusCode.Cancelled)
Logger.Error(rpcException, Logger.LogCategory.Character);
else
throw;
}
catch (Exception ex)
{
Logger.DebugLog(ex, Logger.LogCategory.Character);
}
}
}
}

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPC2NPCGRPCClient.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: dc8dba94e7504aab878f30a19a55fa5a
timeCreated: 1708531595

71
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPCGroup.cs Normal file
View File

@ -0,0 +1,71 @@
using System;
using Convai.Scripts;
using Convai.Scripts.Utils;
using UnityEngine;
/// <summary>
/// A group of NPCs that are currently conversing with each other.
/// </summary>
[Serializable]
public class NPCGroup
{
[field: SerializeField] public ConvaiGroupNPCController GroupNPC1 { get; private set; }
[field: SerializeField] public ConvaiGroupNPCController GroupNPC2 { get; private set; }
public string topic;
[ReadOnly] public string messageToRelay;
public ConvaiGroupNPCController CurrentSpeaker { get; set; }
public ConvaiGroupNPCController CurrentListener => CurrentSpeaker == GroupNPC1 ? GroupNPC2 : GroupNPC1;
private bool _isPlayerNearGroup;
private Action<bool, ConvaiGroupNPCController> _vicinityChangedCallback;
public void Initialize(Action<bool, ConvaiGroupNPCController> vicinityChangedCallback)
{
_vicinityChangedCallback = vicinityChangedCallback;
GroupNPC1.OnPlayerVicinityChanged += HandleVicinity;
GroupNPC1.OnPlayerVicinityChanged += HandleVicinity;
}
~NPCGroup()
{
GroupNPC1.OnPlayerVicinityChanged -= HandleVicinity;
GroupNPC1.OnPlayerVicinityChanged -= HandleVicinity;
}
private void HandleVicinity(bool isPlayerNear, ConvaiGroupNPCController npc)
{
if (isPlayerNear && !_isPlayerNearGroup)
{
_isPlayerNearGroup = true;
_vicinityChangedCallback?.Invoke(true, npc);
}
if (!isPlayerNear && _isPlayerNearGroup)
{
_isPlayerNearGroup = false;
_vicinityChangedCallback?.Invoke(false, npc);
}
}
public bool IsAnyoneTalking()
{
return GroupNPC1.ConvaiNPC.IsCharacterTalking || GroupNPC2.ConvaiNPC.IsCharacterTalking;
}
public bool BelongToGroup(ConvaiGroupNPCController controller)
{
return controller.CharacterID == GroupNPC1.CharacterID || controller.CharacterID == GroupNPC2.CharacterID;
}
public bool BothNPCAreNotNull()
{
return GroupNPC1 != null && GroupNPC2 != null;
}
public bool BothNPCAreNotActiveNPC()
{
ConvaiNPC activeNPC = ConvaiNPCManager.Instance.activeConvaiNPC;
string activeNPCId = activeNPC != null ? activeNPC.characterID : string.Empty;
return !GroupNPC1.CharacterID.Equals(activeNPCId) && !GroupNPC2.CharacterID.Equals(activeNPCId);
}
}

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/NPCGroup.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 6c1f1924c9c44d79a1c9abfaeb720f83
timeCreated: 1713324738

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 0c6246458db44b939c322f228563602e
timeCreated: 1712719472

33
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/ConvaiSpeechBubbleController.cs Normal file
View File

@ -0,0 +1,33 @@
using UnityEngine;
public class ConvaiSpeechBubbleController : MonoBehaviour
{
ConvaiGroupNPCController _convaiGroupNPC;
NPCSpeechBubble _speechBubble;
public void Initialize(NPCSpeechBubble speechBubbleDisplay, ConvaiGroupNPCController convaiGroupNPC)
{
if (_speechBubble != null) return;
_speechBubble = Instantiate(speechBubbleDisplay, transform);
_convaiGroupNPC = convaiGroupNPC;
_convaiGroupNPC.ShowSpeechBubble += ConvaiNPC_ShowSpeechBubble;
_convaiGroupNPC.HideSpeechBubble += ConvaiNPC_HideSpeechBubble;
}
private void ConvaiNPC_HideSpeechBubble()
{
_speechBubble.HideSpeechBubble();
}
private void ConvaiNPC_ShowSpeechBubble(string text)
{
_speechBubble.ShowSpeechBubble(text);
}
void OnDestroy()
{
_convaiGroupNPC.ShowSpeechBubble -= ConvaiNPC_ShowSpeechBubble;
_convaiGroupNPC.HideSpeechBubble -= ConvaiNPC_HideSpeechBubble;
Destroy(_speechBubble.gameObject);
_speechBubble = null;
}
}

11
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/ConvaiSpeechBubbleController.cs.meta Normal file
View File

@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: e4c8a9e9735c471493ad93d4f677e7de
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

8
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/ISpeechBubbleDisplay.cs Normal file
View File

@ -0,0 +1,8 @@
/// <summary>
/// Interface for displaying speech bubbles.
/// </summary>
public interface ISpeechBubbleDisplay
{
void ShowSpeechBubble(string text);
void HideSpeechBubble();
}

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/ISpeechBubbleDisplay.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 6c1403fe2d584d4abbb9e6588b8035f7
timeCreated: 1712719388

26
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/NPCSpeechBubble.cs Normal file
View File

@ -0,0 +1,26 @@
using TMPro;
using UnityEngine;
public class NPCSpeechBubble : MonoBehaviour, ISpeechBubbleDisplay
{
[SerializeField] private TMP_Text speechBubbleText;
[SerializeField] private Canvas speechBubbleCanvas;
/// <summary>
/// Show the speech bubble with the given text.
/// </summary>
/// <param name="text"> The text to display in the speech bubble. </param>
public void ShowSpeechBubble(string text)
{
speechBubbleText.text = text;
speechBubbleCanvas.enabled = true;
}
/// <summary>
/// Hide the speech bubble.
/// </summary>
public void HideSpeechBubble()
{
speechBubbleCanvas.enabled = false;
}
}

3
Assets/Convai/Scripts/Runtime/Features/NPC2NPC/Speech Bubble System/NPCSpeechBubble.cs.meta Normal file
View File

@ -0,0 +1,3 @@
fileFormatVersion: 2
guid: 8a26c5b33822429bb443f94d041a3374
timeCreated: 1712719408