tomh104/Master-Arbeit-Tom-Hempel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactored UDP audio sender and receiver scripts to maintain last known peer IP during connection loss, improved logging for audio clip monitoring and peer discovery status

Browse Source
This commit is contained in:
tom.hempel
2025-10-25 13:59:17 +02:00
parent 4d77a4753a
commit 1274d6277d
4 changed files with 162 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechSender.cs
View File

@ -12,8 +12,18 @@ using System.Collections;
namespace Convai.Scripts.Runtime.Multiplayer
{
/// <summary>
/// UDP Speech Sender - Simple and reliable approach using events
/// Hooks into AudioManager events to capture when clips are about to be played
/// UDP Speech Sender - Captures and transmits NPC speech audio to remote player
///
/// FLOW (Player 1 → Player 2):
/// 1. Player 2 speaks (via ConvaiSimpleUDPAudioSender on their device)
/// 2. Player 1 receives voice input (via ConvaiSimpleUDPAudioReceiver)
/// 3. Player 1's NPC generates response speech (Convai API)
/// 4. THIS COMPONENT monitors Player 1's NPC AudioSource
/// 5. When new AudioClips appear, transmit them to Player 2
/// 6. Player 2's ConvaiUDPSpeechReceiver plays the audio
///
/// This component should be on a NetworkManager or similar persistent object.
/// It will find and monitor ConvaiNPC components on Avatar objects in the scene.
/// </summary>
public class ConvaiUDPSpeechSender : MonoBehaviour
{
@ -158,13 +168,9 @@ namespace Convai.Scripts.Runtime.Multiplayer
private void HandlePeerLost()
{
var cfg = NetworkConfig.Instance;
if (cfg != null)
{
targetIP = cfg.fallbackBroadcastIP;
_targetEndPoint = new IPEndPoint(IPAddress.Parse(targetIP), targetPort);
ConvaiLogger.Warn($"🔊 Speech sender falling back to broadcast: {targetIP}", ConvaiLogger.LogCategory.Character);
}
// Don't change targetIP - keep sending to the last known peer IP
// The peer might come back online and we'll automatically reconnect
ConvaiLogger.Warn($"🔊 Speech sender: Peer connection lost, but continuing to send to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
}
private void InitializeNetwork()
@ -190,10 +196,19 @@ namespace Convai.Scripts.Runtime.Multiplayer
if (localNPC != null)
{
sourceNPC = localNPC;
ConvaiLogger.Info($"Speech Sender: Using local NPC {sourceNPC.characterName}", ConvaiLogger.LogCategory.Character);
}
else if (useActiveNPC)
{
sourceNPC = FindEnabledConvaiNPC();
if (sourceNPC != null)
{
ConvaiLogger.Info($"Speech Sender: Found NPC {sourceNPC.characterName} on {sourceNPC.gameObject.name}", ConvaiLogger.LogCategory.Character);
}
else
{
ConvaiLogger.Warn("Speech Sender: No ConvaiNPC found in scene yet", ConvaiLogger.LogCategory.Character);
}
}
SubscribeToNPCEvents();
@ -224,6 +239,9 @@ namespace Convai.Scripts.Runtime.Multiplayer
sourceNPC.AudioManager.OnAudioTranscriptAvailable += HandleTranscriptAvailable;
ConvaiLogger.Info($"✅ UDP Speech Sender subscribed to NPC: {sourceNPC.characterName} (on {sourceNPC.gameObject.name}), AudioManager: {sourceNPC.AudioManager.name}", ConvaiLogger.LogCategory.Character);
// Also start continuous monitoring as a fallback in case events don't fire
StartCoroutine(ContinuousAudioMonitoring());
}
private void HandleCharacterTalkingChanged(bool isTalking)
@ -256,27 +274,36 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
if (sourceNPC?.AudioManager == null)
{
ConvaiLogger.Warn("MonitorAudioClips: AudioManager is null", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Error("MonitorAudioClips: AudioManager is null on sourceNPC", ConvaiLogger.LogCategory.Character);
yield break;
}
AudioSource audioSource = sourceNPC.AudioManager.GetComponent<AudioSource>();
if (audioSource == null)
{
ConvaiLogger.Warn("MonitorAudioClips: AudioSource is null", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Error($"MonitorAudioClips: No AudioSource found on AudioManager ({sourceNPC.AudioManager.name})", ConvaiLogger.LogCategory.Character);
yield break;
}
ConvaiLogger.Info($"🔊 Started monitoring audio clips on {audioSource.name}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"🔊 Started monitoring audio clips on {audioSource.name} for NPC {sourceNPC.characterName}", ConvaiLogger.LogCategory.Character);
AudioClip lastClip = null;
int checkCount = 0;
while (sourceNPC.IsCharacterTalking)
while (sourceNPC != null && sourceNPC.IsCharacterTalking)
{
checkCount++;
// Log periodically to show we're still monitoring
if (enableDebugLogging && checkCount % 10 == 0)
{
ConvaiLogger.DebugLog($"🔊 Monitoring... check #{checkCount}, current clip: {(audioSource?.clip != null ? audioSource.clip.name : "null")}, isTalking: {sourceNPC.IsCharacterTalking}", ConvaiLogger.LogCategory.Character);
}
if (audioSource?.clip != null && audioSource.clip != lastClip)
{
// New clip detected!
lastClip = audioSource.clip;
ConvaiLogger.Info($"🔊 Detected new audio clip: {lastClip.name}, length: {lastClip.length:F2}s", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"🔊 NEW CLIP DETECTED: {lastClip.name}, length: {lastClip.length:F2}s, samples: {lastClip.samples}, freq: {lastClip.frequency}", ConvaiLogger.LogCategory.Character);
// Only send if we haven't sent this clip before
if (!_sentClips.Contains(lastClip))
@ -287,23 +314,91 @@ namespace Convai.Scripts.Runtime.Multiplayer
string transcript = GetRecentTranscript();
// Send this clip
ConvaiLogger.Info($"🔊 Transmitting audio clip to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"🔊 TRANSMITTING CLIP to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
_ = TransmitAudioClip(lastClip, transcript);
}
else
{
ConvaiLogger.Info($"🔊 Clip already sent, skipping", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Warn($"🔊 Clip already sent, skipping: {lastClip.name}", ConvaiLogger.LogCategory.Character);
}
}
yield return new WaitForSeconds(0.1f); // Check every 100ms
}
ConvaiLogger.Info($"🔊 Stopped monitoring audio clips (NPC stopped talking)", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"🔊 Stopped monitoring audio clips (NPC stopped talking or was destroyed). Checks performed: {checkCount}", ConvaiLogger.LogCategory.Character);
// Clear sent clips when done
_sentClips.Clear();
}
private IEnumerator ContinuousAudioMonitoring()
{
ConvaiLogger.Info("🔊 Starting continuous audio monitoring as fallback", ConvaiLogger.LogCategory.Character);
AudioClip lastMonitoredClip = null;
while (true)
{
// Wait a bit between checks
yield return new WaitForSeconds(0.2f);
// Check if we still have a valid source NPC
if (sourceNPC == null || sourceNPC.AudioManager == null)
{
yield return new WaitForSeconds(1f); // Wait longer if no NPC
continue;
}
// Get the audio source
AudioSource audioSource = sourceNPC.AudioManager.GetComponent<AudioSource>();
if (audioSource == null)
{
yield return new WaitForSeconds(1f);
continue;
}
// Check if there's a new audio clip playing
if (audioSource.clip != null &&
audioSource.clip != lastMonitoredClip &&
audioSource.isPlaying)
{
lastMonitoredClip = audioSource.clip;
// Only send if we haven't sent this clip before
if (!_sentClips.Contains(lastMonitoredClip))
{
_sentClips.Add(lastMonitoredClip);
ConvaiLogger.Info($"🔊 [Continuous Monitor] NEW CLIP DETECTED: {lastMonitoredClip.name}, length: {lastMonitoredClip.length:F2}s", ConvaiLogger.LogCategory.Character);
// Start transmission if not already started
if (!_isSendingSpeech)
{
_isSendingSpeech = true;
OnSpeechTransmission?.Invoke(true);
}
string transcript = "";
_ = TransmitAudioClip(lastMonitoredClip, transcript);
}
}
// Clean up old clips from the sent list if NPC is not talking
if (!sourceNPC.IsCharacterTalking && _sentClips.Count > 0)
{
if (enableDebugLogging)
ConvaiLogger.DebugLog($"🔊 [Continuous Monitor] NPC stopped talking, clearing sent clips list ({_sentClips.Count} clips)", ConvaiLogger.LogCategory.Character);
_sentClips.Clear();
// Send final packet
if (_isSendingSpeech)
{
_ = SendFinalPacket();
}
}
}
}
private string GetRecentTranscript()
{
// Try to get transcript from the NPC's recent activity
@ -569,6 +664,9 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
ConvaiNPCManager.Instance.OnActiveNPCChanged -= HandleActiveNPCChanged;
}
// Stop all coroutines when cleaning up (will restart with new NPC)
StopAllCoroutines();
}
private void CleanupNetwork()