refactored UDP audio sender/receiver scripts to support port reuse for shared access, enhanced logging for recording state and audio clip monitoring

This commit is contained in:
tom.hempel
2025-10-23 03:23:33 +02:00
parent 73b921fc9b
commit fd7e08679f
6 changed files with 47 additions and 16 deletions

View File

@ -234,10 +234,13 @@ namespace Convai.Scripts.Runtime.Multiplayer
try
{
_udpListener = new UdpClient(listenPort);
// Create UDP client with port reuse to allow sharing with UDPPeerDiscovery
_udpListener = new UdpClient();
_udpListener.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_udpListener.Client.Bind(new IPEndPoint(IPAddress.Any, listenPort));
_isListening = true;
ConvaiLogger.Info($"Simple UDP Audio Receiver V2 listening on port {listenPort}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"Simple UDP Audio Receiver V2 listening on port {listenPort} (shared)", ConvaiLogger.LogCategory.Character);
// Start listening for incoming packets
_ = ListenForAudioPackets(_cancellationTokenSource.Token);

View File

@ -487,7 +487,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
_localSamples.Clear();
}
ConvaiLogger.Info("Started recording for UDP transmission (Simple)", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"🎤 Started recording for UDP transmission to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
OnRecordingStateChanged?.Invoke(true);
// Send START control and wait briefly for ACK to ensure receiver is ready

View File

@ -201,10 +201,13 @@ namespace Convai.Scripts.Runtime.Multiplayer
try
{
_udpListener = new UdpClient(listenPort);
// Create UDP client with port reuse to allow sharing with UDPPeerDiscovery
_udpListener = new UdpClient();
_udpListener.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_udpListener.Client.Bind(new IPEndPoint(IPAddress.Any, listenPort));
_isListening = true;
ConvaiLogger.Info($"UDP Speech Receiver listening on port {listenPort}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"UDP Speech Receiver listening on port {listenPort} (shared)", ConvaiLogger.LogCategory.Character);
// Start listening for incoming packets
_ = ListenForSpeechPackets(_cancellationTokenSource.Token);

View File

@ -207,18 +207,23 @@ namespace Convai.Scripts.Runtime.Multiplayer
private void SubscribeToNPCEvents()
{
if (sourceNPC?.AudioManager != null)
if (sourceNPC == null)
{
ConvaiLogger.Warn("SubscribeToNPCEvents: sourceNPC is null", ConvaiLogger.LogCategory.Character);
return;
}
if (sourceNPC.AudioManager == null)
{
ConvaiLogger.Warn($"SubscribeToNPCEvents: AudioManager is null for {sourceNPC.characterName}", ConvaiLogger.LogCategory.Character);
return;
}
// Hook into the character talking events
sourceNPC.AudioManager.OnCharacterTalkingChanged += HandleCharacterTalkingChanged;
sourceNPC.AudioManager.OnAudioTranscriptAvailable += HandleTranscriptAvailable;
ConvaiLogger.Info($"UDP Speech Sender subscribed to NPC: {sourceNPC.characterName}", ConvaiLogger.LogCategory.Character);
}
else
{
ConvaiLogger.Warn("No source NPC available for speech transmission", ConvaiLogger.LogCategory.Character);
}
ConvaiLogger.Info($"UDP Speech Sender subscribed to NPC: {sourceNPC.characterName} (on {sourceNPC.gameObject.name}), AudioManager: {sourceNPC.AudioManager.name}", ConvaiLogger.LogCategory.Character);
}
private void HandleCharacterTalkingChanged(bool isTalking)
@ -227,11 +232,13 @@ namespace Convai.Scripts.Runtime.Multiplayer
if (isTalking)
{
ConvaiLogger.Info($"🔊 NPC {sourceNPC.characterName} started talking, monitoring audio clips...", ConvaiLogger.LogCategory.Character);
// Start monitoring for audio clips
StartCoroutine(MonitorAudioClips());
}
else
{
ConvaiLogger.Info($"🔊 NPC {sourceNPC.characterName} stopped talking", ConvaiLogger.LogCategory.Character);
// End speech transmission
_ = SendFinalPacket();
}
@ -247,9 +254,20 @@ namespace Convai.Scripts.Runtime.Multiplayer
private IEnumerator MonitorAudioClips()
{
if (sourceNPC?.AudioManager == null) yield break;
if (sourceNPC?.AudioManager == null)
{
ConvaiLogger.Warn("MonitorAudioClips: AudioManager is null", ConvaiLogger.LogCategory.Character);
yield break;
}
AudioSource audioSource = sourceNPC.AudioManager.GetComponent<AudioSource>();
if (audioSource == null)
{
ConvaiLogger.Warn("MonitorAudioClips: AudioSource is null", ConvaiLogger.LogCategory.Character);
yield break;
}
ConvaiLogger.Info($"🔊 Started monitoring audio clips on {audioSource.name}", ConvaiLogger.LogCategory.Character);
AudioClip lastClip = null;
while (sourceNPC.IsCharacterTalking)
@ -258,6 +276,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
// New clip detected!
lastClip = audioSource.clip;
ConvaiLogger.Info($"🔊 Detected new audio clip: {lastClip.name}, length: {lastClip.length:F2}s", ConvaiLogger.LogCategory.Character);
// Only send if we haven't sent this clip before
if (!_sentClips.Contains(lastClip))
@ -268,13 +287,19 @@ namespace Convai.Scripts.Runtime.Multiplayer
string transcript = GetRecentTranscript();
// Send this clip
ConvaiLogger.Info($"🔊 Transmitting audio clip to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
_ = TransmitAudioClip(lastClip, transcript);
}
else
{
ConvaiLogger.Info($"🔊 Clip already sent, skipping", ConvaiLogger.LogCategory.Character);
}
}
yield return new WaitForSeconds(0.1f); // Check every 100ms
}
ConvaiLogger.Info($"🔊 Stopped monitoring audio clips (NPC stopped talking)", ConvaiLogger.LogCategory.Character);
// Clear sent clips when done
_sentClips.Clear();
}

Binary file not shown.

Binary file not shown.