tomh104/Master-Arbeit-Tom-Hempel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactored UDP audio sender and receiver scripts to maintain last known peer IP during connection loss, improved logging for audio clip monitoring and peer discovery status

Browse Source
This commit is contained in:
tom.hempel
2025-10-25 13:59:17 +02:00
parent 4d77a4753a
commit 1274d6277d
4 changed files with 162 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs
View File

@ -177,13 +177,9 @@ namespace Convai.Scripts.Runtime.Multiplayer
private void HandlePeerLost()
{
var cfg = NetworkConfig.Instance;
if (cfg != null)
{
targetIP = cfg.fallbackBroadcastIP;
_targetEndPoint = new IPEndPoint(IPAddress.Parse(targetIP), targetPort);
ConvaiLogger.Warn($"🎤 Audio sender falling back to broadcast: {targetIP}", ConvaiLogger.LogCategory.Character);
}
// Don't change targetIP - keep sending to the last known peer IP
// The peer might come back online and we'll automatically reconnect
ConvaiLogger.Warn($"🎤 Audio sender: Peer connection lost, but continuing to send to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
}
private void InitializeNetwork()

14
Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechReceiver.cs
View File

@ -11,8 +11,18 @@ using UnityEngine;
namespace Convai.Scripts.Runtime.Multiplayer
{
/// <summary>
/// UDP Speech Receiver - Receives high-quality Convai speech with proper buffering
/// This version reconstructs the original AudioClip objects for seamless playback
/// UDP Speech Receiver - Receives and plays NPC speech audio from remote player
///
/// FLOW (Player 1 → Player 2):
/// 1. Player 2 speaks to Player 1's NPC
/// 2. Player 1's NPC responds with speech
/// 3. Player 1's ConvaiUDPSpeechSender transmits the audio
/// 4. THIS COMPONENT receives the audio packets
/// 5. Reconstructs AudioClips from the packets
/// 6. Plays them back on local AudioSource
///
/// This component should be on a NetworkManager or similar persistent object.
/// It receives speech from the remote player's NPC.
/// </summary>
public class ConvaiUDPSpeechReceiver : MonoBehaviour
{

132
Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechSender.cs
View File

@ -12,8 +12,18 @@ using System.Collections;
namespace Convai.Scripts.Runtime.Multiplayer
{
/// <summary>
/// UDP Speech Sender - Simple and reliable approach using events
/// Hooks into AudioManager events to capture when clips are about to be played
/// UDP Speech Sender - Captures and transmits NPC speech audio to remote player
///
/// FLOW (Player 1 → Player 2):
/// 1. Player 2 speaks (via ConvaiSimpleUDPAudioSender on their device)
/// 2. Player 1 receives voice input (via ConvaiSimpleUDPAudioReceiver)
/// 3. Player 1's NPC generates response speech (Convai API)
/// 4. THIS COMPONENT monitors Player 1's NPC AudioSource
/// 5. When new AudioClips appear, transmit them to Player 2
/// 6. Player 2's ConvaiUDPSpeechReceiver plays the audio
///
/// This component should be on a NetworkManager or similar persistent object.
/// It will find and monitor ConvaiNPC components on Avatar objects in the scene.
/// </summary>
public class ConvaiUDPSpeechSender : MonoBehaviour
{
@ -158,13 +168,9 @@ namespace Convai.Scripts.Runtime.Multiplayer
private void HandlePeerLost()
{
var cfg = NetworkConfig.Instance;
if (cfg != null)
{
targetIP = cfg.fallbackBroadcastIP;
_targetEndPoint = new IPEndPoint(IPAddress.Parse(targetIP), targetPort);
ConvaiLogger.Warn($"🔊 Speech sender falling back to broadcast: {targetIP}", ConvaiLogger.LogCategory.Character);
}
// Don't change targetIP - keep sending to the last known peer IP
// The peer might come back online and we'll automatically reconnect
ConvaiLogger.Warn($"🔊 Speech sender: Peer connection lost, but continuing to send to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
}
private void InitializeNetwork()
@ -190,10 +196,19 @@ namespace Convai.Scripts.Runtime.Multiplayer
if (localNPC != null)
{
sourceNPC = localNPC;
ConvaiLogger.Info($"Speech Sender: Using local NPC {sourceNPC.characterName}", ConvaiLogger.LogCategory.Character);
}
else if (useActiveNPC)
{
sourceNPC = FindEnabledConvaiNPC();
if (sourceNPC != null)
{
ConvaiLogger.Info($"Speech Sender: Found NPC {sourceNPC.characterName} on {sourceNPC.gameObject.name}", ConvaiLogger.LogCategory.Character);
}
else
{
ConvaiLogger.Warn("Speech Sender: No ConvaiNPC found in scene yet", ConvaiLogger.LogCategory.Character);
}
}
SubscribeToNPCEvents();
@ -224,6 +239,9 @@ namespace Convai.Scripts.Runtime.Multiplayer
sourceNPC.AudioManager.OnAudioTranscriptAvailable += HandleTranscriptAvailable;
ConvaiLogger.Info($"✅ UDP Speech Sender subscribed to NPC: {sourceNPC.characterName} (on {sourceNPC.gameObject.name}), AudioManager: {sourceNPC.AudioManager.name}", ConvaiLogger.LogCategory.Character);
// Also start continuous monitoring as a fallback in case events don't fire
StartCoroutine(ContinuousAudioMonitoring());
}
private void HandleCharacterTalkingChanged(bool isTalking)
@ -256,27 +274,36 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
if (sourceNPC?.AudioManager == null)
{
ConvaiLogger.Warn("MonitorAudioClips: AudioManager is null", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Error("MonitorAudioClips: AudioManager is null on sourceNPC", ConvaiLogger.LogCategory.Character);
yield break;
}
AudioSource audioSource = sourceNPC.AudioManager.GetComponent<AudioSource>();
if (audioSource == null)
{
ConvaiLogger.Warn("MonitorAudioClips: AudioSource is null", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Error($"MonitorAudioClips: No AudioSource found on AudioManager ({sourceNPC.AudioManager.name})", ConvaiLogger.LogCategory.Character);
yield break;
}
ConvaiLogger.Info($"🔊 Started monitoring audio clips on {audioSource.name}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"🔊 Started monitoring audio clips on {audioSource.name} for NPC {sourceNPC.characterName}", ConvaiLogger.LogCategory.Character);
AudioClip lastClip = null;
int checkCount = 0;
while (sourceNPC.IsCharacterTalking)
while (sourceNPC != null && sourceNPC.IsCharacterTalking)
{
checkCount++;
// Log periodically to show we're still monitoring
if (enableDebugLogging && checkCount % 10 == 0)
{
ConvaiLogger.DebugLog($"🔊 Monitoring... check #{checkCount}, current clip: {(audioSource?.clip != null ? audioSource.clip.name : "null")}, isTalking: {sourceNPC.IsCharacterTalking}", ConvaiLogger.LogCategory.Character);
}
if (audioSource?.clip != null && audioSource.clip != lastClip)
{
// New clip detected!
lastClip = audioSource.clip;
ConvaiLogger.Info($"🔊 Detected new audio clip: {lastClip.name}, length: {lastClip.length:F2}s", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"🔊 NEW CLIP DETECTED: {lastClip.name}, length: {lastClip.length:F2}s, samples: {lastClip.samples}, freq: {lastClip.frequency}", ConvaiLogger.LogCategory.Character);
// Only send if we haven't sent this clip before
if (!_sentClips.Contains(lastClip))
@ -287,23 +314,91 @@ namespace Convai.Scripts.Runtime.Multiplayer
string transcript = GetRecentTranscript();
// Send this clip
ConvaiLogger.Info($"🔊 Transmitting audio clip to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"🔊 TRANSMITTING CLIP to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
_ = TransmitAudioClip(lastClip, transcript);
}
else
{
ConvaiLogger.Info($"🔊 Clip already sent, skipping", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Warn($"🔊 Clip already sent, skipping: {lastClip.name}", ConvaiLogger.LogCategory.Character);
}
}
yield return new WaitForSeconds(0.1f); // Check every 100ms
}
ConvaiLogger.Info($"🔊 Stopped monitoring audio clips (NPC stopped talking)", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"🔊 Stopped monitoring audio clips (NPC stopped talking or was destroyed). Checks performed: {checkCount}", ConvaiLogger.LogCategory.Character);
// Clear sent clips when done
_sentClips.Clear();
}
private IEnumerator ContinuousAudioMonitoring()
{
ConvaiLogger.Info("🔊 Starting continuous audio monitoring as fallback", ConvaiLogger.LogCategory.Character);
AudioClip lastMonitoredClip = null;
while (true)
{
// Wait a bit between checks
yield return new WaitForSeconds(0.2f);
// Check if we still have a valid source NPC
if (sourceNPC == null || sourceNPC.AudioManager == null)
{
yield return new WaitForSeconds(1f); // Wait longer if no NPC
continue;
}
// Get the audio source
AudioSource audioSource = sourceNPC.AudioManager.GetComponent<AudioSource>();
if (audioSource == null)
{
yield return new WaitForSeconds(1f);
continue;
}
// Check if there's a new audio clip playing
if (audioSource.clip != null &&
audioSource.clip != lastMonitoredClip &&
audioSource.isPlaying)
{
lastMonitoredClip = audioSource.clip;
// Only send if we haven't sent this clip before
if (!_sentClips.Contains(lastMonitoredClip))
{
_sentClips.Add(lastMonitoredClip);
ConvaiLogger.Info($"🔊 [Continuous Monitor] NEW CLIP DETECTED: {lastMonitoredClip.name}, length: {lastMonitoredClip.length:F2}s", ConvaiLogger.LogCategory.Character);
// Start transmission if not already started
if (!_isSendingSpeech)
{
_isSendingSpeech = true;
OnSpeechTransmission?.Invoke(true);
}
string transcript = "";
_ = TransmitAudioClip(lastMonitoredClip, transcript);
}
}
// Clean up old clips from the sent list if NPC is not talking
if (!sourceNPC.IsCharacterTalking && _sentClips.Count > 0)
{
if (enableDebugLogging)
ConvaiLogger.DebugLog($"🔊 [Continuous Monitor] NPC stopped talking, clearing sent clips list ({_sentClips.Count} clips)", ConvaiLogger.LogCategory.Character);
_sentClips.Clear();
// Send final packet
if (_isSendingSpeech)
{
_ = SendFinalPacket();
}
}
}
}
private string GetRecentTranscript()
{
// Try to get transcript from the NPC's recent activity
@ -569,6 +664,9 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
ConvaiNPCManager.Instance.OnActiveNPCChanged -= HandleActiveNPCChanged;
}
// Stop all coroutines when cleaning up (will restart with new NPC)
StopAllCoroutines();
}
private void CleanupNetwork()

48
Unity-Master/Assets/Scripts/Multiplayer/UDPPeerDiscovery.cs
View File

@ -39,6 +39,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
private byte _peerPlayerID = 0;
private DateTime _lastPeerPacketTime;
private ConnectionState _connectionState = ConnectionState.Disconnected;
private bool _hasEverConnected = false; // Track if we've connected this session
// Discovery protocol constants
private const uint DISCOVERY_MAGIC = 0x44495343; // "DISC" in hex
@ -98,6 +99,8 @@ namespace Convai.Scripts.Runtime.Multiplayer
}
_cancellationTokenSource = new CancellationTokenSource();
// Always start with discovery on game launch (IP addresses may have changed)
LogEvent($"🔍 Discovery started (Player {localPlayerID})");
StartDiscovery();
}
@ -200,8 +203,9 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
try
{
// Only broadcast if we're still discovering
if (_connectionState == ConnectionState.Discovering || _connectionState == ConnectionState.Lost)
// Only broadcast if we're discovering AND haven't connected yet this session
// Once connected, we remember the peer IP for this session
if (_connectionState == ConnectionState.Discovering && !_hasEverConnected)
{
await SendDiscoveryRequest();
}
@ -227,8 +231,10 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
await Task.Delay((int)(heartbeatInterval * 1000), cancellationToken);
// Only send heartbeat if connected
if (_connectionState == ConnectionState.Connected && !string.IsNullOrEmpty(_peerIP))
// Send heartbeat if we have a peer IP (Connected or Lost state)
// This allows automatic reconnection when peer comes back online
if (!string.IsNullOrEmpty(_peerIP) &&
(_connectionState == ConnectionState.Connected || _connectionState == ConnectionState.Lost || _connectionState == ConnectionState.Discovering))
{
await SendHeartbeat();
}
@ -382,13 +388,20 @@ namespace Convai.Scripts.Runtime.Multiplayer
case PACKET_TYPE_HEARTBEAT:
// Heartbeat keeps connection alive
if (_connectionState == ConnectionState.Connected && _peerIP == senderIP)
if (_peerIP == senderIP)
{
// Connection still alive
// Same peer - update timestamp and reconnect if we were in Lost state
if (_connectionState == ConnectionState.Lost || _connectionState == ConnectionState.Discovering)
{
// Reconnected to known peer
SetConnectionState(ConnectionState.Connected);
ConvaiLogger.Info($"🔄 Reconnected to peer Player {senderPlayerID} at {senderIP}", ConvaiLogger.LogCategory.Character);
LogEvent($"🔄 Reconnected to peer Player {senderPlayerID}");
}
}
else if (_connectionState != ConnectionState.Connected)
{
// Reconnected
// Different peer IP than expected - this is a new peer
HandlePeerDiscovered(senderIP, senderPlayerID);
}
break;
@ -405,10 +418,11 @@ namespace Convai.Scripts.Runtime.Multiplayer
_peerIP = peerIP;
_peerPlayerID = peerPlayerID;
_lastPeerPacketTime = DateTime.UtcNow;
_hasEverConnected = true; // Mark that we've connected this session
SetConnectionState(ConnectionState.Connected);
ConvaiLogger.Info($"✅ Peer discovered! Player {peerPlayerID} at {peerIP}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"✅ Peer discovered! Player {peerPlayerID} at {peerIP} (will remember for this session)", ConvaiLogger.LogCategory.Character);
LogEvent($"✅ Peer discovered! Player {peerPlayerID} at {peerIP}");
// Notify listeners
@ -417,20 +431,18 @@ namespace Convai.Scripts.Runtime.Multiplayer
private void HandlePeerLost()
{
ConvaiLogger.Warn($"⚠️ Peer connection lost (Player {_peerPlayerID} at {_peerIP})", ConvaiLogger.LogCategory.Character);
LogEvent($"⚠️ Peer connection lost (Player {_peerPlayerID})");
string lostPeerIP = _peerIP;
_peerIP = "";
_peerPlayerID = 0;
ConvaiLogger.Warn($"⚠️ Peer connection lost (Player {_peerPlayerID} at {_peerIP}) - will keep trying to reconnect", ConvaiLogger.LogCategory.Character);
LogEvent($"⚠️ Peer connection lost (Player {_peerPlayerID}) - reconnecting...");
SetConnectionState(ConnectionState.Lost);
// Notify listeners
OnPeerLost?.Invoke();
// Restart discovery
SetConnectionState(ConnectionState.Discovering);
// Don't clear peer IP - keep it and keep trying to reconnect
// The heartbeat system will continue sending packets to the known peer
// and will automatically reconnect when the peer comes back online
ConvaiLogger.Info($"🔄 Will continue sending heartbeats to {_peerIP}", ConvaiLogger.LogCategory.Character);
}
private void SetConnectionState(ConnectionState newState)
@ -452,8 +464,11 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
ConvaiLogger.Info("Manually restarting peer discovery", ConvaiLogger.LogCategory.Character);
LogEvent("🔄 Manually restarting discovery");
// Reset state
_peerIP = "";
_peerPlayerID = 0;
_hasEverConnected = false;
SetConnectionState(ConnectionState.Discovering);
}
@ -465,6 +480,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
ConvaiLogger.Info($"Peer IP: {(_peerIP ?? "None")}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"Peer Player ID: {_peerPlayerID}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"Listen Port: {_listenPort}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"Has Ever Connected This Session: {_hasEverConnected}", ConvaiLogger.LogCategory.Character);
}
private void LogEvent(string message)