diff --git a/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs b/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs
index a7ab0ae..4f1d474 100644
--- a/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs
+++ b/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs
@@ -177,13 +177,9 @@ namespace Convai.Scripts.Runtime.Multiplayer
private void HandlePeerLost()
{
- var cfg = NetworkConfig.Instance;
- if (cfg != null)
- {
- targetIP = cfg.fallbackBroadcastIP;
- _targetEndPoint = new IPEndPoint(IPAddress.Parse(targetIP), targetPort);
- ConvaiLogger.Warn($"🎤 Audio sender falling back to broadcast: {targetIP}", ConvaiLogger.LogCategory.Character);
- }
+ // Don't change targetIP - keep sending to the last known peer IP
+ // The peer might come back online and we'll automatically reconnect
+ ConvaiLogger.Warn($"🎤 Audio sender: Peer connection lost, but continuing to send to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
}
private void InitializeNetwork()
diff --git a/Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechReceiver.cs b/Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechReceiver.cs
index 89ed8b8..4cb95f2 100644
--- a/Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechReceiver.cs
+++ b/Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechReceiver.cs
@@ -11,8 +11,18 @@ using UnityEngine;
namespace Convai.Scripts.Runtime.Multiplayer
{
///
- /// UDP Speech Receiver - Receives high-quality Convai speech with proper buffering
- /// This version reconstructs the original AudioClip objects for seamless playback
+ /// UDP Speech Receiver - Receives and plays NPC speech audio from remote player
+ ///
+ /// FLOW (Player 1 → Player 2):
+ /// 1. Player 2 speaks to Player 1's NPC
+ /// 2. Player 1's NPC responds with speech
+ /// 3. Player 1's ConvaiUDPSpeechSender transmits the audio
+ /// 4. THIS COMPONENT receives the audio packets
+ /// 5. Reconstructs AudioClips from the packets
+ /// 6. Plays them back on local AudioSource
+ ///
+ /// This component should be on a NetworkManager or similar persistent object.
+ /// It receives speech from the remote player's NPC.
///
public class ConvaiUDPSpeechReceiver : MonoBehaviour
{
diff --git a/Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechSender.cs b/Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechSender.cs
index ec45a7e..2629e4d 100644
--- a/Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechSender.cs
+++ b/Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechSender.cs
@@ -12,8 +12,18 @@ using System.Collections;
namespace Convai.Scripts.Runtime.Multiplayer
{
///
- /// UDP Speech Sender - Simple and reliable approach using events
- /// Hooks into AudioManager events to capture when clips are about to be played
+ /// UDP Speech Sender - Captures and transmits NPC speech audio to remote player
+ ///
+ /// FLOW (Player 1 → Player 2):
+ /// 1. Player 2 speaks (via ConvaiSimpleUDPAudioSender on their device)
+ /// 2. Player 1 receives voice input (via ConvaiSimpleUDPAudioReceiver)
+ /// 3. Player 1's NPC generates response speech (Convai API)
+ /// 4. THIS COMPONENT monitors Player 1's NPC AudioSource
+ /// 5. When new AudioClips appear, transmit them to Player 2
+ /// 6. Player 2's ConvaiUDPSpeechReceiver plays the audio
+ ///
+ /// This component should be on a NetworkManager or similar persistent object.
+ /// It will find and monitor ConvaiNPC components on Avatar objects in the scene.
///
public class ConvaiUDPSpeechSender : MonoBehaviour
{
@@ -158,13 +168,9 @@ namespace Convai.Scripts.Runtime.Multiplayer
private void HandlePeerLost()
{
- var cfg = NetworkConfig.Instance;
- if (cfg != null)
- {
- targetIP = cfg.fallbackBroadcastIP;
- _targetEndPoint = new IPEndPoint(IPAddress.Parse(targetIP), targetPort);
- ConvaiLogger.Warn($"🔊 Speech sender falling back to broadcast: {targetIP}", ConvaiLogger.LogCategory.Character);
- }
+ // Don't change targetIP - keep sending to the last known peer IP
+ // The peer might come back online and we'll automatically reconnect
+ ConvaiLogger.Warn($"🔊 Speech sender: Peer connection lost, but continuing to send to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
}
private void InitializeNetwork()
@@ -190,10 +196,19 @@ namespace Convai.Scripts.Runtime.Multiplayer
if (localNPC != null)
{
sourceNPC = localNPC;
+ ConvaiLogger.Info($"Speech Sender: Using local NPC {sourceNPC.characterName}", ConvaiLogger.LogCategory.Character);
}
else if (useActiveNPC)
{
sourceNPC = FindEnabledConvaiNPC();
+ if (sourceNPC != null)
+ {
+ ConvaiLogger.Info($"Speech Sender: Found NPC {sourceNPC.characterName} on {sourceNPC.gameObject.name}", ConvaiLogger.LogCategory.Character);
+ }
+ else
+ {
+ ConvaiLogger.Warn("Speech Sender: No ConvaiNPC found in scene yet", ConvaiLogger.LogCategory.Character);
+ }
}
SubscribeToNPCEvents();
@@ -224,6 +239,9 @@ namespace Convai.Scripts.Runtime.Multiplayer
sourceNPC.AudioManager.OnAudioTranscriptAvailable += HandleTranscriptAvailable;
ConvaiLogger.Info($"✅ UDP Speech Sender subscribed to NPC: {sourceNPC.characterName} (on {sourceNPC.gameObject.name}), AudioManager: {sourceNPC.AudioManager.name}", ConvaiLogger.LogCategory.Character);
+
+ // Also start continuous monitoring as a fallback in case events don't fire
+ StartCoroutine(ContinuousAudioMonitoring());
}
private void HandleCharacterTalkingChanged(bool isTalking)
@@ -256,27 +274,36 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
if (sourceNPC?.AudioManager == null)
{
- ConvaiLogger.Warn("MonitorAudioClips: AudioManager is null", ConvaiLogger.LogCategory.Character);
+ ConvaiLogger.Error("MonitorAudioClips: AudioManager is null on sourceNPC", ConvaiLogger.LogCategory.Character);
yield break;
}
AudioSource audioSource = sourceNPC.AudioManager.GetComponent();
if (audioSource == null)
{
- ConvaiLogger.Warn("MonitorAudioClips: AudioSource is null", ConvaiLogger.LogCategory.Character);
+ ConvaiLogger.Error($"MonitorAudioClips: No AudioSource found on AudioManager ({sourceNPC.AudioManager.name})", ConvaiLogger.LogCategory.Character);
yield break;
}
- ConvaiLogger.Info($"🔊 Started monitoring audio clips on {audioSource.name}", ConvaiLogger.LogCategory.Character);
+ ConvaiLogger.Info($"🔊 Started monitoring audio clips on {audioSource.name} for NPC {sourceNPC.characterName}", ConvaiLogger.LogCategory.Character);
AudioClip lastClip = null;
+ int checkCount = 0;
- while (sourceNPC.IsCharacterTalking)
+ while (sourceNPC != null && sourceNPC.IsCharacterTalking)
{
+ checkCount++;
+
+ // Log periodically to show we're still monitoring
+ if (enableDebugLogging && checkCount % 10 == 0)
+ {
+ ConvaiLogger.DebugLog($"🔊 Monitoring... check #{checkCount}, current clip: {(audioSource?.clip != null ? audioSource.clip.name : "null")}, isTalking: {sourceNPC.IsCharacterTalking}", ConvaiLogger.LogCategory.Character);
+ }
+
if (audioSource?.clip != null && audioSource.clip != lastClip)
{
// New clip detected!
lastClip = audioSource.clip;
- ConvaiLogger.Info($"🔊 Detected new audio clip: {lastClip.name}, length: {lastClip.length:F2}s", ConvaiLogger.LogCategory.Character);
+ ConvaiLogger.Info($"🔊 NEW CLIP DETECTED: {lastClip.name}, length: {lastClip.length:F2}s, samples: {lastClip.samples}, freq: {lastClip.frequency}", ConvaiLogger.LogCategory.Character);
// Only send if we haven't sent this clip before
if (!_sentClips.Contains(lastClip))
@@ -287,23 +314,91 @@ namespace Convai.Scripts.Runtime.Multiplayer
string transcript = GetRecentTranscript();
// Send this clip
- ConvaiLogger.Info($"🔊 Transmitting audio clip to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
+ ConvaiLogger.Info($"🔊 TRANSMITTING CLIP to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
_ = TransmitAudioClip(lastClip, transcript);
}
else
{
- ConvaiLogger.Info($"🔊 Clip already sent, skipping", ConvaiLogger.LogCategory.Character);
+ ConvaiLogger.Warn($"🔊 Clip already sent, skipping: {lastClip.name}", ConvaiLogger.LogCategory.Character);
}
}
yield return new WaitForSeconds(0.1f); // Check every 100ms
}
- ConvaiLogger.Info($"🔊 Stopped monitoring audio clips (NPC stopped talking)", ConvaiLogger.LogCategory.Character);
+ ConvaiLogger.Info($"🔊 Stopped monitoring audio clips (NPC stopped talking or was destroyed). Checks performed: {checkCount}", ConvaiLogger.LogCategory.Character);
// Clear sent clips when done
_sentClips.Clear();
}
+ private IEnumerator ContinuousAudioMonitoring()
+ {
+ ConvaiLogger.Info("🔊 Starting continuous audio monitoring as fallback", ConvaiLogger.LogCategory.Character);
+ AudioClip lastMonitoredClip = null;
+
+ while (true)
+ {
+ // Wait a bit between checks
+ yield return new WaitForSeconds(0.2f);
+
+ // Check if we still have a valid source NPC
+ if (sourceNPC == null || sourceNPC.AudioManager == null)
+ {
+ yield return new WaitForSeconds(1f); // Wait longer if no NPC
+ continue;
+ }
+
+ // Get the audio source
+ AudioSource audioSource = sourceNPC.AudioManager.GetComponent();
+ if (audioSource == null)
+ {
+ yield return new WaitForSeconds(1f);
+ continue;
+ }
+
+ // Check if there's a new audio clip playing
+ if (audioSource.clip != null &&
+ audioSource.clip != lastMonitoredClip &&
+ audioSource.isPlaying)
+ {
+ lastMonitoredClip = audioSource.clip;
+
+ // Only send if we haven't sent this clip before
+ if (!_sentClips.Contains(lastMonitoredClip))
+ {
+ _sentClips.Add(lastMonitoredClip);
+
+ ConvaiLogger.Info($"🔊 [Continuous Monitor] NEW CLIP DETECTED: {lastMonitoredClip.name}, length: {lastMonitoredClip.length:F2}s", ConvaiLogger.LogCategory.Character);
+
+ // Start transmission if not already started
+ if (!_isSendingSpeech)
+ {
+ _isSendingSpeech = true;
+ OnSpeechTransmission?.Invoke(true);
+ }
+
+ string transcript = "";
+ _ = TransmitAudioClip(lastMonitoredClip, transcript);
+ }
+ }
+
+ // Clean up old clips from the sent list if NPC is not talking
+ if (!sourceNPC.IsCharacterTalking && _sentClips.Count > 0)
+ {
+ if (enableDebugLogging)
+ ConvaiLogger.DebugLog($"🔊 [Continuous Monitor] NPC stopped talking, clearing sent clips list ({_sentClips.Count} clips)", ConvaiLogger.LogCategory.Character);
+
+ _sentClips.Clear();
+
+ // Send final packet
+ if (_isSendingSpeech)
+ {
+ _ = SendFinalPacket();
+ }
+ }
+ }
+ }
+
private string GetRecentTranscript()
{
// Try to get transcript from the NPC's recent activity
@@ -569,6 +664,9 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
ConvaiNPCManager.Instance.OnActiveNPCChanged -= HandleActiveNPCChanged;
}
+
+ // Stop all coroutines when cleaning up (will restart with new NPC)
+ StopAllCoroutines();
}
private void CleanupNetwork()
diff --git a/Unity-Master/Assets/Scripts/Multiplayer/UDPPeerDiscovery.cs b/Unity-Master/Assets/Scripts/Multiplayer/UDPPeerDiscovery.cs
index 311d7e7..6fe1850 100644
--- a/Unity-Master/Assets/Scripts/Multiplayer/UDPPeerDiscovery.cs
+++ b/Unity-Master/Assets/Scripts/Multiplayer/UDPPeerDiscovery.cs
@@ -39,6 +39,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
private byte _peerPlayerID = 0;
private DateTime _lastPeerPacketTime;
private ConnectionState _connectionState = ConnectionState.Disconnected;
+ private bool _hasEverConnected = false; // Track if we've connected this session
// Discovery protocol constants
private const uint DISCOVERY_MAGIC = 0x44495343; // "DISC" in hex
@@ -98,6 +99,8 @@ namespace Convai.Scripts.Runtime.Multiplayer
}
_cancellationTokenSource = new CancellationTokenSource();
+
+ // Always start with discovery on game launch (IP addresses may have changed)
LogEvent($"🔍 Discovery started (Player {localPlayerID})");
StartDiscovery();
}
@@ -200,8 +203,9 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
try
{
- // Only broadcast if we're still discovering
- if (_connectionState == ConnectionState.Discovering || _connectionState == ConnectionState.Lost)
+ // Only broadcast if we're discovering AND haven't connected yet this session
+ // Once connected, we remember the peer IP for this session
+ if (_connectionState == ConnectionState.Discovering && !_hasEverConnected)
{
await SendDiscoveryRequest();
}
@@ -227,8 +231,10 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
await Task.Delay((int)(heartbeatInterval * 1000), cancellationToken);
- // Only send heartbeat if connected
- if (_connectionState == ConnectionState.Connected && !string.IsNullOrEmpty(_peerIP))
+ // Send heartbeat if we have a peer IP (Connected or Lost state)
+ // This allows automatic reconnection when peer comes back online
+ if (!string.IsNullOrEmpty(_peerIP) &&
+ (_connectionState == ConnectionState.Connected || _connectionState == ConnectionState.Lost || _connectionState == ConnectionState.Discovering))
{
await SendHeartbeat();
}
@@ -382,13 +388,20 @@ namespace Convai.Scripts.Runtime.Multiplayer
case PACKET_TYPE_HEARTBEAT:
// Heartbeat keeps connection alive
- if (_connectionState == ConnectionState.Connected && _peerIP == senderIP)
+ if (_peerIP == senderIP)
{
- // Connection still alive
+ // Same peer - update timestamp and reconnect if we were in Lost state
+ if (_connectionState == ConnectionState.Lost || _connectionState == ConnectionState.Discovering)
+ {
+ // Reconnected to known peer
+ SetConnectionState(ConnectionState.Connected);
+ ConvaiLogger.Info($"🔄 Reconnected to peer Player {senderPlayerID} at {senderIP}", ConvaiLogger.LogCategory.Character);
+ LogEvent($"🔄 Reconnected to peer Player {senderPlayerID}");
+ }
}
else if (_connectionState != ConnectionState.Connected)
{
- // Reconnected
+ // Different peer IP than expected - this is a new peer
HandlePeerDiscovered(senderIP, senderPlayerID);
}
break;
@@ -405,10 +418,11 @@ namespace Convai.Scripts.Runtime.Multiplayer
_peerIP = peerIP;
_peerPlayerID = peerPlayerID;
_lastPeerPacketTime = DateTime.UtcNow;
+ _hasEverConnected = true; // Mark that we've connected this session
SetConnectionState(ConnectionState.Connected);
- ConvaiLogger.Info($"✅ Peer discovered! Player {peerPlayerID} at {peerIP}", ConvaiLogger.LogCategory.Character);
+ ConvaiLogger.Info($"✅ Peer discovered! Player {peerPlayerID} at {peerIP} (will remember for this session)", ConvaiLogger.LogCategory.Character);
LogEvent($"✅ Peer discovered! Player {peerPlayerID} at {peerIP}");
// Notify listeners
@@ -417,20 +431,18 @@ namespace Convai.Scripts.Runtime.Multiplayer
private void HandlePeerLost()
{
- ConvaiLogger.Warn($"⚠️ Peer connection lost (Player {_peerPlayerID} at {_peerIP})", ConvaiLogger.LogCategory.Character);
- LogEvent($"⚠️ Peer connection lost (Player {_peerPlayerID})");
-
- string lostPeerIP = _peerIP;
- _peerIP = "";
- _peerPlayerID = 0;
+ ConvaiLogger.Warn($"⚠️ Peer connection lost (Player {_peerPlayerID} at {_peerIP}) - will keep trying to reconnect", ConvaiLogger.LogCategory.Character);
+ LogEvent($"⚠️ Peer connection lost (Player {_peerPlayerID}) - reconnecting...");
SetConnectionState(ConnectionState.Lost);
// Notify listeners
OnPeerLost?.Invoke();
- // Restart discovery
- SetConnectionState(ConnectionState.Discovering);
+ // Don't clear peer IP - keep it and keep trying to reconnect
+ // The heartbeat system will continue sending packets to the known peer
+ // and will automatically reconnect when the peer comes back online
+ ConvaiLogger.Info($"🔄 Will continue sending heartbeats to {_peerIP}", ConvaiLogger.LogCategory.Character);
}
private void SetConnectionState(ConnectionState newState)
@@ -452,8 +464,11 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
ConvaiLogger.Info("Manually restarting peer discovery", ConvaiLogger.LogCategory.Character);
LogEvent("🔄 Manually restarting discovery");
+
+ // Reset state
_peerIP = "";
_peerPlayerID = 0;
+ _hasEverConnected = false;
SetConnectionState(ConnectionState.Discovering);
}
@@ -465,6 +480,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
ConvaiLogger.Info($"Peer IP: {(_peerIP ?? "None")}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"Peer Player ID: {_peerPlayerID}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"Listen Port: {_listenPort}", ConvaiLogger.LogCategory.Character);
+ ConvaiLogger.Info($"Has Ever Connected This Session: {_hasEverConnected}", ConvaiLogger.LogCategory.Character);
}
private void LogEvent(string message)