From 3f72973ff51a52bb35484a43d75b93003a3f894f Mon Sep 17 00:00:00 2001 From: "tom.hempel" Date: Sat, 25 Oct 2025 14:58:22 +0200 Subject: [PATCH] created centralized UDP Listener --- .../Assets/Resources/NetworkConfig.asset | 4 +- .../ConvaiSimpleUDPAudioReceiver.cs | 144 +++++-------- .../Multiplayer/ConvaiSimpleUDPAudioSender.cs | 141 ++----------- .../Multiplayer/ConvaiUDPSpeechReceiver.cs | 62 +++--- .../Multiplayer/ConvaiUDPSpeechSender.cs | 32 ++- .../Scripts/Multiplayer/NetworkDebugUI.cs | 149 +++++++++++++- .../Scripts/Multiplayer/SharedUDPListener.cs | 192 ++++++++++++++++++ .../Scripts/Multiplayer/UDPPeerDiscovery.cs | 69 +++---- .../Assets/Scripts/UDPAvatarReceiver.cs | 163 +++++---------- .../Assets/Scripts/UDPAvatarReceiverAgent.cs | 163 +++++---------- 10 files changed, 585 insertions(+), 534 deletions(-) create mode 100644 Unity-Master/Assets/Scripts/Multiplayer/SharedUDPListener.cs diff --git a/Unity-Master/Assets/Resources/NetworkConfig.asset b/Unity-Master/Assets/Resources/NetworkConfig.asset index 358fcda..a80de39 100644 --- a/Unity-Master/Assets/Resources/NetworkConfig.asset +++ b/Unity-Master/Assets/Resources/NetworkConfig.asset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:54d451cef255fb6db3a3a443957065843bf5aa42025cebc34890aa3933c4e5d9 -size 559 +oid sha256:0fae866d31d471672104cfeffea2b604d88c956afbd84c04455c9679bd727cf4 +size 557 diff --git a/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioReceiver.cs b/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioReceiver.cs index 3521308..4fb5a23 100644 --- a/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioReceiver.cs +++ b/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioReceiver.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Net; -using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; using Convai.Scripts.Runtime.Core; @@ -37,7 +36,6 @@ namespace Convai.Scripts.Runtime.Multiplayer public int ListenPort => listenPort; // Network components - private UdpClient _udpListener; private IPEndPoint _remoteEndPoint; private bool _isListening = false; private int listenPort; @@ -47,24 +45,19 @@ namespace Convai.Scripts.Runtime.Multiplayer private bool _isReceivingAudio = false; private int _expectedSequence = 0; private const uint MAGIC_NUMBER = 0xC0A1; // Simple magic number for packet validation - private const uint ACK_MAGIC = 0xC0A2; // ACK magic to confirm START control // Timing for auto-stop private float _lastPacketTime; private const float AUTO_STOP_DELAY = 1.0f; // Stop listening after 1 second of no packets - // Packet structure (matching ConvaiSimpleUDPAudioSender) + // SIMPLIFIED Packet structure (matching ConvaiSimpleUDPAudioSender) private struct AudioPacketData { public uint magicNumber; public int sequence; public int sampleCount; - public int microphonePosition; - public bool isEndSignal; - public bool isStartSignal; public short[] audioSamples; - public long timestamp; } [Header("Recording Storage")] @@ -234,20 +227,22 @@ namespace Convai.Scripts.Runtime.Multiplayer try { - // Create UDP client with port reuse to allow sharing with UDPPeerDiscovery - _udpListener = new UdpClient(); - _udpListener.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); - _udpListener.Client.Bind(new IPEndPoint(IPAddress.Any, listenPort)); + // Wait for shared listener to be ready + if (SharedUDPListener.Instance == null) + { + ConvaiLogger.Error("SharedUDPListener not found! Make sure it's in the scene.", ConvaiLogger.LogCategory.Character); + return; + } + + // Subscribe to shared listener + SharedUDPListener.Instance.OnPacketReceived += HandlePacketReceived; _isListening = true; - ConvaiLogger.Info($"Simple UDP Audio Receiver V2 listening on port {listenPort} (shared)", ConvaiLogger.LogCategory.Character); - - // Start listening for incoming packets - _ = ListenForAudioPackets(_cancellationTokenSource.Token); + ConvaiLogger.Info($"✅ Audio Receiver subscribed to shared listener, listening for magic 0x{MAGIC_NUMBER:X}", ConvaiLogger.LogCategory.Character); } catch (Exception ex) { - ConvaiLogger.Error($"Failed to start UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character); + ConvaiLogger.Error($"❌ FAILED to subscribe Audio Receiver: {ex.Message}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Error($"Stack trace: {ex.StackTrace}", ConvaiLogger.LogCategory.Character); } } @@ -258,9 +253,12 @@ namespace Convai.Scripts.Runtime.Multiplayer return; _isListening = false; - _udpListener?.Close(); - _udpListener?.Dispose(); - _udpListener = null; + + // Unsubscribe from shared listener + if (SharedUDPListener.Instance != null) + { + SharedUDPListener.Instance.OnPacketReceived -= HandlePacketReceived; + } // Stop any ongoing simulation StopTalkingSimulation(); @@ -268,26 +266,19 @@ namespace Convai.Scripts.Runtime.Multiplayer ConvaiLogger.Info("Stopped UDP Audio Receiver V2", ConvaiLogger.LogCategory.Character); } - private async Task ListenForAudioPackets(CancellationToken cancellationToken) + private void HandlePacketReceived(byte[] data, IPEndPoint senderEndPoint) { - try - { - while (_isListening && !cancellationToken.IsCancellationRequested) - { - var result = await _udpListener.ReceiveAsync(); - _remoteEndPoint = result.RemoteEndPoint; - - await ProcessReceivedPacket(result.Buffer, result.RemoteEndPoint); - } - } - catch (ObjectDisposedException) - { - // Normal when stopping - } - catch (Exception ex) - { - ConvaiLogger.Error($"Error in UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character); - } + // Check if this is an audio packet (by magic number) + if (data.Length < 12) return; + + uint magic = BitConverter.ToUInt32(data, 0); + if (magic != MAGIC_NUMBER) return; + + // Update remote endpoint + _remoteEndPoint = senderEndPoint; + + // Process audio packet + _ = ProcessReceivedPacket(data, senderEndPoint); } private Task ProcessReceivedPacket(byte[] data, IPEndPoint sender) @@ -305,28 +296,17 @@ namespace Convai.Scripts.Runtime.Multiplayer _totalPacketsReceived++; _lastPacketReceivedTime = DateTime.UtcNow; - if (enableDebugLogging) - { - if (packet.isEndSignal) - ConvaiLogger.DebugLog($"Received end signal from {sender}", ConvaiLogger.LogCategory.Character); - else - ConvaiLogger.DebugLog($"Received audio packet {packet.sequence} with {packet.sampleCount} samples", ConvaiLogger.LogCategory.Character); - } + // SIMPLIFIED: Check for end signal (sampleCount == -1) + bool isEndSignal = (packet.sampleCount == -1); - // Handle START control: acknowledge and begin simulation - if (packet.isStartSignal) + if (enableDebugLogging && packet.sequence % 50 == 0) { - SendStartAck(sender); - if (!_isReceivingAudio) - { - StartTalkingSimulation(); - } - OnAudioReceiving?.Invoke(true); - return Task.CompletedTask; + ConvaiLogger.DebugLog($"📥 Received audio packet #{packet.sequence} (magic: 0x{packet.magicNumber:X}) from {sender}, {packet.sampleCount} samples", ConvaiLogger.LogCategory.Character); } - if (packet.isEndSignal) + if (isEndSignal) { + ConvaiLogger.Info($"📥 Received END signal from {sender}", ConvaiLogger.LogCategory.Character); StopTalkingSimulation(); OnAudioReceiving?.Invoke(false); } @@ -335,6 +315,7 @@ namespace Convai.Scripts.Runtime.Multiplayer // If this is the first packet, start the talking simulation if (packet.sequence == 0 && !_isReceivingAudio) { + ConvaiLogger.Info($"📥 Received FIRST audio packet from {sender}, starting simulation", ConvaiLogger.LogCategory.Character); StartTalkingSimulation(); } @@ -418,7 +399,7 @@ namespace Convai.Scripts.Runtime.Multiplayer private AudioPacketData? ParseSimpleAudioPacket(byte[] data) { - if (data.Length < 17) // Minimum header size to match sender + if (data.Length < 12) // SIMPLIFIED: Minimum header size (4+4+4) return null; try @@ -430,27 +411,22 @@ namespace Convai.Scripts.Runtime.Multiplayer offset += 4; if (magic != MAGIC_NUMBER) + { + if (enableDebugLogging) + ConvaiLogger.Warn($"❌ Invalid magic number: expected 0x{MAGIC_NUMBER:X}, got 0x{magic:X}", ConvaiLogger.LogCategory.Character); return null; + } - // Read header (matching sender's 17-byte format) + // SIMPLIFIED header (12 bytes total) int sequence = BitConverter.ToInt32(data, offset); offset += 4; int sampleCount = BitConverter.ToInt32(data, offset); offset += 4; - int microphonePosition = BitConverter.ToInt32(data, offset); - offset += 4; - - byte flags = data[offset]; - offset += 1; - - bool isEndSignal = (flags == 1); - bool isStartSignal = (flags == 2); - - // Read audio data + // Read audio data (if not end signal) short[] audioSamples = null; - if (!isEndSignal && !isStartSignal && sampleCount > 0) + if (sampleCount > 0) { int audioDataSize = sampleCount * sizeof(short); if (data.Length >= offset + audioDataSize) @@ -458,6 +434,10 @@ namespace Convai.Scripts.Runtime.Multiplayer audioSamples = new short[sampleCount]; Buffer.BlockCopy(data, offset, audioSamples, 0, audioDataSize); } + else + { + ConvaiLogger.Warn($"⚠️ Packet too small: expected {offset + audioDataSize} bytes, got {data.Length}", ConvaiLogger.LogCategory.Character); + } } return new AudioPacketData @@ -465,11 +445,7 @@ namespace Convai.Scripts.Runtime.Multiplayer magicNumber = magic, sequence = sequence, sampleCount = sampleCount, - microphonePosition = microphonePosition, - isEndSignal = isEndSignal, - isStartSignal = isStartSignal, - audioSamples = audioSamples, - timestamp = 0 // Not provided in sender format + audioSamples = audioSamples }; } catch (Exception ex) @@ -479,26 +455,6 @@ namespace Convai.Scripts.Runtime.Multiplayer } } - private void SendStartAck(IPEndPoint sender) - { - try - { - if (_udpListener == null || sender == null) - return; - - byte[] ack = new byte[8]; - BitConverter.GetBytes(ACK_MAGIC).CopyTo(ack, 0); - BitConverter.GetBytes(-1).CopyTo(ack, 4); - _udpListener.SendAsync(ack, ack.Length, sender); - - if (enableDebugLogging) - ConvaiLogger.DebugLog($"Sent START ACK to {sender}", ConvaiLogger.LogCategory.Character); - } - catch (Exception ex) - { - ConvaiLogger.Warn($"Failed to send START ACK: {ex.Message}", ConvaiLogger.LogCategory.Character); - } - } private void BufferAudioPacket(int sequence, short[] samples) { diff --git a/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs b/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs index 4f1d474..b6aa6a6 100644 --- a/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs +++ b/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs @@ -48,25 +48,21 @@ namespace Convai.Scripts.Runtime.Multiplayer private AudioClip _audioClip; private bool _isRecording = false; private CancellationTokenSource _cancellationTokenSource; - private CancellationTokenSource _ackCancellationTokenSource; private int _lastMicrophonePosition = 0; private float[] _audioBuffer; private string _selectedMicrophone; private int _packetSequence = 0; - private volatile bool _startAckReceived = false; private bool _xrAButtonPrevPressed = false; private InputAction _xrTalkAction; private InputAction _xrTestAction; private bool _usingExternalTalkAction = false; private InputAction _externalTalkAction; - // Protocol constants + // Protocol constants - SIMPLIFIED private const uint AUDIO_MAGIC = 0xC0A1; - private const uint ACK_MAGIC = 0xC0A2; private const byte FLAG_AUDIO = 0; private const byte FLAG_END = 1; - private const byte FLAG_START = 2; public event Action OnRecordingStateChanged; @@ -118,10 +114,6 @@ namespace Convai.Scripts.Runtime.Multiplayer InitializeAudio(); _persistentDataPath = Application.persistentDataPath; _cancellationTokenSource = new CancellationTokenSource(); - _ackCancellationTokenSource = new CancellationTokenSource(); - - // Start ACK listener - _ = ListenForAcks(_ackCancellationTokenSource.Token); // Setup Input System action for XR A/primary button if (useInputSystemXR) @@ -158,8 +150,6 @@ namespace Convai.Scripts.Runtime.Multiplayer StopRecording(); _cancellationTokenSource?.Cancel(); _cancellationTokenSource?.Dispose(); - _ackCancellationTokenSource?.Cancel(); - _ackCancellationTokenSource?.Dispose(); if (_usingExternalTalkAction) TeardownExternalTalkInputAction(); else @@ -476,7 +466,6 @@ namespace Convai.Scripts.Runtime.Multiplayer _isRecording = true; _lastMicrophonePosition = 0; _packetSequence = 0; - _startAckReceived = false; _localSessionStart = DateTime.UtcNow; lock (_localAudioLock) { @@ -485,11 +474,8 @@ namespace Convai.Scripts.Runtime.Multiplayer ConvaiLogger.Info($"🎤 Started recording for UDP transmission to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character); OnRecordingStateChanged?.Invoke(true); - - // Send START control and wait briefly for ACK to ensure receiver is ready - _ = SendStartOfRecordingSignalAndAwaitAck(); - // Start continuous audio processing + // Start continuous audio processing immediately (SIMPLIFIED - no ACK waiting) _ = ProcessAudioContinuously(_cancellationTokenSource.Token); } catch (Exception ex) @@ -640,15 +626,13 @@ namespace Convai.Scripts.Runtime.Multiplayer private byte[] CreateSimpleAudioPacket(float[] audioData, int startIndex, int sampleCount) { - // Simple packet structure: + // SIMPLIFIED packet structure: // 4 bytes: Magic number (0xC0A1) // 4 bytes: Packet sequence number // 4 bytes: Sample count in this packet - // 4 bytes: Start position in stream - // 1 byte: Flags (0 = normal audio, 1 = end of recording) // N bytes: Audio data (converted to shorts) - int headerSize = 17; // 4 + 4 + 4 + 4 + 1 + int headerSize = 12; // 4 + 4 + 4 (SIMPLIFIED from 17) int audioDataSize = sampleCount * sizeof(short); byte[] packet = new byte[headerSize + audioDataSize]; @@ -666,15 +650,7 @@ namespace Convai.Scripts.Runtime.Multiplayer BitConverter.GetBytes(sampleCount).CopyTo(packet, offset); offset += 4; - // Start position - BitConverter.GetBytes(_lastMicrophonePosition + startIndex).CopyTo(packet, offset); - offset += 4; - - // Flags (0 for normal audio) - packet[offset] = FLAG_AUDIO; - offset += 1; - - // Convert audio samples to bytes (same as Convai approach) + // Convert audio samples to bytes for (int i = 0; i < sampleCount; i++) { float sample = audioData[startIndex + i]; @@ -685,6 +661,11 @@ namespace Convai.Scripts.Runtime.Multiplayer offset += 2; } + if (enableDebugLogging && _packetSequence % 50 == 0) + { + ConvaiLogger.DebugLog($"📤 Sent audio packet #{_packetSequence} (magic: 0x{AUDIO_MAGIC:X}) to {targetIP}:{targetPort}, {sampleCount} samples", ConvaiLogger.LogCategory.Character); + } + return packet; } @@ -692,8 +673,8 @@ namespace Convai.Scripts.Runtime.Multiplayer { try { - // Create end packet - byte[] packet = new byte[17]; // Header only, no audio data + // SIMPLIFIED end packet + byte[] packet = new byte[12]; // Header only, no audio data int offset = 0; // Magic number @@ -704,18 +685,12 @@ namespace Convai.Scripts.Runtime.Multiplayer BitConverter.GetBytes(_packetSequence).CopyTo(packet, offset); offset += 4; - // Sample count (0 for end signal) - BitConverter.GetBytes(0).CopyTo(packet, offset); - offset += 4; - - // Start position - BitConverter.GetBytes(_lastMicrophonePosition).CopyTo(packet, offset); - offset += 4; - - // Flags (1 for end of recording) - packet[offset] = FLAG_END; + // Sample count (-1 for end signal) + BitConverter.GetBytes(-1).CopyTo(packet, offset); _udpClient.SendAsync(packet, packet.Length, _targetEndPoint); + + ConvaiLogger.Info($"📤 Sent END signal (magic: 0x{AUDIO_MAGIC:X}) to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character); } catch (Exception ex) { @@ -817,90 +792,6 @@ namespace Convai.Scripts.Runtime.Multiplayer } } - private async Task SendStartOfRecordingSignalAndAwaitAck() - { - try - { - const int maxAttempts = 3; - const int ackTimeoutMs = 250; - - for (int attempt = 1; attempt <= maxAttempts && !_startAckReceived; attempt++) - { - // Build START control packet (no audio, special flag) - byte[] packet = new byte[17]; - int offset = 0; - BitConverter.GetBytes(AUDIO_MAGIC).CopyTo(packet, offset); - offset += 4; - // Use -1 as the special sequence for START control - BitConverter.GetBytes(-1).CopyTo(packet, offset); - offset += 4; - BitConverter.GetBytes(0).CopyTo(packet, offset); - offset += 4; - BitConverter.GetBytes(_lastMicrophonePosition).CopyTo(packet, offset); - offset += 4; - packet[offset] = FLAG_START; - - await _udpClient.SendAsync(packet, packet.Length, _targetEndPoint); - - // Wait for ACK - int waited = 0; - while (!_startAckReceived && waited < ackTimeoutMs) - { - await Task.Delay(10); - waited += 10; - } - - if (_startAckReceived) - { - if (enableDebugLogging) - ConvaiLogger.DebugLog("Received START ACK from receiver", ConvaiLogger.LogCategory.Character); - break; - } - else if (enableDebugLogging) - { - ConvaiLogger.Warn($"No START ACK (attempt {attempt}/{maxAttempts}), retrying...", ConvaiLogger.LogCategory.Character); - } - } - } - catch (Exception ex) - { - ConvaiLogger.Warn($"Error during START ACK process: {ex.Message}", ConvaiLogger.LogCategory.Character); - } - } - - private async Task ListenForAcks(CancellationToken token) - { - // Use the same UDP client; acks will be sent back to our ephemeral local port - while (!token.IsCancellationRequested) - { - try - { - var result = await _udpClient.ReceiveAsync(); - var data = result.Buffer; - if (data == null || data.Length < 8) - continue; - - uint magic = BitConverter.ToUInt32(data, 0); - if (magic != ACK_MAGIC) - continue; - - int seq = BitConverter.ToInt32(data, 4); - if (seq == -1) - { - _startAckReceived = true; - } - } - catch (ObjectDisposedException) - { - break; - } - catch (Exception) - { - // Ignore and keep listening - } - } - } - // Public methods for external control public void SetTargetEndpoint(string ip, int port) { diff --git a/Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechReceiver.cs b/Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechReceiver.cs index e20b7a9..ffc7417 100644 --- a/Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechReceiver.cs +++ b/Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechReceiver.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Net; -using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; using Convai.Scripts.Runtime.LoggerSystem; @@ -39,7 +38,6 @@ namespace Convai.Scripts.Runtime.Multiplayer [SerializeField] private bool showTranscripts = true; // Network components - private UdpClient _udpListener; private IPEndPoint _remoteEndPoint; private bool _isListening = false; private int listenPort; @@ -211,20 +209,22 @@ namespace Convai.Scripts.Runtime.Multiplayer try { - // Create UDP client with port reuse to allow sharing with UDPPeerDiscovery - _udpListener = new UdpClient(); - _udpListener.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); - _udpListener.Client.Bind(new IPEndPoint(IPAddress.Any, listenPort)); + // Wait for shared listener to be ready + if (SharedUDPListener.Instance == null) + { + ConvaiLogger.Error("SharedUDPListener not found! Make sure it's in the scene.", ConvaiLogger.LogCategory.Character); + return; + } + + // Subscribe to shared listener + SharedUDPListener.Instance.OnPacketReceived += HandlePacketReceived; _isListening = true; - ConvaiLogger.Info($"UDP Speech Receiver listening on port {listenPort} (shared)", ConvaiLogger.LogCategory.Character); - - // Start listening for incoming packets - _ = ListenForSpeechPackets(_cancellationTokenSource.Token); + ConvaiLogger.Info($"✅ Speech Receiver subscribed to shared listener, listening for magic 0x{MAGIC_NUMBER:X}", ConvaiLogger.LogCategory.Character); } catch (Exception ex) { - ConvaiLogger.Error($"Failed to start UDP speech receiver: {ex.Message}", ConvaiLogger.LogCategory.Character); + ConvaiLogger.Error($"❌ FAILED to subscribe Speech Receiver: {ex.Message}", ConvaiLogger.LogCategory.Character); } } @@ -234,9 +234,12 @@ namespace Convai.Scripts.Runtime.Multiplayer return; _isListening = false; - _udpListener?.Close(); - _udpListener?.Dispose(); - _udpListener = null; + + // Unsubscribe from shared listener + if (SharedUDPListener.Instance != null) + { + SharedUDPListener.Instance.OnPacketReceived -= HandlePacketReceived; + } // Stop any ongoing playback StopSpeechPlayback(); @@ -244,26 +247,19 @@ namespace Convai.Scripts.Runtime.Multiplayer ConvaiLogger.Info("Stopped UDP Speech Receiver", ConvaiLogger.LogCategory.Character); } - private async Task ListenForSpeechPackets(CancellationToken cancellationToken) + private void HandlePacketReceived(byte[] data, IPEndPoint senderEndPoint) { - try - { - while (_isListening && !cancellationToken.IsCancellationRequested) - { - var result = await _udpListener.ReceiveAsync(); - _remoteEndPoint = result.RemoteEndPoint; - - await ProcessReceivedPacket(result.Buffer, result.RemoteEndPoint); - } - } - catch (ObjectDisposedException) - { - // Normal when stopping - } - catch (Exception ex) - { - ConvaiLogger.Error($"Error in UDP speech listener: {ex.Message}", ConvaiLogger.LogCategory.Character); - } + // Check if this is a speech packet (by magic number) + if (data.Length < 4) return; + + uint magic = BitConverter.ToUInt32(data, 0); + if (magic != MAGIC_NUMBER) return; + + // Update remote endpoint + _remoteEndPoint = senderEndPoint; + + // Process speech packet + _ = ProcessReceivedPacket(data, senderEndPoint); } private Task ProcessReceivedPacket(byte[] data, IPEndPoint sender) diff --git a/Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechSender.cs b/Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechSender.cs index c11fa6e..d33486c 100644 --- a/Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechSender.cs +++ b/Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechSender.cs @@ -334,6 +334,7 @@ namespace Convai.Scripts.Runtime.Multiplayer { ConvaiLogger.Info("🔊 Starting continuous audio monitoring as fallback", ConvaiLogger.LogCategory.Character); AudioClip lastMonitoredClip = null; + float lastNPCCheckLog = 0f; while (true) { @@ -343,6 +344,29 @@ namespace Convai.Scripts.Runtime.Multiplayer // Check if we still have a valid source NPC if (sourceNPC == null || sourceNPC.AudioManager == null) { + // Log every 2 seconds if no NPC found (AGGRESSIVE LOGGING) + if (Time.time - lastNPCCheckLog > 2f) + { + if (sourceNPC == null) + ConvaiLogger.Warn("⚠️ [Speech Sender] No source NPC found! Looking for ConvaiNPC in scene...", ConvaiLogger.LogCategory.Character); + else + ConvaiLogger.Warn($"⚠️ [Speech Sender] NPC '{sourceNPC.characterName}' has no AudioManager!", ConvaiLogger.LogCategory.Character); + + // Try to find NPC again + if (useActiveNPC) + { + var foundNPC = FindEnabledConvaiNPC(); + if (foundNPC != null && foundNPC != sourceNPC) + { + ConvaiLogger.Info($"✅ [Speech Sender] Found NPC: {foundNPC.characterName} on GameObject '{foundNPC.gameObject.name}'", ConvaiLogger.LogCategory.Character); + sourceNPC = foundNPC; + SubscribeToNPCEvents(); + } + } + + lastNPCCheckLog = Time.time; + } + yield return new WaitForSeconds(1f); // Wait longer if no NPC continue; } @@ -351,6 +375,11 @@ namespace Convai.Scripts.Runtime.Multiplayer AudioSource audioSource = sourceNPC.AudioManager.GetComponent(); if (audioSource == null) { + if (Time.time - lastNPCCheckLog > 2f) + { + ConvaiLogger.Warn($"⚠️ [Speech Sender] NPC '{sourceNPC.characterName}' AudioManager has no AudioSource component!", ConvaiLogger.LogCategory.Character); + lastNPCCheckLog = Time.time; + } yield return new WaitForSeconds(1f); continue; } @@ -455,8 +484,7 @@ namespace Convai.Scripts.Runtime.Multiplayer byte[] packet = CreateAudioStartPacket(audioClip, transcript, sequence); await _udpClient.SendAsync(packet, packet.Length, _targetEndPoint); - if (enableDebugLogging) - ConvaiLogger.DebugLog($"📤 Sent start packet {sequence}: {audioClip.samples} samples", ConvaiLogger.LogCategory.Character); + ConvaiLogger.Info($"📤 Sent speech START packet #{sequence} (magic: 0x{MAGIC_NUMBER:X}) to {targetIP}:{targetPort}, {audioClip.samples} samples", ConvaiLogger.LogCategory.Character); } private async Task SendAudioClipInChunks(AudioClip audioClip, int sequence) diff --git a/Unity-Master/Assets/Scripts/Multiplayer/NetworkDebugUI.cs b/Unity-Master/Assets/Scripts/Multiplayer/NetworkDebugUI.cs index 42df4da..b607d2f 100644 --- a/Unity-Master/Assets/Scripts/Multiplayer/NetworkDebugUI.cs +++ b/Unity-Master/Assets/Scripts/Multiplayer/NetworkDebugUI.cs @@ -44,6 +44,15 @@ namespace Convai.Scripts.Runtime.Multiplayer private float _lastUpdateTime; private InputAction _toggleAction; + // Packet tracking for enhanced debugging + private System.Collections.Generic.Queue _packetLog = new System.Collections.Generic.Queue(); + private const int MAX_PACKET_LOG = 15; + private int _lastAudioSentCount = 0; + private int _lastAudioReceivedCount = 0; + private int _lastSpeechSentCount = 0; + private int _lastSpeechReceivedCount = 0; + private float _lastRateCheckTime = 0f; + private void Start() { _mainCamera = Camera.main; @@ -126,7 +135,7 @@ namespace Convai.Scripts.Runtime.Multiplayer panelImage.color = new Color(0, 0, 0, 0.85f); RectTransform panelRect = _panel.GetComponent(); - panelRect.sizeDelta = new Vector2(800, 900); + panelRect.sizeDelta = new Vector2(900, 1200); // Increased size for more info // Create text GameObject textObj = new GameObject("DebugText"); @@ -148,7 +157,7 @@ namespace Convai.Scripts.Runtime.Multiplayer // Setup canvas transform for VR RectTransform canvasRect = _canvas.GetComponent(); - canvasRect.sizeDelta = new Vector2(800, 900); + canvasRect.sizeDelta = new Vector2(900, 1200); canvasRect.localScale = Vector3.one * 0.001f; // Scale down for VR viewing } @@ -237,6 +246,10 @@ namespace Convai.Scripts.Runtime.Multiplayer sb.AppendLine($"Time: {DateTime.Now:HH:mm:ss}"); sb.AppendLine(); + // Port Binding Status + AppendPortBindingStatus(sb); + sb.AppendLine(); + // Peer Discovery Status AppendPeerDiscoveryStatus(sb); sb.AppendLine(); @@ -257,12 +270,54 @@ namespace Convai.Scripts.Runtime.Multiplayer AppendSpeechReceiverStatus(sb); sb.AppendLine(); + // Packet Rates + AppendPacketRates(sb); + sb.AppendLine(); + // Event Log AppendEventLog(sb); + sb.AppendLine(); + + // Packet Log + AppendPacketLog(sb); _debugText.text = sb.ToString(); } + private void AppendPortBindingStatus(StringBuilder sb) + { + sb.AppendLine("🔌 PORT BINDING STATUS"); + + // Check if shared listener is active + bool sharedListenerActive = SharedUDPListener.Instance != null && SharedUDPListener.Instance.IsListening; + + if (sharedListenerActive) + { + sb.AppendLine($"✅ Shared UDP Listener ACTIVE"); + sb.AppendLine($" Port: {SharedUDPListener.Instance.ListenPort}"); + sb.AppendLine($" Total Packets: {SharedUDPListener.Instance.TotalPacketsReceived}"); + } + else + { + sb.AppendLine($"❌ Shared UDP Listener NOT FOUND!"); + sb.AppendLine($" Add SharedUDPListener to scene!"); + } + + // Check which components are subscribed + bool discoveryActive = _peerDiscovery != null; + bool audioReceiverActive = _audioReceiver != null; + bool speechReceiverActive = _speechReceiver != null; + + sb.AppendLine($"Discovery: {(discoveryActive ? "✅ SUBSCRIBED" : "❌ NOT FOUND")} (0x44495343)"); + sb.AppendLine($"Audio RX: {(audioReceiverActive ? "✅ SUBSCRIBED" : "❌ NOT FOUND")} (0xC0A1)"); + sb.AppendLine($"Speech RX: {(speechReceiverActive ? "✅ SUBSCRIBED" : "❌ NOT FOUND")} (0xC0A3)"); + + if (!sharedListenerActive) + { + sb.AppendLine("⚠️ CRITICAL: No shared listener!"); + } + } + private void AppendPeerDiscoveryStatus(StringBuilder sb) { sb.AppendLine("🔍 PEER DISCOVERY"); @@ -506,13 +561,99 @@ namespace Convai.Scripts.Runtime.Multiplayer } } + private void AppendPacketRates(StringBuilder sb) + { + sb.AppendLine("📊 PACKET RATES (last second)"); + + float currentTime = Time.time; + float deltaTime = currentTime - _lastRateCheckTime; + + if (deltaTime >= 1f) + { + // Calculate rates + int audioSentDelta = 0; + int audioReceivedDelta = 0; + int speechSentDelta = 0; + int speechReceivedDelta = 0; + + if (_audioSender != null) + { + audioSentDelta = _audioSender.TotalPacketsSent - _lastAudioSentCount; + _lastAudioSentCount = _audioSender.TotalPacketsSent; + } + + if (_audioReceiver != null) + { + audioReceivedDelta = _audioReceiver.TotalPacketsReceived - _lastAudioReceivedCount; + _lastAudioReceivedCount = _audioReceiver.TotalPacketsReceived; + } + + if (_speechSender != null) + { + speechSentDelta = _speechSender.TotalClipsSent - _lastSpeechSentCount; + _lastSpeechSentCount = _speechSender.TotalClipsSent; + } + + if (_speechReceiver != null) + { + speechReceivedDelta = _speechReceiver.TotalClipsReceived - _lastSpeechReceivedCount; + _lastSpeechReceivedCount = _speechReceiver.TotalClipsReceived; + } + + _lastRateCheckTime = currentTime; + + // Log significant activity + if (audioSentDelta > 0) + LogPacketActivity($"Sent {audioSentDelta} voice packets (0xC0A1)"); + if (audioReceivedDelta > 0) + LogPacketActivity($"Received {audioReceivedDelta} voice packets (0xC0A1)"); + if (speechSentDelta > 0) + LogPacketActivity($"Sent {speechSentDelta} speech clips (0xC0A3)"); + if (speechReceivedDelta > 0) + LogPacketActivity($"Received {speechReceivedDelta} speech clips (0xC0A3)"); + } + + sb.AppendLine($"Voice Sent: {(_audioSender != null ? _audioSender.TotalPacketsSent : 0)} total"); + sb.AppendLine($"Voice Received: {(_audioReceiver != null ? _audioReceiver.TotalPacketsReceived : 0)} total"); + sb.AppendLine($"Speech Sent: {(_speechSender != null ? _speechSender.TotalClipsSent : 0)} clips"); + sb.AppendLine($"Speech Received: {(_speechReceiver != null ? _speechReceiver.TotalClipsReceived : 0)} clips"); + } + + private void LogPacketActivity(string message) + { + string timestamp = DateTime.Now.ToString("HH:mm:ss"); + _packetLog.Enqueue($"[{timestamp}] {message}"); + + while (_packetLog.Count > MAX_PACKET_LOG) + { + _packetLog.Dequeue(); + } + } + + private void AppendPacketLog(StringBuilder sb) + { + sb.AppendLine("📦 PACKET ACTIVITY LOG"); + + if (_packetLog.Count > 0) + { + foreach (var entry in _packetLog) + { + sb.AppendLine(entry); + } + } + else + { + sb.AppendLine("No packet activity yet"); + } + } + private void AppendEventLog(StringBuilder sb) { - sb.AppendLine("📋 EVENT LOG"); + sb.AppendLine("📋 CONNECTION EVENTS"); if (_peerDiscovery != null && _peerDiscovery.EventLog.Count > 0) { - int startIndex = Math.Max(0, _peerDiscovery.EventLog.Count - 8); + int startIndex = Math.Max(0, _peerDiscovery.EventLog.Count - 6); for (int i = startIndex; i < _peerDiscovery.EventLog.Count; i++) { sb.AppendLine(_peerDiscovery.EventLog[i]); diff --git a/Unity-Master/Assets/Scripts/Multiplayer/SharedUDPListener.cs b/Unity-Master/Assets/Scripts/Multiplayer/SharedUDPListener.cs new file mode 100644 index 0000000..492d1ab --- /dev/null +++ b/Unity-Master/Assets/Scripts/Multiplayer/SharedUDPListener.cs @@ -0,0 +1,192 @@ +using System; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using Convai.Scripts.Runtime.LoggerSystem; +using UnityEngine; + +namespace Convai.Scripts.Runtime.Multiplayer +{ + /// + /// Shared UDP Listener - Single UDP socket that dispatches packets to appropriate handlers + /// Solves the port binding issue where multiple components tried to bind to the same port + /// + public class SharedUDPListener : MonoBehaviour + { + // Singleton + private static SharedUDPListener _instance; + public static SharedUDPListener Instance => _instance; + + // Network components + private UdpClient _udpClient; + private bool _isListening = false; + private int _listenPort; + private CancellationTokenSource _cancellationTokenSource; + + // Packet handlers - registered by other components + public event Action OnPacketReceived; + + // Metrics + private int _totalPacketsReceived = 0; + private DateTime _lastPacketTime; + public int TotalPacketsReceived => _totalPacketsReceived; + public bool IsListening => _isListening; + public int ListenPort => _listenPort; + + private void Awake() + { + // Singleton pattern + if (_instance != null && _instance != this) + { + Destroy(gameObject); + return; + } + _instance = this; + } + + private void Start() + { + // Get network config + var cfg = NetworkConfig.Instance; + if (cfg != null) + { + _listenPort = cfg.port; + } + else + { + ConvaiLogger.Error("NetworkConfig not found! Using default port 1221", ConvaiLogger.LogCategory.Character); + _listenPort = 1221; + } + + _cancellationTokenSource = new CancellationTokenSource(); + StartListening(); + } + + private void OnDestroy() + { + StopListening(); + _cancellationTokenSource?.Cancel(); + _cancellationTokenSource?.Dispose(); + + if (_instance == this) + _instance = null; + } + + private void StartListening() + { + if (_isListening) return; + + try + { + // Create single UDP client for ALL network traffic + _udpClient = new UdpClient(); + _udpClient.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); + _udpClient.Client.Bind(new IPEndPoint(IPAddress.Any, _listenPort)); + _udpClient.EnableBroadcast = true; + + _isListening = true; + + ConvaiLogger.Info($"✅ Shared UDP Listener BOUND to port {_listenPort}", ConvaiLogger.LogCategory.Character); + ConvaiLogger.Info($" Will dispatch packets to handlers based on magic numbers:", ConvaiLogger.LogCategory.Character); + ConvaiLogger.Info($" - Discovery: 0x44495343", ConvaiLogger.LogCategory.Character); + ConvaiLogger.Info($" - Voice Audio: 0xC0A1", ConvaiLogger.LogCategory.Character); + ConvaiLogger.Info($" - NPC Speech: 0xC0A3", ConvaiLogger.LogCategory.Character); + ConvaiLogger.Info($" - Avatar Sync: 0xC0A0", ConvaiLogger.LogCategory.Character); + + // Start listening for all packets + _ = ListenForPackets(_cancellationTokenSource.Token); + } + catch (Exception ex) + { + ConvaiLogger.Error($"❌ FAILED to bind Shared UDP Listener to port {_listenPort}: {ex.Message}", ConvaiLogger.LogCategory.Character); + ConvaiLogger.Error($"Stack trace: {ex.StackTrace}", ConvaiLogger.LogCategory.Character); + } + } + + private void StopListening() + { + if (!_isListening) return; + + _isListening = false; + _udpClient?.Close(); + _udpClient?.Dispose(); + _udpClient = null; + + ConvaiLogger.Info("Shared UDP Listener stopped", ConvaiLogger.LogCategory.Character); + } + + private async Task ListenForPackets(CancellationToken cancellationToken) + { + try + { + while (_isListening && !cancellationToken.IsCancellationRequested) + { + var result = await _udpClient.ReceiveAsync(); + _totalPacketsReceived++; + _lastPacketTime = DateTime.UtcNow; + + // Dispatch to all registered handlers + OnPacketReceived?.Invoke(result.Buffer, result.RemoteEndPoint); + } + } + catch (ObjectDisposedException) + { + // Normal when stopping + } + catch (Exception ex) + { + if (_isListening) + { + ConvaiLogger.Error($"Error in shared UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character); + } + } + } + + /// + /// Send a packet through the shared UDP client + /// + public async Task SendAsync(byte[] data, IPEndPoint endPoint) + { + if (_udpClient == null || !_isListening) + { + ConvaiLogger.Warn("Cannot send packet - UDP client not initialized", ConvaiLogger.LogCategory.Character); + return 0; + } + + try + { + return await _udpClient.SendAsync(data, data.Length, endPoint); + } + catch (Exception ex) + { + ConvaiLogger.Error($"Failed to send UDP packet: {ex.Message}", ConvaiLogger.LogCategory.Character); + return 0; + } + } + + /// + /// Send a broadcast packet + /// + public async Task SendBroadcastAsync(byte[] data, int port) + { + if (_udpClient == null || !_isListening) + { + ConvaiLogger.Warn("Cannot send broadcast - UDP client not initialized", ConvaiLogger.LogCategory.Character); + return 0; + } + + try + { + var broadcastEndpoint = new IPEndPoint(IPAddress.Broadcast, port); + return await _udpClient.SendAsync(data, data.Length, broadcastEndpoint); + } + catch (Exception ex) + { + ConvaiLogger.Error($"Failed to send broadcast packet: {ex.Message}", ConvaiLogger.LogCategory.Character); + return 0; + } + } + } +} + diff --git a/Unity-Master/Assets/Scripts/Multiplayer/UDPPeerDiscovery.cs b/Unity-Master/Assets/Scripts/Multiplayer/UDPPeerDiscovery.cs index 6fe1850..2322606 100644 --- a/Unity-Master/Assets/Scripts/Multiplayer/UDPPeerDiscovery.cs +++ b/Unity-Master/Assets/Scripts/Multiplayer/UDPPeerDiscovery.cs @@ -1,6 +1,5 @@ using System; using System.Net; -using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; using Convai.Scripts.Runtime.LoggerSystem; @@ -29,7 +28,6 @@ namespace Convai.Scripts.Runtime.Multiplayer public event Action OnConnectionStateChanged; // Network components - private UdpClient _udpClient; private bool _isRunning = false; private int _listenPort; private CancellationTokenSource _cancellationTokenSource; @@ -134,19 +132,20 @@ namespace Convai.Scripts.Runtime.Multiplayer try { - // Create UDP client with port reuse for shared port - _udpClient = new UdpClient(); - _udpClient.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); - _udpClient.Client.Bind(new IPEndPoint(IPAddress.Any, _listenPort)); - _udpClient.EnableBroadcast = true; + // Wait for shared listener to be ready + if (SharedUDPListener.Instance == null) + { + ConvaiLogger.Error("SharedUDPListener not found! Make sure it's in the scene.", ConvaiLogger.LogCategory.Character); + return; + } + + // Subscribe to shared listener + SharedUDPListener.Instance.OnPacketReceived += HandlePacketReceived; _isRunning = true; SetConnectionState(ConnectionState.Discovering); - ConvaiLogger.Info($"🔍 Peer Discovery started - Player {localPlayerID} on port {_listenPort}", ConvaiLogger.LogCategory.Character); - - // Start listening for discovery packets - _ = ListenForDiscoveryPackets(_cancellationTokenSource.Token); + ConvaiLogger.Info($"✅ Peer Discovery subscribed to shared listener - Player {localPlayerID}, listening for magic 0x{DISCOVERY_MAGIC:X}", ConvaiLogger.LogCategory.Character); // Start broadcasting discovery requests _ = BroadcastDiscoveryRequests(_cancellationTokenSource.Token); @@ -165,36 +164,27 @@ namespace Convai.Scripts.Runtime.Multiplayer if (!_isRunning) return; _isRunning = false; - _udpClient?.Close(); - _udpClient?.Dispose(); - _udpClient = null; + + // Unsubscribe from shared listener + if (SharedUDPListener.Instance != null) + { + SharedUDPListener.Instance.OnPacketReceived -= HandlePacketReceived; + } SetConnectionState(ConnectionState.Disconnected); ConvaiLogger.Info("Peer Discovery stopped", ConvaiLogger.LogCategory.Character); } - private async Task ListenForDiscoveryPackets(CancellationToken cancellationToken) + private void HandlePacketReceived(byte[] data, IPEndPoint senderEndPoint) { - while (_isRunning && !cancellationToken.IsCancellationRequested) - { - try - { - var result = await _udpClient.ReceiveAsync(); - await ProcessDiscoveryPacket(result.Buffer, result.RemoteEndPoint); - } - catch (ObjectDisposedException) - { - // Normal when stopping - break; - } - catch (Exception ex) - { - if (_isRunning) - { - ConvaiLogger.Error($"Error receiving discovery packet: {ex.Message}", ConvaiLogger.LogCategory.Character); - } - } - } + // Check if this is a discovery packet (by magic number) + if (data.Length < 14) return; + + uint magic = BitConverter.ToUInt32(data, 0); + if (magic != DISCOVERY_MAGIC) return; + + // Process discovery packet + _ = ProcessDiscoveryPacket(data, senderEndPoint); } private async Task BroadcastDiscoveryRequests(CancellationToken cancellationToken) @@ -256,9 +246,8 @@ namespace Convai.Scripts.Runtime.Multiplayer { byte[] packet = CreateDiscoveryPacket(PACKET_TYPE_REQUEST); - // Broadcast to subnet (will be blocked by AP isolation but we try anyway) - var broadcastEndPoint = new IPEndPoint(IPAddress.Broadcast, _listenPort); - await _udpClient.SendAsync(packet, packet.Length, broadcastEndPoint); + // Broadcast to subnet using shared listener + await SharedUDPListener.Instance.SendBroadcastAsync(packet, _listenPort); if (enableDebugLogging && UnityEngine.Random.value < 0.1f) // Log 10% of broadcasts to reduce spam { @@ -276,7 +265,7 @@ namespace Convai.Scripts.Runtime.Multiplayer try { byte[] packet = CreateDiscoveryPacket(PACKET_TYPE_RESPONSE); - await _udpClient.SendAsync(packet, packet.Length, targetEndPoint); + await SharedUDPListener.Instance.SendAsync(packet, targetEndPoint); if (enableDebugLogging) { @@ -297,7 +286,7 @@ namespace Convai.Scripts.Runtime.Multiplayer byte[] packet = CreateDiscoveryPacket(PACKET_TYPE_HEARTBEAT); var peerEndPoint = new IPEndPoint(IPAddress.Parse(_peerIP), _listenPort); - await _udpClient.SendAsync(packet, packet.Length, peerEndPoint); + await SharedUDPListener.Instance.SendAsync(packet, peerEndPoint); } catch (Exception ex) { diff --git a/Unity-Master/Assets/Scripts/UDPAvatarReceiver.cs b/Unity-Master/Assets/Scripts/UDPAvatarReceiver.cs index 2ed9623..e31d9f3 100644 --- a/Unity-Master/Assets/Scripts/UDPAvatarReceiver.cs +++ b/Unity-Master/Assets/Scripts/UDPAvatarReceiver.cs @@ -2,9 +2,6 @@ using System; using System.Collections.Generic; using System.IO; using System.Net; -using System.Net.Sockets; -using System.Text; -using System.Threading; using UnityEngine; public class UDPAvatarReceiver : MonoBehaviour @@ -38,8 +35,6 @@ public class UDPAvatarReceiver : MonoBehaviour [SerializeField] private bool showDebugInfo = false; [SerializeField] private bool logReceivedPackets = false; - private UdpClient udpClient; - private Thread udpListenerThread; private bool threadRunning = false; private int listenPort; private Dictionary boneCache; @@ -205,122 +200,63 @@ public class UDPAvatarReceiver : MonoBehaviour { try { - if (allowPortSharing) + // Wait for shared listener to be ready + if (Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance == null) { - // Create UDP client with port reuse for local testing - udpClient = new UdpClient(); - udpClient.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); - udpClient.Client.Bind(new IPEndPoint(IPAddress.Any, listenPort)); - } - else - { - // Standard UDP client binding - udpClient = new UdpClient(listenPort); + Debug.LogError("SharedUDPListener not found! Make sure it's in the scene."); + enableReceiver = false; + return; } + // Subscribe to shared listener + Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance.OnPacketReceived += HandlePacketReceived; threadRunning = true; - udpListenerThread = new Thread(new ThreadStart(UDPListenerThread)); - udpListenerThread.IsBackground = true; - udpListenerThread.Start(); if (showDebugInfo) - Debug.Log($"UDP Avatar Receiver started on port {listenPort} (filtering binary avatar data only, Port sharing: {allowPortSharing})"); + Debug.Log($"UDP Avatar Receiver subscribed to shared listener on port {listenPort} (filtering avatar magic 0x{AVATAR_MAGIC:X})"); } catch (Exception e) { - if (allowPortSharing) + Debug.LogError($"Failed to subscribe to shared UDP listener: {e.Message}"); + enableReceiver = false; + } + } + + void HandlePacketReceived(byte[] data, IPEndPoint senderEndPoint) + { + // Check if this is avatar data (by magic number) + if (!IsAvatarData(data)) return; + + // Process avatar packet + CompactAvatarData avatarData = DeserializeCompactData(data); + + // Check if this is from the target player (0 means accept from any player) + if (targetPlayerID == 0 || avatarData.playerID == targetPlayerID) + { + // Check for packet loss + if (avatarData.sequenceNumber > lastSequenceNumber + 1) { - Debug.LogWarning($"Failed to start UDP listener with port sharing: {e.Message}"); - Debug.LogWarning("Trying with different port..."); - TryAlternativePort(); + packetsDropped += (int)(avatarData.sequenceNumber - lastSequenceNumber - 1); } - else + + lastSequenceNumber = avatarData.sequenceNumber; + packetsReceived++; + + // Store the new data (thread-safe) + lock (dataLock) { - Debug.LogError($"Failed to start UDP listener: {e.Message}"); - enableReceiver = false; + lastReceivedData = avatarData; + hasNewData = true; + } + + if (logReceivedPackets && packetsReceived % 30 == 0) + { + string modeStr = avatarData.isFullDataMode ? "FULL" : "OPT"; + Debug.Log($"Received {modeStr} packet #{avatarData.sequenceNumber} from player {avatarData.playerID}, size: {data.Length} bytes, bones: {avatarData.bones?.Length ?? 0}, blend shapes: {avatarData.blendShapes?.Length ?? 0}"); } } } - void TryAlternativePort() - { - // Try a few alternative ports for local testing - int[] alternativePorts = { 8081, 8082, 8083, 8084, 8085 }; - - foreach (int port in alternativePorts) - { - try - { - udpClient = new UdpClient(port); - threadRunning = true; - udpListenerThread = new Thread(new ThreadStart(UDPListenerThread)); - udpListenerThread.IsBackground = true; - udpListenerThread.Start(); - - Debug.Log($"UDP listener started on alternative port {port}"); - return; - } - catch (Exception) - { - // Try next port - continue; - } - } - - Debug.LogError("Failed to start UDP listener on any available port"); - enableReceiver = false; - } - - void UDPListenerThread() - { - IPEndPoint remoteEndPoint = new IPEndPoint(IPAddress.Any, 0); - - while (threadRunning) - { - try - { - byte[] data = udpClient.Receive(ref remoteEndPoint); - - if (data.Length > 0 && IsAvatarData(data)) - { - CompactAvatarData avatarData = DeserializeCompactData(data); - - // Check if this is from the target player (0 means accept from any player) - if (targetPlayerID == 0 || avatarData.playerID == targetPlayerID) - { - // Check for packet loss - if (avatarData.sequenceNumber > lastSequenceNumber + 1) - { - packetsDropped += (int)(avatarData.sequenceNumber - lastSequenceNumber - 1); - } - - lastSequenceNumber = avatarData.sequenceNumber; - packetsReceived++; - - // Store the new data (thread-safe) - lock (dataLock) - { - lastReceivedData = avatarData; - hasNewData = true; - } - - if (logReceivedPackets && packetsReceived % 30 == 0) - { - string modeStr = avatarData.isFullDataMode ? "FULL" : "OPT"; - Debug.Log($"Received {modeStr} packet #{avatarData.sequenceNumber} from player {avatarData.playerID}, size: {data.Length} bytes, bones: {avatarData.bones?.Length ?? 0}, blend shapes: {avatarData.blendShapes?.Length ?? 0}"); - } - } - } - } - catch (Exception e) - { - if (threadRunning) // Only log errors if we're supposed to be running - { - Debug.LogError($"UDP receive error: {e.Message}"); - } - } - } - } bool IsAvatarData(byte[] data) { @@ -618,17 +554,10 @@ public class UDPAvatarReceiver : MonoBehaviour { threadRunning = false; - if (udpClient != null) + // Unsubscribe from shared listener + if (Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance != null) { - udpClient.Close(); - udpClient.Dispose(); - udpClient = null; - } - - if (udpListenerThread != null) - { - udpListenerThread.Join(1000); // Wait up to 1 second for thread to finish - udpListenerThread = null; + Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance.OnPacketReceived -= HandlePacketReceived; } } @@ -688,9 +617,9 @@ public class UDPAvatarReceiver : MonoBehaviour int GetActualListenPort() { - if (udpClient?.Client?.LocalEndPoint != null) + if (Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance != null) { - return ((IPEndPoint)udpClient.Client.LocalEndPoint).Port; + return Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance.ListenPort; } return listenPort; } diff --git a/Unity-Master/Assets/Scripts/UDPAvatarReceiverAgent.cs b/Unity-Master/Assets/Scripts/UDPAvatarReceiverAgent.cs index 0a32ade..6e80df2 100644 --- a/Unity-Master/Assets/Scripts/UDPAvatarReceiverAgent.cs +++ b/Unity-Master/Assets/Scripts/UDPAvatarReceiverAgent.cs @@ -2,9 +2,6 @@ using System; using System.Collections.Generic; using System.IO; using System.Net; -using System.Net.Sockets; -using System.Text; -using System.Threading; using UnityEngine; /// @@ -41,8 +38,6 @@ public class UDPAvatarReceiverAgent : MonoBehaviour [SerializeField] private bool showDebugInfo = false; [SerializeField] private bool logReceivedPackets = false; - private UdpClient udpClient; - private Thread udpListenerThread; private bool threadRunning = false; private int listenPort; private Dictionary boneCache; @@ -208,122 +203,63 @@ public class UDPAvatarReceiverAgent : MonoBehaviour { try { - if (allowPortSharing) + // Wait for shared listener to be ready + if (Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance == null) { - // Create UDP client with port reuse for local testing - udpClient = new UdpClient(); - udpClient.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); - udpClient.Client.Bind(new IPEndPoint(IPAddress.Any, listenPort)); - } - else - { - // Standard UDP client binding - udpClient = new UdpClient(listenPort); + Debug.LogError("SharedUDPListener not found! Make sure it's in the scene."); + enableReceiver = false; + return; } + // Subscribe to shared listener + Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance.OnPacketReceived += HandlePacketReceived; threadRunning = true; - udpListenerThread = new Thread(new ThreadStart(UDPListenerThread)); - udpListenerThread.IsBackground = true; - udpListenerThread.Start(); if (showDebugInfo) - Debug.Log($"UDP Avatar Receiver Agent started on port {listenPort} (filtering binary avatar data only, Port sharing: {allowPortSharing})"); + Debug.Log($"UDP Avatar Receiver Agent subscribed to shared listener on port {listenPort} (filtering avatar magic 0x{AVATAR_MAGIC:X})"); } catch (Exception e) { - if (allowPortSharing) + Debug.LogError($"Failed to subscribe to shared UDP listener: {e.Message}"); + enableReceiver = false; + } + } + + void HandlePacketReceived(byte[] data, IPEndPoint senderEndPoint) + { + // Check if this is avatar data (by magic number) + if (!IsAvatarData(data)) return; + + // Process avatar packet + CompactAvatarData avatarData = DeserializeCompactData(data); + + // Check if this is from the target player (0 means accept from any player) + if (targetPlayerID == 0 || avatarData.playerID == targetPlayerID) + { + // Check for packet loss + if (avatarData.sequenceNumber > lastSequenceNumber + 1) { - Debug.LogWarning($"Failed to start UDP listener with port sharing: {e.Message}"); - Debug.LogWarning("Trying with different port..."); - TryAlternativePort(); + packetsDropped += (int)(avatarData.sequenceNumber - lastSequenceNumber - 1); } - else + + lastSequenceNumber = avatarData.sequenceNumber; + packetsReceived++; + + // Store the new data (thread-safe) + lock (dataLock) { - Debug.LogError($"Failed to start UDP listener: {e.Message}"); - enableReceiver = false; + lastReceivedData = avatarData; + hasNewData = true; + } + + if (logReceivedPackets && packetsReceived % 30 == 0) + { + string modeStr = avatarData.isFullDataMode ? "FULL" : "OPT"; + Debug.Log($"Received {modeStr} packet #{avatarData.sequenceNumber} from player {avatarData.playerID}, size: {data.Length} bytes, bones: {avatarData.bones?.Length ?? 0}, blend shapes: {avatarData.blendShapes?.Length ?? 0}"); } } } - void TryAlternativePort() - { - // Try a few alternative ports for local testing - int[] alternativePorts = { 8081, 8082, 8083, 8084, 8085 }; - - foreach (int port in alternativePorts) - { - try - { - udpClient = new UdpClient(port); - threadRunning = true; - udpListenerThread = new Thread(new ThreadStart(UDPListenerThread)); - udpListenerThread.IsBackground = true; - udpListenerThread.Start(); - - Debug.Log($"UDP listener started on alternative port {port}"); - return; - } - catch (Exception) - { - // Try next port - continue; - } - } - - Debug.LogError("Failed to start UDP listener on any available port"); - enableReceiver = false; - } - - void UDPListenerThread() - { - IPEndPoint remoteEndPoint = new IPEndPoint(IPAddress.Any, 0); - - while (threadRunning) - { - try - { - byte[] data = udpClient.Receive(ref remoteEndPoint); - - if (data.Length > 0 && IsAvatarData(data)) - { - CompactAvatarData avatarData = DeserializeCompactData(data); - - // Check if this is from the target player (0 means accept from any player) - if (targetPlayerID == 0 || avatarData.playerID == targetPlayerID) - { - // Check for packet loss - if (avatarData.sequenceNumber > lastSequenceNumber + 1) - { - packetsDropped += (int)(avatarData.sequenceNumber - lastSequenceNumber - 1); - } - - lastSequenceNumber = avatarData.sequenceNumber; - packetsReceived++; - - // Store the new data (thread-safe) - lock (dataLock) - { - lastReceivedData = avatarData; - hasNewData = true; - } - - if (logReceivedPackets && packetsReceived % 30 == 0) - { - string modeStr = avatarData.isFullDataMode ? "FULL" : "OPT"; - Debug.Log($"Received {modeStr} packet #{avatarData.sequenceNumber} from player {avatarData.playerID}, size: {data.Length} bytes, bones: {avatarData.bones?.Length ?? 0}, blend shapes: {avatarData.blendShapes?.Length ?? 0}"); - } - } - } - } - catch (Exception e) - { - if (threadRunning) // Only log errors if we're supposed to be running - { - Debug.LogError($"UDP receive error: {e.Message}"); - } - } - } - } bool IsAvatarData(byte[] data) { @@ -621,17 +557,10 @@ public class UDPAvatarReceiverAgent : MonoBehaviour { threadRunning = false; - if (udpClient != null) + // Unsubscribe from shared listener + if (Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance != null) { - udpClient.Close(); - udpClient.Dispose(); - udpClient = null; - } - - if (udpListenerThread != null) - { - udpListenerThread.Join(1000); // Wait up to 1 second for thread to finish - udpListenerThread = null; + Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance.OnPacketReceived -= HandlePacketReceived; } } @@ -691,9 +620,9 @@ public class UDPAvatarReceiverAgent : MonoBehaviour int GetActualListenPort() { - if (udpClient?.Client?.LocalEndPoint != null) + if (Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance != null) { - return ((IPEndPoint)udpClient.Client.LocalEndPoint).Port; + return Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance.ListenPort; } return listenPort; }