tomh104/Master-Arbeit-Tom-Hempel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

created centralized UDP Listener

Browse Source
This commit is contained in:
tom.hempel
2025-10-25 14:58:22 +02:00
parent 4ddb89d011
commit 3f72973ff5
10 changed files with 585 additions and 534 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
Unity-Master/Assets/Resources/NetworkConfig.asset (Stored with Git LFS)
View File

Binary file not shown.

144
Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioReceiver.cs
View File

@ -1,7 +1,6 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
using Convai.Scripts.Runtime.Core;
@ -37,7 +36,6 @@ namespace Convai.Scripts.Runtime.Multiplayer
public int ListenPort => listenPort;
// Network components
private UdpClient _udpListener;
private IPEndPoint _remoteEndPoint;
private bool _isListening = false;
private int listenPort;
@ -47,24 +45,19 @@ namespace Convai.Scripts.Runtime.Multiplayer
private bool _isReceivingAudio = false;
private int _expectedSequence = 0;
private const uint MAGIC_NUMBER = 0xC0A1; // Simple magic number for packet validation
private const uint ACK_MAGIC = 0xC0A2; // ACK magic to confirm START control
// Timing for auto-stop
private float _lastPacketTime;
private const float AUTO_STOP_DELAY = 1.0f; // Stop listening after 1 second of no packets
// Packet structure (matching ConvaiSimpleUDPAudioSender)
// SIMPLIFIED Packet structure (matching ConvaiSimpleUDPAudioSender)
private struct AudioPacketData
{
public uint magicNumber;
public int sequence;
public int sampleCount;
public int microphonePosition;
public bool isEndSignal;
public bool isStartSignal;
public short[] audioSamples;
public long timestamp;
}
[Header("Recording Storage")]
@ -234,20 +227,22 @@ namespace Convai.Scripts.Runtime.Multiplayer
try
{
// Create UDP client with port reuse to allow sharing with UDPPeerDiscovery
_udpListener = new UdpClient();
_udpListener.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_udpListener.Client.Bind(new IPEndPoint(IPAddress.Any, listenPort));
// Wait for shared listener to be ready
if (SharedUDPListener.Instance == null)
{
ConvaiLogger.Error("SharedUDPListener not found! Make sure it's in the scene.", ConvaiLogger.LogCategory.Character);
return;
}
// Subscribe to shared listener
SharedUDPListener.Instance.OnPacketReceived += HandlePacketReceived;
_isListening = true;
ConvaiLogger.Info($"Simple UDP Audio Receiver V2 listening on port {listenPort} (shared)", ConvaiLogger.LogCategory.Character);
// Start listening for incoming packets
_ = ListenForAudioPackets(_cancellationTokenSource.Token);
ConvaiLogger.Info($"✅ Audio Receiver subscribed to shared listener, listening for magic 0x{MAGIC_NUMBER:X}", ConvaiLogger.LogCategory.Character);
}
catch (Exception ex)
{
ConvaiLogger.Error($"Failed to start UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Error($"❌ FAILED to subscribe Audio Receiver: {ex.Message}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Error($"Stack trace: {ex.StackTrace}", ConvaiLogger.LogCategory.Character);
}
}
@ -258,9 +253,12 @@ namespace Convai.Scripts.Runtime.Multiplayer
return;
_isListening = false;
_udpListener?.Close();
_udpListener?.Dispose();
_udpListener = null;
// Unsubscribe from shared listener
if (SharedUDPListener.Instance != null)
{
SharedUDPListener.Instance.OnPacketReceived -= HandlePacketReceived;
}
// Stop any ongoing simulation
StopTalkingSimulation();
@ -268,26 +266,19 @@ namespace Convai.Scripts.Runtime.Multiplayer
ConvaiLogger.Info("Stopped UDP Audio Receiver V2", ConvaiLogger.LogCategory.Character);
}
private async Task ListenForAudioPackets(CancellationToken cancellationToken)
private void HandlePacketReceived(byte[] data, IPEndPoint senderEndPoint)
{
try
{
while (_isListening && !cancellationToken.IsCancellationRequested)
{
var result = await _udpListener.ReceiveAsync();
_remoteEndPoint = result.RemoteEndPoint;
// Check if this is an audio packet (by magic number)
if (data.Length < 12) return;
await ProcessReceivedPacket(result.Buffer, result.RemoteEndPoint);
}
}
catch (ObjectDisposedException)
{
// Normal when stopping
}
catch (Exception ex)
{
ConvaiLogger.Error($"Error in UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
uint magic = BitConverter.ToUInt32(data, 0);
if (magic != MAGIC_NUMBER) return;
// Update remote endpoint
_remoteEndPoint = senderEndPoint;
// Process audio packet
_ = ProcessReceivedPacket(data, senderEndPoint);
}
private Task ProcessReceivedPacket(byte[] data, IPEndPoint sender)
@ -305,28 +296,17 @@ namespace Convai.Scripts.Runtime.Multiplayer
_totalPacketsReceived++;
_lastPacketReceivedTime = DateTime.UtcNow;
if (enableDebugLogging)
// SIMPLIFIED: Check for end signal (sampleCount == -1)
bool isEndSignal = (packet.sampleCount == -1);
if (enableDebugLogging && packet.sequence % 50 == 0)
{
if (packet.isEndSignal)
ConvaiLogger.DebugLog($"Received end signal from {sender}", ConvaiLogger.LogCategory.Character);
else
ConvaiLogger.DebugLog($"Received audio packet {packet.sequence} with {packet.sampleCount} samples", ConvaiLogger.LogCategory.Character);
ConvaiLogger.DebugLog($"📥 Received audio packet #{packet.sequence} (magic: 0x{packet.magicNumber:X}) from {sender}, {packet.sampleCount} samples", ConvaiLogger.LogCategory.Character);
}
// Handle START control: acknowledge and begin simulation
if (packet.isStartSignal)
{
SendStartAck(sender);
if (!_isReceivingAudio)
{
StartTalkingSimulation();
}
OnAudioReceiving?.Invoke(true);
return Task.CompletedTask;
}
if (packet.isEndSignal)
if (isEndSignal)
{
ConvaiLogger.Info($"📥 Received END signal from {sender}", ConvaiLogger.LogCategory.Character);
StopTalkingSimulation();
OnAudioReceiving?.Invoke(false);
}
@ -335,6 +315,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
// If this is the first packet, start the talking simulation
if (packet.sequence == 0 && !_isReceivingAudio)
{
ConvaiLogger.Info($"📥 Received FIRST audio packet from {sender}, starting simulation", ConvaiLogger.LogCategory.Character);
StartTalkingSimulation();
}
@ -418,7 +399,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
private AudioPacketData? ParseSimpleAudioPacket(byte[] data)
{
if (data.Length < 17) // Minimum header size to match sender
if (data.Length < 12) // SIMPLIFIED: Minimum header size (4+4+4)
return null;
try
@ -430,27 +411,22 @@ namespace Convai.Scripts.Runtime.Multiplayer
offset += 4;
if (magic != MAGIC_NUMBER)
{
if (enableDebugLogging)
ConvaiLogger.Warn($"❌ Invalid magic number: expected 0x{MAGIC_NUMBER:X}, got 0x{magic:X}", ConvaiLogger.LogCategory.Character);
return null;
}
// Read header (matching sender's 17-byte format)
// SIMPLIFIED header (12 bytes total)
int sequence = BitConverter.ToInt32(data, offset);
offset += 4;
int sampleCount = BitConverter.ToInt32(data, offset);
offset += 4;
int microphonePosition = BitConverter.ToInt32(data, offset);
offset += 4;
byte flags = data[offset];
offset += 1;
bool isEndSignal = (flags == 1);
bool isStartSignal = (flags == 2);
// Read audio data
// Read audio data (if not end signal)
short[] audioSamples = null;
if (!isEndSignal && !isStartSignal && sampleCount > 0)
if (sampleCount > 0)
{
int audioDataSize = sampleCount * sizeof(short);
if (data.Length >= offset + audioDataSize)
@ -458,6 +434,10 @@ namespace Convai.Scripts.Runtime.Multiplayer
audioSamples = new short[sampleCount];
Buffer.BlockCopy(data, offset, audioSamples, 0, audioDataSize);
}
else
{
ConvaiLogger.Warn($"⚠️ Packet too small: expected {offset + audioDataSize} bytes, got {data.Length}", ConvaiLogger.LogCategory.Character);
}
}
return new AudioPacketData
@ -465,11 +445,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
magicNumber = magic,
sequence = sequence,
sampleCount = sampleCount,
microphonePosition = microphonePosition,
isEndSignal = isEndSignal,
isStartSignal = isStartSignal,
audioSamples = audioSamples,
timestamp = 0 // Not provided in sender format
audioSamples = audioSamples
};
}
catch (Exception ex)
@ -479,26 +455,6 @@ namespace Convai.Scripts.Runtime.Multiplayer
}
}
private void SendStartAck(IPEndPoint sender)
{
try
{
if (_udpListener == null || sender == null)
return;
byte[] ack = new byte[8];
BitConverter.GetBytes(ACK_MAGIC).CopyTo(ack, 0);
BitConverter.GetBytes(-1).CopyTo(ack, 4);
_udpListener.SendAsync(ack, ack.Length, sender);
if (enableDebugLogging)
ConvaiLogger.DebugLog($"Sent START ACK to {sender}", ConvaiLogger.LogCategory.Character);
}
catch (Exception ex)
{
ConvaiLogger.Warn($"Failed to send START ACK: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
private void BufferAudioPacket(int sequence, short[] samples)
{

141
Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs
View File

@ -48,25 +48,21 @@ namespace Convai.Scripts.Runtime.Multiplayer
private AudioClip _audioClip;
private bool _isRecording = false;
private CancellationTokenSource _cancellationTokenSource;
private CancellationTokenSource _ackCancellationTokenSource;
private int _lastMicrophonePosition = 0;
private float[] _audioBuffer;
private string _selectedMicrophone;
private int _packetSequence = 0;
private volatile bool _startAckReceived = false;
private bool _xrAButtonPrevPressed = false;
private InputAction _xrTalkAction;
private InputAction _xrTestAction;
private bool _usingExternalTalkAction = false;
private InputAction _externalTalkAction;
// Protocol constants
// Protocol constants - SIMPLIFIED
private const uint AUDIO_MAGIC = 0xC0A1;
private const uint ACK_MAGIC = 0xC0A2;
private const byte FLAG_AUDIO = 0;
private const byte FLAG_END = 1;
private const byte FLAG_START = 2;
public event Action<bool> OnRecordingStateChanged;
@ -118,10 +114,6 @@ namespace Convai.Scripts.Runtime.Multiplayer
InitializeAudio();
_persistentDataPath = Application.persistentDataPath;
_cancellationTokenSource = new CancellationTokenSource();
_ackCancellationTokenSource = new CancellationTokenSource();
// Start ACK listener
_ = ListenForAcks(_ackCancellationTokenSource.Token);
// Setup Input System action for XR A/primary button
if (useInputSystemXR)
@ -158,8 +150,6 @@ namespace Convai.Scripts.Runtime.Multiplayer
StopRecording();
_cancellationTokenSource?.Cancel();
_cancellationTokenSource?.Dispose();
_ackCancellationTokenSource?.Cancel();
_ackCancellationTokenSource?.Dispose();
if (_usingExternalTalkAction)
TeardownExternalTalkInputAction();
else
@ -476,7 +466,6 @@ namespace Convai.Scripts.Runtime.Multiplayer
_isRecording = true;
_lastMicrophonePosition = 0;
_packetSequence = 0;
_startAckReceived = false;
_localSessionStart = DateTime.UtcNow;
lock (_localAudioLock)
{
@ -486,10 +475,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
ConvaiLogger.Info($"🎤 Started recording for UDP transmission to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
OnRecordingStateChanged?.Invoke(true);
// Send START control and wait briefly for ACK to ensure receiver is ready
_ = SendStartOfRecordingSignalAndAwaitAck();
// Start continuous audio processing
// Start continuous audio processing immediately (SIMPLIFIED - no ACK waiting)
_ = ProcessAudioContinuously(_cancellationTokenSource.Token);
}
catch (Exception ex)
@ -640,15 +626,13 @@ namespace Convai.Scripts.Runtime.Multiplayer
private byte[] CreateSimpleAudioPacket(float[] audioData, int startIndex, int sampleCount)
{
// Simple packet structure:
// SIMPLIFIED packet structure:
// 4 bytes: Magic number (0xC0A1)
// 4 bytes: Packet sequence number
// 4 bytes: Sample count in this packet
// 4 bytes: Start position in stream
// 1 byte: Flags (0 = normal audio, 1 = end of recording)
// N bytes: Audio data (converted to shorts)
int headerSize = 17; // 4 + 4 + 4 + 4 + 1
int headerSize = 12; // 4 + 4 + 4 (SIMPLIFIED from 17)
int audioDataSize = sampleCount * sizeof(short);
byte[] packet = new byte[headerSize + audioDataSize];
@ -666,15 +650,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
BitConverter.GetBytes(sampleCount).CopyTo(packet, offset);
offset += 4;
// Start position
BitConverter.GetBytes(_lastMicrophonePosition + startIndex).CopyTo(packet, offset);
offset += 4;
// Flags (0 for normal audio)
packet[offset] = FLAG_AUDIO;
offset += 1;
// Convert audio samples to bytes (same as Convai approach)
// Convert audio samples to bytes
for (int i = 0; i < sampleCount; i++)
{
float sample = audioData[startIndex + i];
@ -685,6 +661,11 @@ namespace Convai.Scripts.Runtime.Multiplayer
offset += 2;
}
if (enableDebugLogging && _packetSequence % 50 == 0)
{
ConvaiLogger.DebugLog($"📤 Sent audio packet #{_packetSequence} (magic: 0x{AUDIO_MAGIC:X}) to {targetIP}:{targetPort}, {sampleCount} samples", ConvaiLogger.LogCategory.Character);
}
return packet;
}
@ -692,8 +673,8 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
try
{
// Create end packet
byte[] packet = new byte[17]; // Header only, no audio data
// SIMPLIFIED end packet
byte[] packet = new byte[12]; // Header only, no audio data
int offset = 0;
// Magic number
@ -704,18 +685,12 @@ namespace Convai.Scripts.Runtime.Multiplayer
BitConverter.GetBytes(_packetSequence).CopyTo(packet, offset);
offset += 4;
// Sample count (0 for end signal)
BitConverter.GetBytes(0).CopyTo(packet, offset);
offset += 4;
// Start position
BitConverter.GetBytes(_lastMicrophonePosition).CopyTo(packet, offset);
offset += 4;
// Flags (1 for end of recording)
packet[offset] = FLAG_END;
// Sample count (-1 for end signal)
BitConverter.GetBytes(-1).CopyTo(packet, offset);
_udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
ConvaiLogger.Info($"📤 Sent END signal (magic: 0x{AUDIO_MAGIC:X}) to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
}
catch (Exception ex)
{
@ -817,90 +792,6 @@ namespace Convai.Scripts.Runtime.Multiplayer
}
}
private async Task SendStartOfRecordingSignalAndAwaitAck()
{
try
{
const int maxAttempts = 3;
const int ackTimeoutMs = 250;
for (int attempt = 1; attempt <= maxAttempts && !_startAckReceived; attempt++)
{
// Build START control packet (no audio, special flag)
byte[] packet = new byte[17];
int offset = 0;
BitConverter.GetBytes(AUDIO_MAGIC).CopyTo(packet, offset);
offset += 4;
// Use -1 as the special sequence for START control
BitConverter.GetBytes(-1).CopyTo(packet, offset);
offset += 4;
BitConverter.GetBytes(0).CopyTo(packet, offset);
offset += 4;
BitConverter.GetBytes(_lastMicrophonePosition).CopyTo(packet, offset);
offset += 4;
packet[offset] = FLAG_START;
await _udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
// Wait for ACK
int waited = 0;
while (!_startAckReceived && waited < ackTimeoutMs)
{
await Task.Delay(10);
waited += 10;
}
if (_startAckReceived)
{
if (enableDebugLogging)
ConvaiLogger.DebugLog("Received START ACK from receiver", ConvaiLogger.LogCategory.Character);
break;
}
else if (enableDebugLogging)
{
ConvaiLogger.Warn($"No START ACK (attempt {attempt}/{maxAttempts}), retrying...", ConvaiLogger.LogCategory.Character);
}
}
}
catch (Exception ex)
{
ConvaiLogger.Warn($"Error during START ACK process: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
private async Task ListenForAcks(CancellationToken token)
{
// Use the same UDP client; acks will be sent back to our ephemeral local port
while (!token.IsCancellationRequested)
{
try
{
var result = await _udpClient.ReceiveAsync();
var data = result.Buffer;
if (data == null || data.Length < 8)
continue;
uint magic = BitConverter.ToUInt32(data, 0);
if (magic != ACK_MAGIC)
continue;
int seq = BitConverter.ToInt32(data, 4);
if (seq == -1)
{
_startAckReceived = true;
}
}
catch (ObjectDisposedException)
{
break;
}
catch (Exception)
{
// Ignore and keep listening
}
}
}
// Public methods for external control
public void SetTargetEndpoint(string ip, int port)
{

60
Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechReceiver.cs
View File

@ -1,7 +1,6 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
using Convai.Scripts.Runtime.LoggerSystem;
@ -39,7 +38,6 @@ namespace Convai.Scripts.Runtime.Multiplayer
[SerializeField] private bool showTranscripts = true;
// Network components
private UdpClient _udpListener;
private IPEndPoint _remoteEndPoint;
private bool _isListening = false;
private int listenPort;
@ -211,20 +209,22 @@ namespace Convai.Scripts.Runtime.Multiplayer
try
{
// Create UDP client with port reuse to allow sharing with UDPPeerDiscovery
_udpListener = new UdpClient();
_udpListener.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_udpListener.Client.Bind(new IPEndPoint(IPAddress.Any, listenPort));
// Wait for shared listener to be ready
if (SharedUDPListener.Instance == null)
{
ConvaiLogger.Error("SharedUDPListener not found! Make sure it's in the scene.", ConvaiLogger.LogCategory.Character);
return;
}
// Subscribe to shared listener
SharedUDPListener.Instance.OnPacketReceived += HandlePacketReceived;
_isListening = true;
ConvaiLogger.Info($"UDP Speech Receiver listening on port {listenPort} (shared)", ConvaiLogger.LogCategory.Character);
// Start listening for incoming packets
_ = ListenForSpeechPackets(_cancellationTokenSource.Token);
ConvaiLogger.Info($"✅ Speech Receiver subscribed to shared listener, listening for magic 0x{MAGIC_NUMBER:X}", ConvaiLogger.LogCategory.Character);
}
catch (Exception ex)
{
ConvaiLogger.Error($"Failed to start UDP speech receiver: {ex.Message}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Error($"❌ FAILED to subscribe Speech Receiver: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
@ -234,9 +234,12 @@ namespace Convai.Scripts.Runtime.Multiplayer
return;
_isListening = false;
_udpListener?.Close();
_udpListener?.Dispose();
_udpListener = null;
// Unsubscribe from shared listener
if (SharedUDPListener.Instance != null)
{
SharedUDPListener.Instance.OnPacketReceived -= HandlePacketReceived;
}
// Stop any ongoing playback
StopSpeechPlayback();
@ -244,26 +247,19 @@ namespace Convai.Scripts.Runtime.Multiplayer
ConvaiLogger.Info("Stopped UDP Speech Receiver", ConvaiLogger.LogCategory.Character);
}
private async Task ListenForSpeechPackets(CancellationToken cancellationToken)
private void HandlePacketReceived(byte[] data, IPEndPoint senderEndPoint)
{
try
{
while (_isListening && !cancellationToken.IsCancellationRequested)
{
var result = await _udpListener.ReceiveAsync();
_remoteEndPoint = result.RemoteEndPoint;
// Check if this is a speech packet (by magic number)
if (data.Length < 4) return;
await ProcessReceivedPacket(result.Buffer, result.RemoteEndPoint);
}
}
catch (ObjectDisposedException)
{
// Normal when stopping
}
catch (Exception ex)
{
ConvaiLogger.Error($"Error in UDP speech listener: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
uint magic = BitConverter.ToUInt32(data, 0);
if (magic != MAGIC_NUMBER) return;
// Update remote endpoint
_remoteEndPoint = senderEndPoint;
// Process speech packet
_ = ProcessReceivedPacket(data, senderEndPoint);
}
private Task ProcessReceivedPacket(byte[] data, IPEndPoint sender)

32
Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechSender.cs
View File

@ -334,6 +334,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
ConvaiLogger.Info("🔊 Starting continuous audio monitoring as fallback", ConvaiLogger.LogCategory.Character);
AudioClip lastMonitoredClip = null;
float lastNPCCheckLog = 0f;
while (true)
{
@ -343,6 +344,29 @@ namespace Convai.Scripts.Runtime.Multiplayer
// Check if we still have a valid source NPC
if (sourceNPC == null || sourceNPC.AudioManager == null)
{
// Log every 2 seconds if no NPC found (AGGRESSIVE LOGGING)
if (Time.time - lastNPCCheckLog > 2f)
{
if (sourceNPC == null)
ConvaiLogger.Warn("⚠️ [Speech Sender] No source NPC found! Looking for ConvaiNPC in scene...", ConvaiLogger.LogCategory.Character);
else
ConvaiLogger.Warn($"⚠️ [Speech Sender] NPC '{sourceNPC.characterName}' has no AudioManager!", ConvaiLogger.LogCategory.Character);
// Try to find NPC again
if (useActiveNPC)
{
var foundNPC = FindEnabledConvaiNPC();
if (foundNPC != null && foundNPC != sourceNPC)
{
ConvaiLogger.Info($"✅ [Speech Sender] Found NPC: {foundNPC.characterName} on GameObject '{foundNPC.gameObject.name}'", ConvaiLogger.LogCategory.Character);
sourceNPC = foundNPC;
SubscribeToNPCEvents();
}
}
lastNPCCheckLog = Time.time;
}
yield return new WaitForSeconds(1f); // Wait longer if no NPC
continue;
}
@ -351,6 +375,11 @@ namespace Convai.Scripts.Runtime.Multiplayer
AudioSource audioSource = sourceNPC.AudioManager.GetComponent<AudioSource>();
if (audioSource == null)
{
if (Time.time - lastNPCCheckLog > 2f)
{
ConvaiLogger.Warn($"⚠️ [Speech Sender] NPC '{sourceNPC.characterName}' AudioManager has no AudioSource component!", ConvaiLogger.LogCategory.Character);
lastNPCCheckLog = Time.time;
}
yield return new WaitForSeconds(1f);
continue;
}
@ -455,8 +484,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
byte[] packet = CreateAudioStartPacket(audioClip, transcript, sequence);
await _udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
if (enableDebugLogging)
ConvaiLogger.DebugLog($"📤 Sent start packet {sequence}: {audioClip.samples} samples", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"📤 Sent speech START packet #{sequence} (magic: 0x{MAGIC_NUMBER:X}) to {targetIP}:{targetPort}, {audioClip.samples} samples", ConvaiLogger.LogCategory.Character);
}
private async Task SendAudioClipInChunks(AudioClip audioClip, int sequence)

149
Unity-Master/Assets/Scripts/Multiplayer/NetworkDebugUI.cs
View File

@ -44,6 +44,15 @@ namespace Convai.Scripts.Runtime.Multiplayer
private float _lastUpdateTime;
private InputAction _toggleAction;
// Packet tracking for enhanced debugging
private System.Collections.Generic.Queue<string> _packetLog = new System.Collections.Generic.Queue<string>();
private const int MAX_PACKET_LOG = 15;
private int _lastAudioSentCount = 0;
private int _lastAudioReceivedCount = 0;
private int _lastSpeechSentCount = 0;
private int _lastSpeechReceivedCount = 0;
private float _lastRateCheckTime = 0f;
private void Start()
{
_mainCamera = Camera.main;
@ -126,7 +135,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
panelImage.color = new Color(0, 0, 0, 0.85f);
RectTransform panelRect = _panel.GetComponent<RectTransform>();
panelRect.sizeDelta = new Vector2(800, 900);
panelRect.sizeDelta = new Vector2(900, 1200); // Increased size for more info
// Create text
GameObject textObj = new GameObject("DebugText");
@ -148,7 +157,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
// Setup canvas transform for VR
RectTransform canvasRect = _canvas.GetComponent<RectTransform>();
canvasRect.sizeDelta = new Vector2(800, 900);
canvasRect.sizeDelta = new Vector2(900, 1200);
canvasRect.localScale = Vector3.one * 0.001f; // Scale down for VR viewing
}
@ -237,6 +246,10 @@ namespace Convai.Scripts.Runtime.Multiplayer
sb.AppendLine($"Time: {DateTime.Now:HH:mm:ss}");
sb.AppendLine();
// Port Binding Status
AppendPortBindingStatus(sb);
sb.AppendLine();
// Peer Discovery Status
AppendPeerDiscoveryStatus(sb);
sb.AppendLine();
@ -257,12 +270,54 @@ namespace Convai.Scripts.Runtime.Multiplayer
AppendSpeechReceiverStatus(sb);
sb.AppendLine();
// Packet Rates
AppendPacketRates(sb);
sb.AppendLine();
// Event Log
AppendEventLog(sb);
sb.AppendLine();
// Packet Log
AppendPacketLog(sb);
_debugText.text = sb.ToString();
}
private void AppendPortBindingStatus(StringBuilder sb)
{
sb.AppendLine("🔌 PORT BINDING STATUS");
// Check if shared listener is active
bool sharedListenerActive = SharedUDPListener.Instance != null && SharedUDPListener.Instance.IsListening;
if (sharedListenerActive)
{
sb.AppendLine($"✅ Shared UDP Listener ACTIVE");
sb.AppendLine($" Port: {SharedUDPListener.Instance.ListenPort}");
sb.AppendLine($" Total Packets: {SharedUDPListener.Instance.TotalPacketsReceived}");
}
else
{
sb.AppendLine($"❌ Shared UDP Listener NOT FOUND!");
sb.AppendLine($" Add SharedUDPListener to scene!");
}
// Check which components are subscribed
bool discoveryActive = _peerDiscovery != null;
bool audioReceiverActive = _audioReceiver != null;
bool speechReceiverActive = _speechReceiver != null;
sb.AppendLine($"Discovery: {(discoveryActive ? "✅ SUBSCRIBED" : "❌ NOT FOUND")} (0x44495343)");
sb.AppendLine($"Audio RX: {(audioReceiverActive ? "✅ SUBSCRIBED" : "❌ NOT FOUND")} (0xC0A1)");
sb.AppendLine($"Speech RX: {(speechReceiverActive ? "✅ SUBSCRIBED" : "❌ NOT FOUND")} (0xC0A3)");
if (!sharedListenerActive)
{
sb.AppendLine("⚠️ CRITICAL: No shared listener!");
}
}
private void AppendPeerDiscoveryStatus(StringBuilder sb)
{
sb.AppendLine("🔍 PEER DISCOVERY");
@ -506,13 +561,99 @@ namespace Convai.Scripts.Runtime.Multiplayer
}
}
private void AppendPacketRates(StringBuilder sb)
{
sb.AppendLine("📊 PACKET RATES (last second)");
float currentTime = Time.time;
float deltaTime = currentTime - _lastRateCheckTime;
if (deltaTime >= 1f)
{
// Calculate rates
int audioSentDelta = 0;
int audioReceivedDelta = 0;
int speechSentDelta = 0;
int speechReceivedDelta = 0;
if (_audioSender != null)
{
audioSentDelta = _audioSender.TotalPacketsSent - _lastAudioSentCount;
_lastAudioSentCount = _audioSender.TotalPacketsSent;
}
if (_audioReceiver != null)
{
audioReceivedDelta = _audioReceiver.TotalPacketsReceived - _lastAudioReceivedCount;
_lastAudioReceivedCount = _audioReceiver.TotalPacketsReceived;
}
if (_speechSender != null)
{
speechSentDelta = _speechSender.TotalClipsSent - _lastSpeechSentCount;
_lastSpeechSentCount = _speechSender.TotalClipsSent;
}
if (_speechReceiver != null)
{
speechReceivedDelta = _speechReceiver.TotalClipsReceived - _lastSpeechReceivedCount;
_lastSpeechReceivedCount = _speechReceiver.TotalClipsReceived;
}
_lastRateCheckTime = currentTime;
// Log significant activity
if (audioSentDelta > 0)
LogPacketActivity($"Sent {audioSentDelta} voice packets (0xC0A1)");
if (audioReceivedDelta > 0)
LogPacketActivity($"Received {audioReceivedDelta} voice packets (0xC0A1)");
if (speechSentDelta > 0)
LogPacketActivity($"Sent {speechSentDelta} speech clips (0xC0A3)");
if (speechReceivedDelta > 0)
LogPacketActivity($"Received {speechReceivedDelta} speech clips (0xC0A3)");
}
sb.AppendLine($"Voice Sent: {(_audioSender != null ? _audioSender.TotalPacketsSent : 0)} total");
sb.AppendLine($"Voice Received: {(_audioReceiver != null ? _audioReceiver.TotalPacketsReceived : 0)} total");
sb.AppendLine($"Speech Sent: {(_speechSender != null ? _speechSender.TotalClipsSent : 0)} clips");
sb.AppendLine($"Speech Received: {(_speechReceiver != null ? _speechReceiver.TotalClipsReceived : 0)} clips");
}
private void LogPacketActivity(string message)
{
string timestamp = DateTime.Now.ToString("HH:mm:ss");
_packetLog.Enqueue($"[{timestamp}] {message}");
while (_packetLog.Count > MAX_PACKET_LOG)
{
_packetLog.Dequeue();
}
}
private void AppendPacketLog(StringBuilder sb)
{
sb.AppendLine("📦 PACKET ACTIVITY LOG");
if (_packetLog.Count > 0)
{
foreach (var entry in _packetLog)
{
sb.AppendLine(entry);
}
}
else
{
sb.AppendLine("No packet activity yet");
}
}
private void AppendEventLog(StringBuilder sb)
{
sb.AppendLine("📋 EVENT LOG");
sb.AppendLine("📋 CONNECTION EVENTS");
if (_peerDiscovery != null && _peerDiscovery.EventLog.Count > 0)
{
int startIndex = Math.Max(0, _peerDiscovery.EventLog.Count - 8);
int startIndex = Math.Max(0, _peerDiscovery.EventLog.Count - 6);
for (int i = startIndex; i < _peerDiscovery.EventLog.Count; i++)
{
sb.AppendLine(_peerDiscovery.EventLog[i]);

192
Unity-Master/Assets/Scripts/Multiplayer/SharedUDPListener.cs Normal file
View File

@ -0,0 +1,192 @@
using System;
using System.Net;
using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
using Convai.Scripts.Runtime.LoggerSystem;
using UnityEngine;
namespace Convai.Scripts.Runtime.Multiplayer
{
/// <summary>
/// Shared UDP Listener - Single UDP socket that dispatches packets to appropriate handlers
/// Solves the port binding issue where multiple components tried to bind to the same port
/// </summary>
public class SharedUDPListener : MonoBehaviour
{
// Singleton
private static SharedUDPListener _instance;
public static SharedUDPListener Instance => _instance;
// Network components
private UdpClient _udpClient;
private bool _isListening = false;
private int _listenPort;
private CancellationTokenSource _cancellationTokenSource;
// Packet handlers - registered by other components
public event Action<byte[], IPEndPoint> OnPacketReceived;
// Metrics
private int _totalPacketsReceived = 0;
private DateTime _lastPacketTime;
public int TotalPacketsReceived => _totalPacketsReceived;
public bool IsListening => _isListening;
public int ListenPort => _listenPort;
private void Awake()
{
// Singleton pattern
if (_instance != null && _instance != this)
{
Destroy(gameObject);
return;
}
_instance = this;
}
private void Start()
{
// Get network config
var cfg = NetworkConfig.Instance;
if (cfg != null)
{
_listenPort = cfg.port;
}
else
{
ConvaiLogger.Error("NetworkConfig not found! Using default port 1221", ConvaiLogger.LogCategory.Character);
_listenPort = 1221;
}
_cancellationTokenSource = new CancellationTokenSource();
StartListening();
}
private void OnDestroy()
{
StopListening();
_cancellationTokenSource?.Cancel();
_cancellationTokenSource?.Dispose();
if (_instance == this)
_instance = null;
}
private void StartListening()
{
if (_isListening) return;
try
{
// Create single UDP client for ALL network traffic
_udpClient = new UdpClient();
_udpClient.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_udpClient.Client.Bind(new IPEndPoint(IPAddress.Any, _listenPort));
_udpClient.EnableBroadcast = true;
_isListening = true;
ConvaiLogger.Info($"✅ Shared UDP Listener BOUND to port {_listenPort}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($" Will dispatch packets to handlers based on magic numbers:", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($" - Discovery: 0x44495343", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($" - Voice Audio: 0xC0A1", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($" - NPC Speech: 0xC0A3", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($" - Avatar Sync: 0xC0A0", ConvaiLogger.LogCategory.Character);
// Start listening for all packets
_ = ListenForPackets(_cancellationTokenSource.Token);
}
catch (Exception ex)
{
ConvaiLogger.Error($"❌ FAILED to bind Shared UDP Listener to port {_listenPort}: {ex.Message}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Error($"Stack trace: {ex.StackTrace}", ConvaiLogger.LogCategory.Character);
}
}
private void StopListening()
{
if (!_isListening) return;
_isListening = false;
_udpClient?.Close();
_udpClient?.Dispose();
_udpClient = null;
ConvaiLogger.Info("Shared UDP Listener stopped", ConvaiLogger.LogCategory.Character);
}
private async Task ListenForPackets(CancellationToken cancellationToken)
{
try
{
while (_isListening && !cancellationToken.IsCancellationRequested)
{
var result = await _udpClient.ReceiveAsync();
_totalPacketsReceived++;
_lastPacketTime = DateTime.UtcNow;
// Dispatch to all registered handlers
OnPacketReceived?.Invoke(result.Buffer, result.RemoteEndPoint);
}
}
catch (ObjectDisposedException)
{
// Normal when stopping
}
catch (Exception ex)
{
if (_isListening)
{
ConvaiLogger.Error($"Error in shared UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
}
/// <summary>
/// Send a packet through the shared UDP client
/// </summary>
public async Task<int> SendAsync(byte[] data, IPEndPoint endPoint)
{
if (_udpClient == null || !_isListening)
{
ConvaiLogger.Warn("Cannot send packet - UDP client not initialized", ConvaiLogger.LogCategory.Character);
return 0;
}
try
{
return await _udpClient.SendAsync(data, data.Length, endPoint);
}
catch (Exception ex)
{
ConvaiLogger.Error($"Failed to send UDP packet: {ex.Message}", ConvaiLogger.LogCategory.Character);
return 0;
}
}
/// <summary>
/// Send a broadcast packet
/// </summary>
public async Task<int> SendBroadcastAsync(byte[] data, int port)
{
if (_udpClient == null || !_isListening)
{
ConvaiLogger.Warn("Cannot send broadcast - UDP client not initialized", ConvaiLogger.LogCategory.Character);
return 0;
}
try
{
var broadcastEndpoint = new IPEndPoint(IPAddress.Broadcast, port);
return await _udpClient.SendAsync(data, data.Length, broadcastEndpoint);
}
catch (Exception ex)
{
ConvaiLogger.Error($"Failed to send broadcast packet: {ex.Message}", ConvaiLogger.LogCategory.Character);
return 0;
}
}
}
}

69
Unity-Master/Assets/Scripts/Multiplayer/UDPPeerDiscovery.cs
View File

@ -1,6 +1,5 @@
using System;
using System.Net;
using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
using Convai.Scripts.Runtime.LoggerSystem;
@ -29,7 +28,6 @@ namespace Convai.Scripts.Runtime.Multiplayer
public event Action<ConnectionState> OnConnectionStateChanged;
// Network components
private UdpClient _udpClient;
private bool _isRunning = false;
private int _listenPort;
private CancellationTokenSource _cancellationTokenSource;
@ -134,19 +132,20 @@ namespace Convai.Scripts.Runtime.Multiplayer
try
{
// Create UDP client with port reuse for shared port
_udpClient = new UdpClient();
_udpClient.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_udpClient.Client.Bind(new IPEndPoint(IPAddress.Any, _listenPort));
_udpClient.EnableBroadcast = true;
// Wait for shared listener to be ready
if (SharedUDPListener.Instance == null)
{
ConvaiLogger.Error("SharedUDPListener not found! Make sure it's in the scene.", ConvaiLogger.LogCategory.Character);
return;
}
// Subscribe to shared listener
SharedUDPListener.Instance.OnPacketReceived += HandlePacketReceived;
_isRunning = true;
SetConnectionState(ConnectionState.Discovering);
ConvaiLogger.Info($"🔍 Peer Discovery started - Player {localPlayerID} on port {_listenPort}", ConvaiLogger.LogCategory.Character);
// Start listening for discovery packets
_ = ListenForDiscoveryPackets(_cancellationTokenSource.Token);
ConvaiLogger.Info($"✅ Peer Discovery subscribed to shared listener - Player {localPlayerID}, listening for magic 0x{DISCOVERY_MAGIC:X}", ConvaiLogger.LogCategory.Character);
// Start broadcasting discovery requests
_ = BroadcastDiscoveryRequests(_cancellationTokenSource.Token);
@ -165,36 +164,27 @@ namespace Convai.Scripts.Runtime.Multiplayer
if (!_isRunning) return;
_isRunning = false;
_udpClient?.Close();
_udpClient?.Dispose();
_udpClient = null;
// Unsubscribe from shared listener
if (SharedUDPListener.Instance != null)
{
SharedUDPListener.Instance.OnPacketReceived -= HandlePacketReceived;
}
SetConnectionState(ConnectionState.Disconnected);
ConvaiLogger.Info("Peer Discovery stopped", ConvaiLogger.LogCategory.Character);
}
private async Task ListenForDiscoveryPackets(CancellationToken cancellationToken)
private void HandlePacketReceived(byte[] data, IPEndPoint senderEndPoint)
{
while (_isRunning && !cancellationToken.IsCancellationRequested)
{
try
{
var result = await _udpClient.ReceiveAsync();
await ProcessDiscoveryPacket(result.Buffer, result.RemoteEndPoint);
}
catch (ObjectDisposedException)
{
// Normal when stopping
break;
}
catch (Exception ex)
{
if (_isRunning)
{
ConvaiLogger.Error($"Error receiving discovery packet: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
}
// Check if this is a discovery packet (by magic number)
if (data.Length < 14) return;
uint magic = BitConverter.ToUInt32(data, 0);
if (magic != DISCOVERY_MAGIC) return;
// Process discovery packet
_ = ProcessDiscoveryPacket(data, senderEndPoint);
}
private async Task BroadcastDiscoveryRequests(CancellationToken cancellationToken)
@ -256,9 +246,8 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
byte[] packet = CreateDiscoveryPacket(PACKET_TYPE_REQUEST);
// Broadcast to subnet (will be blocked by AP isolation but we try anyway)
var broadcastEndPoint = new IPEndPoint(IPAddress.Broadcast, _listenPort);
await _udpClient.SendAsync(packet, packet.Length, broadcastEndPoint);
// Broadcast to subnet using shared listener
await SharedUDPListener.Instance.SendBroadcastAsync(packet, _listenPort);
if (enableDebugLogging && UnityEngine.Random.value < 0.1f) // Log 10% of broadcasts to reduce spam
{
@ -276,7 +265,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
try
{
byte[] packet = CreateDiscoveryPacket(PACKET_TYPE_RESPONSE);
await _udpClient.SendAsync(packet, packet.Length, targetEndPoint);
await SharedUDPListener.Instance.SendAsync(packet, targetEndPoint);
if (enableDebugLogging)
{
@ -297,7 +286,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
byte[] packet = CreateDiscoveryPacket(PACKET_TYPE_HEARTBEAT);
var peerEndPoint = new IPEndPoint(IPAddress.Parse(_peerIP), _listenPort);
await _udpClient.SendAsync(packet, packet.Length, peerEndPoint);
await SharedUDPListener.Instance.SendAsync(packet, peerEndPoint);
}
catch (Exception ex)
{

147
Unity-Master/Assets/Scripts/UDPAvatarReceiver.cs
View File

@ -2,9 +2,6 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using UnityEngine;
public class UDPAvatarReceiver : MonoBehaviour
@ -38,8 +35,6 @@ public class UDPAvatarReceiver : MonoBehaviour
[SerializeField] private bool showDebugInfo = false;
[SerializeField] private bool logReceivedPackets = false;
private UdpClient udpClient;
private Thread udpListenerThread;
private bool threadRunning = false;
private int listenPort;
private Dictionary<string, Transform> boneCache;
@ -205,123 +200,64 @@ public class UDPAvatarReceiver : MonoBehaviour
{
try
{
if (allowPortSharing)
// Wait for shared listener to be ready
if (Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance == null)
{
// Create UDP client with port reuse for local testing
udpClient = new UdpClient();
udpClient.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
udpClient.Client.Bind(new IPEndPoint(IPAddress.Any, listenPort));
}
else
{
// Standard UDP client binding
udpClient = new UdpClient(listenPort);
Debug.LogError("SharedUDPListener not found! Make sure it's in the scene.");
enableReceiver = false;
return;
}
// Subscribe to shared listener
Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance.OnPacketReceived += HandlePacketReceived;
threadRunning = true;
udpListenerThread = new Thread(new ThreadStart(UDPListenerThread));
udpListenerThread.IsBackground = true;
udpListenerThread.Start();
if (showDebugInfo)
Debug.Log($"UDP Avatar Receiver started on port {listenPort} (filtering binary avatar data only, Port sharing: {allowPortSharing})");
Debug.Log($"UDP Avatar Receiver subscribed to shared listener on port {listenPort} (filtering avatar magic 0x{AVATAR_MAGIC:X})");
}
catch (Exception e)
{
if (allowPortSharing)
{
Debug.LogWarning($"Failed to start UDP listener with port sharing: {e.Message}");
Debug.LogWarning("Trying with different port...");
TryAlternativePort();
}
else
{
Debug.LogError($"Failed to start UDP listener: {e.Message}");
enableReceiver = false;
}
Debug.LogError($"Failed to subscribe to shared UDP listener: {e.Message}");
enableReceiver = false;
}
}
void TryAlternativePort()
void HandlePacketReceived(byte[] data, IPEndPoint senderEndPoint)
{
// Try a few alternative ports for local testing
int[] alternativePorts = { 8081, 8082, 8083, 8084, 8085 };
// Check if this is avatar data (by magic number)
if (!IsAvatarData(data)) return;
foreach (int port in alternativePorts)
// Process avatar packet
CompactAvatarData avatarData = DeserializeCompactData(data);
// Check if this is from the target player (0 means accept from any player)
if (targetPlayerID == 0 || avatarData.playerID == targetPlayerID)
{
try
// Check for packet loss
if (avatarData.sequenceNumber > lastSequenceNumber + 1)
{
udpClient = new UdpClient(port);
threadRunning = true;
udpListenerThread = new Thread(new ThreadStart(UDPListenerThread));
udpListenerThread.IsBackground = true;
udpListenerThread.Start();
Debug.Log($"UDP listener started on alternative port {port}");
return;
packetsDropped += (int)(avatarData.sequenceNumber - lastSequenceNumber - 1);
}
catch (Exception)
lastSequenceNumber = avatarData.sequenceNumber;
packetsReceived++;
// Store the new data (thread-safe)
lock (dataLock)
{
// Try next port
continue;
lastReceivedData = avatarData;
hasNewData = true;
}
}
Debug.LogError("Failed to start UDP listener on any available port");
enableReceiver = false;
}
void UDPListenerThread()
{
IPEndPoint remoteEndPoint = new IPEndPoint(IPAddress.Any, 0);
while (threadRunning)
{
try
if (logReceivedPackets && packetsReceived % 30 == 0)
{
byte[] data = udpClient.Receive(ref remoteEndPoint);
if (data.Length > 0 && IsAvatarData(data))
{
CompactAvatarData avatarData = DeserializeCompactData(data);
// Check if this is from the target player (0 means accept from any player)
if (targetPlayerID == 0 || avatarData.playerID == targetPlayerID)
{
// Check for packet loss
if (avatarData.sequenceNumber > lastSequenceNumber + 1)
{
packetsDropped += (int)(avatarData.sequenceNumber - lastSequenceNumber - 1);
}
lastSequenceNumber = avatarData.sequenceNumber;
packetsReceived++;
// Store the new data (thread-safe)
lock (dataLock)
{
lastReceivedData = avatarData;
hasNewData = true;
}
if (logReceivedPackets && packetsReceived % 30 == 0)
{
string modeStr = avatarData.isFullDataMode ? "FULL" : "OPT";
Debug.Log($"Received {modeStr} packet #{avatarData.sequenceNumber} from player {avatarData.playerID}, size: {data.Length} bytes, bones: {avatarData.bones?.Length ?? 0}, blend shapes: {avatarData.blendShapes?.Length ?? 0}");
}
}
}
}
catch (Exception e)
{
if (threadRunning) // Only log errors if we're supposed to be running
{
Debug.LogError($"UDP receive error: {e.Message}");
}
string modeStr = avatarData.isFullDataMode ? "FULL" : "OPT";
Debug.Log($"Received {modeStr} packet #{avatarData.sequenceNumber} from player {avatarData.playerID}, size: {data.Length} bytes, bones: {avatarData.bones?.Length ?? 0}, blend shapes: {avatarData.blendShapes?.Length ?? 0}");
}
}
}
bool IsAvatarData(byte[] data)
{
try
@ -618,17 +554,10 @@ public class UDPAvatarReceiver : MonoBehaviour
{
threadRunning = false;
if (udpClient != null)
// Unsubscribe from shared listener
if (Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance != null)
{
udpClient.Close();
udpClient.Dispose();
udpClient = null;
}
if (udpListenerThread != null)
{
udpListenerThread.Join(1000); // Wait up to 1 second for thread to finish
udpListenerThread = null;
Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance.OnPacketReceived -= HandlePacketReceived;
}
}
@ -688,9 +617,9 @@ public class UDPAvatarReceiver : MonoBehaviour
int GetActualListenPort()
{
if (udpClient?.Client?.LocalEndPoint != null)
if (Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance != null)
{
return ((IPEndPoint)udpClient.Client.LocalEndPoint).Port;
return Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance.ListenPort;
}
return listenPort;
}

147
Unity-Master/Assets/Scripts/UDPAvatarReceiverAgent.cs
View File

@ -2,9 +2,6 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using UnityEngine;
/// <summary>
@ -41,8 +38,6 @@ public class UDPAvatarReceiverAgent : MonoBehaviour
[SerializeField] private bool showDebugInfo = false;
[SerializeField] private bool logReceivedPackets = false;
private UdpClient udpClient;
private Thread udpListenerThread;
private bool threadRunning = false;
private int listenPort;
private Dictionary<string, Transform> boneCache;
@ -208,123 +203,64 @@ public class UDPAvatarReceiverAgent : MonoBehaviour
{
try
{
if (allowPortSharing)
// Wait for shared listener to be ready
if (Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance == null)
{
// Create UDP client with port reuse for local testing
udpClient = new UdpClient();
udpClient.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
udpClient.Client.Bind(new IPEndPoint(IPAddress.Any, listenPort));
}
else
{
// Standard UDP client binding
udpClient = new UdpClient(listenPort);
Debug.LogError("SharedUDPListener not found! Make sure it's in the scene.");
enableReceiver = false;
return;
}
// Subscribe to shared listener
Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance.OnPacketReceived += HandlePacketReceived;
threadRunning = true;
udpListenerThread = new Thread(new ThreadStart(UDPListenerThread));
udpListenerThread.IsBackground = true;
udpListenerThread.Start();
if (showDebugInfo)
Debug.Log($"UDP Avatar Receiver Agent started on port {listenPort} (filtering binary avatar data only, Port sharing: {allowPortSharing})");
Debug.Log($"UDP Avatar Receiver Agent subscribed to shared listener on port {listenPort} (filtering avatar magic 0x{AVATAR_MAGIC:X})");
}
catch (Exception e)
{
if (allowPortSharing)
{
Debug.LogWarning($"Failed to start UDP listener with port sharing: {e.Message}");
Debug.LogWarning("Trying with different port...");
TryAlternativePort();
}
else
{
Debug.LogError($"Failed to start UDP listener: {e.Message}");
enableReceiver = false;
}
Debug.LogError($"Failed to subscribe to shared UDP listener: {e.Message}");
enableReceiver = false;
}
}
void TryAlternativePort()
void HandlePacketReceived(byte[] data, IPEndPoint senderEndPoint)
{
// Try a few alternative ports for local testing
int[] alternativePorts = { 8081, 8082, 8083, 8084, 8085 };
// Check if this is avatar data (by magic number)
if (!IsAvatarData(data)) return;
foreach (int port in alternativePorts)
// Process avatar packet
CompactAvatarData avatarData = DeserializeCompactData(data);
// Check if this is from the target player (0 means accept from any player)
if (targetPlayerID == 0 || avatarData.playerID == targetPlayerID)
{
try
// Check for packet loss
if (avatarData.sequenceNumber > lastSequenceNumber + 1)
{
udpClient = new UdpClient(port);
threadRunning = true;
udpListenerThread = new Thread(new ThreadStart(UDPListenerThread));
udpListenerThread.IsBackground = true;
udpListenerThread.Start();
Debug.Log($"UDP listener started on alternative port {port}");
return;
packetsDropped += (int)(avatarData.sequenceNumber - lastSequenceNumber - 1);
}
catch (Exception)
lastSequenceNumber = avatarData.sequenceNumber;
packetsReceived++;
// Store the new data (thread-safe)
lock (dataLock)
{
// Try next port
continue;
lastReceivedData = avatarData;
hasNewData = true;
}
}
Debug.LogError("Failed to start UDP listener on any available port");
enableReceiver = false;
}
void UDPListenerThread()
{
IPEndPoint remoteEndPoint = new IPEndPoint(IPAddress.Any, 0);
while (threadRunning)
{
try
if (logReceivedPackets && packetsReceived % 30 == 0)
{
byte[] data = udpClient.Receive(ref remoteEndPoint);
if (data.Length > 0 && IsAvatarData(data))
{
CompactAvatarData avatarData = DeserializeCompactData(data);
// Check if this is from the target player (0 means accept from any player)
if (targetPlayerID == 0 || avatarData.playerID == targetPlayerID)
{
// Check for packet loss
if (avatarData.sequenceNumber > lastSequenceNumber + 1)
{
packetsDropped += (int)(avatarData.sequenceNumber - lastSequenceNumber - 1);
}
lastSequenceNumber = avatarData.sequenceNumber;
packetsReceived++;
// Store the new data (thread-safe)
lock (dataLock)
{
lastReceivedData = avatarData;
hasNewData = true;
}
if (logReceivedPackets && packetsReceived % 30 == 0)
{
string modeStr = avatarData.isFullDataMode ? "FULL" : "OPT";
Debug.Log($"Received {modeStr} packet #{avatarData.sequenceNumber} from player {avatarData.playerID}, size: {data.Length} bytes, bones: {avatarData.bones?.Length ?? 0}, blend shapes: {avatarData.blendShapes?.Length ?? 0}");
}
}
}
}
catch (Exception e)
{
if (threadRunning) // Only log errors if we're supposed to be running
{
Debug.LogError($"UDP receive error: {e.Message}");
}
string modeStr = avatarData.isFullDataMode ? "FULL" : "OPT";
Debug.Log($"Received {modeStr} packet #{avatarData.sequenceNumber} from player {avatarData.playerID}, size: {data.Length} bytes, bones: {avatarData.bones?.Length ?? 0}, blend shapes: {avatarData.blendShapes?.Length ?? 0}");
}
}
}
bool IsAvatarData(byte[] data)
{
try
@ -621,17 +557,10 @@ public class UDPAvatarReceiverAgent : MonoBehaviour
{
threadRunning = false;
if (udpClient != null)
// Unsubscribe from shared listener
if (Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance != null)
{
udpClient.Close();
udpClient.Dispose();
udpClient = null;
}
if (udpListenerThread != null)
{
udpListenerThread.Join(1000); // Wait up to 1 second for thread to finish
udpListenerThread = null;
Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance.OnPacketReceived -= HandlePacketReceived;
}
}
@ -691,9 +620,9 @@ public class UDPAvatarReceiverAgent : MonoBehaviour
int GetActualListenPort()
{
if (udpClient?.Client?.LocalEndPoint != null)
if (Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance != null)
{
return ((IPEndPoint)udpClient.Client.LocalEndPoint).Port;
return Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance.ListenPort;
}
return listenPort;
}