using System; using System.Collections.Generic; using System.Net; using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; using Convai.Scripts.Runtime.Core; using Convai.Scripts.Runtime.LoggerSystem; using Convai.Scripts.Runtime.Utils; using UnityEngine; using System.IO; namespace Convai.Scripts.Runtime.Multiplayer { /// /// Simple UDP Audio Receiver V2 - Simulates microphone input by triggering normal Convai flow /// This approach is much simpler and more reliable than trying to replicate gRPC calls /// public class ConvaiSimpleUDPAudioReceiver : MonoBehaviour { [Header("Network Configuration")] [SerializeField] private bool enableDebugLogging = true; [Header("NPC Target")] [SerializeField] private bool useActiveNPC = true; [SerializeField] private ConvaiNPC targetNPC; // Events public Action OnAudioReceiving; // Metrics for debug UI private int _totalPacketsReceived = 0; private DateTime _lastPacketReceivedTime; public int TotalPacketsReceived => _totalPacketsReceived; public float TimeSinceLastReceive => _lastPacketReceivedTime != default ? (float)(DateTime.UtcNow - _lastPacketReceivedTime).TotalSeconds : -1f; public int ListenPort => listenPort; // Network components private UdpClient _udpListener; private IPEndPoint _remoteEndPoint; private bool _isListening = false; private int listenPort; private CancellationTokenSource _cancellationTokenSource; // Audio state tracking private bool _isReceivingAudio = false; private int _expectedSequence = 0; private const uint MAGIC_NUMBER = 0xC0A1; // Simple magic number for packet validation private const uint ACK_MAGIC = 0xC0A2; // ACK magic to confirm START control // Timing for auto-stop private float _lastPacketTime; private const float AUTO_STOP_DELAY = 1.0f; // Stop listening after 1 second of no packets // Packet structure (matching ConvaiSimpleUDPAudioSender) private struct AudioPacketData { public uint magicNumber; public int sequence; public int sampleCount; public int microphonePosition; public bool isEndSignal; public bool isStartSignal; public short[] audioSamples; public long timestamp; } [Header("Recording Storage")] [SerializeField] private bool saveReceivedAudio = true; [SerializeField] private int receivedSampleRate = 16000; // Should match sender [SerializeField] private string outputFilePrefix = "received_audio"; private readonly object _audioBufferLock = new object(); private List _receivedSamples = new List(64 * 1024); private Dictionary _pendingPackets = new Dictionary(); private int _nextSequenceToWrite = 0; private DateTime _sessionStartTime; private bool _saveInProgress = false; private string _persistentDataPath; private void Start() { _cancellationTokenSource = new CancellationTokenSource(); _persistentDataPath = Application.persistentDataPath; // Get network config from global instance var cfg = NetworkConfig.Instance; if (cfg != null) { listenPort = cfg.port; } else { Debug.LogError("NetworkConfig not found! Please ensure NetworkConfig.asset exists in Resources folder."); listenPort = 1221; } InitializeNetwork(); InitializeConvai(); // Subscribe to NPC manager events to handle late NPC activation if (ConvaiNPCManager.Instance != null) { ConvaiNPCManager.Instance.OnActiveNPCChanged += HandleActiveNPCChanged; } } private void OnEnable() { // When re-enabled, ensure listener is running if (_cancellationTokenSource == null) { _cancellationTokenSource = new CancellationTokenSource(); } StartListening(); // Immediately try to assign an enabled NPC if (useActiveNPC && targetNPC == null) { var currentActiveNPC = FindEnabledConvaiNPC(); if (currentActiveNPC != null) { targetNPC = currentActiveNPC; ConvaiLogger.Info($"🔄 UDP Audio Receiver assigned target NPC on enable: {targetNPC.characterName} (on {targetNPC.gameObject.name})", ConvaiLogger.LogCategory.Character); } } } private void OnDestroy() { // Unsubscribe from events if (ConvaiNPCManager.Instance != null) { ConvaiNPCManager.Instance.OnActiveNPCChanged -= HandleActiveNPCChanged; } StopListening(); _cancellationTokenSource?.Cancel(); _cancellationTokenSource?.Dispose(); } private void OnDisable() { // Free the UDP port when this NPC gets disabled StopListening(); } private void Update() { // Auto-stop listening if no packets received for a while if (_isReceivingAudio && Time.time - _lastPacketTime > AUTO_STOP_DELAY) { StopTalkingSimulation(); } // Continuously update target NPC if using active NPC mode if (useActiveNPC) { var currentActiveNPC = FindEnabledConvaiNPC(); // Update whenever the active NPC changes (including null → NPC or NPC → different NPC) if (currentActiveNPC != targetNPC) { targetNPC = currentActiveNPC; if (targetNPC != null) { ConvaiLogger.Info($"🔄 UDP Audio Receiver updated target NPC to: {targetNPC.characterName} (on {targetNPC.gameObject.name})", ConvaiLogger.LogCategory.Character); } else { ConvaiLogger.Info($"🔄 UDP Audio Receiver cleared target NPC", ConvaiLogger.LogCategory.Character); } } } } /// /// Finds an enabled ConvaiNPC in the scene (doesn't rely on ConvaiNPCManager raycasting) /// private ConvaiNPC FindEnabledConvaiNPC() { // Find all ConvaiNPC components in the scene (including inactive GameObjects) var allNPCs = FindObjectsOfType(true); // Return the first one that's on an active GameObject foreach (var npc in allNPCs) { if (npc.gameObject.activeInHierarchy && npc.enabled) { return npc; } } return null; } private void InitializeNetwork() { try { StartListening(); } catch (Exception ex) { ConvaiLogger.Error($"Failed to initialize UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character); } } private void InitializeConvai() { // Get target NPC by finding enabled NPCs in the scene if (useActiveNPC) { targetNPC = FindEnabledConvaiNPC(); } if (targetNPC == null) { ConvaiLogger.Warn("No target NPC found yet, will wait for NPC to become active", ConvaiLogger.LogCategory.Character); } else { ConvaiLogger.Info($"UDP Audio Receiver V2 initialized with NPC: {targetNPC.characterName} (on {targetNPC.gameObject.name})", ConvaiLogger.LogCategory.Character); } } public void StartListening() { if (_isListening || _cancellationTokenSource == null) return; try { // Create UDP client with port reuse to allow sharing with UDPPeerDiscovery _udpListener = new UdpClient(); _udpListener.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); _udpListener.Client.Bind(new IPEndPoint(IPAddress.Any, listenPort)); _isListening = true; ConvaiLogger.Info($"Simple UDP Audio Receiver V2 listening on port {listenPort} (shared)", ConvaiLogger.LogCategory.Character); // Start listening for incoming packets _ = ListenForAudioPackets(_cancellationTokenSource.Token); } catch (Exception ex) { ConvaiLogger.Error($"Failed to start UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Error($"Stack trace: {ex.StackTrace}", ConvaiLogger.LogCategory.Character); } } public void StopListening() { if (!_isListening) return; _isListening = false; _udpListener?.Close(); _udpListener?.Dispose(); _udpListener = null; // Stop any ongoing simulation StopTalkingSimulation(); ConvaiLogger.Info("Stopped UDP Audio Receiver V2", ConvaiLogger.LogCategory.Character); } private async Task ListenForAudioPackets(CancellationToken cancellationToken) { try { while (_isListening && !cancellationToken.IsCancellationRequested) { var result = await _udpListener.ReceiveAsync(); _remoteEndPoint = result.RemoteEndPoint; await ProcessReceivedPacket(result.Buffer, result.RemoteEndPoint); } } catch (ObjectDisposedException) { // Normal when stopping } catch (Exception ex) { ConvaiLogger.Error($"Error in UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character); } } private Task ProcessReceivedPacket(byte[] data, IPEndPoint sender) { try { var packetData = ParseSimpleAudioPacket(data); if (packetData.HasValue) { var packet = packetData.Value; _lastPacketTime = Time.time; // Update metrics _totalPacketsReceived++; _lastPacketReceivedTime = DateTime.UtcNow; if (enableDebugLogging) { if (packet.isEndSignal) ConvaiLogger.DebugLog($"Received end signal from {sender}", ConvaiLogger.LogCategory.Character); else ConvaiLogger.DebugLog($"Received audio packet {packet.sequence} with {packet.sampleCount} samples", ConvaiLogger.LogCategory.Character); } // Handle START control: acknowledge and begin simulation if (packet.isStartSignal) { SendStartAck(sender); if (!_isReceivingAudio) { StartTalkingSimulation(); } OnAudioReceiving?.Invoke(true); return Task.CompletedTask; } if (packet.isEndSignal) { StopTalkingSimulation(); OnAudioReceiving?.Invoke(false); } else { // If this is the first packet, start the talking simulation if (packet.sequence == 0 && !_isReceivingAudio) { StartTalkingSimulation(); } // Buffer audio samples for saving if (packet.audioSamples != null && packet.audioSamples.Length > 0) { BufferAudioPacket(packet.sequence, packet.audioSamples); } OnAudioReceiving?.Invoke(true); } } else { // Not our audio packet format, might be a test message string message = System.Text.Encoding.UTF8.GetString(data); if (enableDebugLogging) ConvaiLogger.Info($"Received test message from {sender}: {message}", ConvaiLogger.LogCategory.Character); } } catch (Exception ex) { ConvaiLogger.Error($"Error processing received packet: {ex.Message}", ConvaiLogger.LogCategory.Character); } return Task.CompletedTask; } private void StartTalkingSimulation() { if (_isReceivingAudio) return; MainThreadDispatcher.Instance.RunOnMainThread(() => { // Update target NPC by finding enabled NPCs in the scene if (useActiveNPC) { targetNPC = FindEnabledConvaiNPC(); } if (targetNPC == null) { ConvaiLogger.Warn("No target NPC available for audio simulation", ConvaiLogger.LogCategory.Character); return; } _isReceivingAudio = true; _expectedSequence = 0; _nextSequenceToWrite = 0; _sessionStartTime = DateTime.UtcNow; lock (_audioBufferLock) { _receivedSamples.Clear(); _pendingPackets.Clear(); } // This is the KEY! Simulate a talk key press to trigger normal Convai flow ConvaiInputManager.Instance.talkKeyInteract?.Invoke(true); ConvaiLogger.Info($"🎤 Started talking simulation for {targetNPC.characterName} (on {targetNPC.gameObject.name}) (remote player audio)", ConvaiLogger.LogCategory.Character); }); } private void StopTalkingSimulation() { if (!_isReceivingAudio) return; MainThreadDispatcher.Instance.RunOnMainThread(() => { _isReceivingAudio = false; // Simulate talk key release to stop recording ConvaiInputManager.Instance.talkKeyInteract?.Invoke(false); ConvaiLogger.Info($"🎤 Stopped talking simulation for {targetNPC?.characterName ?? "NPC"} (remote player audio)", ConvaiLogger.LogCategory.Character); if (saveReceivedAudio) { TrySaveReceivedAudioAsync(); } }); } private AudioPacketData? ParseSimpleAudioPacket(byte[] data) { if (data.Length < 17) // Minimum header size to match sender return null; try { int offset = 0; // Read magic number uint magic = BitConverter.ToUInt32(data, offset); offset += 4; if (magic != MAGIC_NUMBER) return null; // Read header (matching sender's 17-byte format) int sequence = BitConverter.ToInt32(data, offset); offset += 4; int sampleCount = BitConverter.ToInt32(data, offset); offset += 4; int microphonePosition = BitConverter.ToInt32(data, offset); offset += 4; byte flags = data[offset]; offset += 1; bool isEndSignal = (flags == 1); bool isStartSignal = (flags == 2); // Read audio data short[] audioSamples = null; if (!isEndSignal && !isStartSignal && sampleCount > 0) { int audioDataSize = sampleCount * sizeof(short); if (data.Length >= offset + audioDataSize) { audioSamples = new short[sampleCount]; Buffer.BlockCopy(data, offset, audioSamples, 0, audioDataSize); } } return new AudioPacketData { magicNumber = magic, sequence = sequence, sampleCount = sampleCount, microphonePosition = microphonePosition, isEndSignal = isEndSignal, isStartSignal = isStartSignal, audioSamples = audioSamples, timestamp = 0 // Not provided in sender format }; } catch (Exception ex) { ConvaiLogger.Error($"Error parsing audio packet: {ex.Message}", ConvaiLogger.LogCategory.Character); return null; } } private void SendStartAck(IPEndPoint sender) { try { if (_udpListener == null || sender == null) return; byte[] ack = new byte[8]; BitConverter.GetBytes(ACK_MAGIC).CopyTo(ack, 0); BitConverter.GetBytes(-1).CopyTo(ack, 4); _udpListener.SendAsync(ack, ack.Length, sender); if (enableDebugLogging) ConvaiLogger.DebugLog($"Sent START ACK to {sender}", ConvaiLogger.LogCategory.Character); } catch (Exception ex) { ConvaiLogger.Warn($"Failed to send START ACK: {ex.Message}", ConvaiLogger.LogCategory.Character); } } private void BufferAudioPacket(int sequence, short[] samples) { if (samples == null || samples.Length == 0) return; lock (_audioBufferLock) { if (sequence < _nextSequenceToWrite) { return; // old/duplicate packet } if (sequence == _nextSequenceToWrite) { _receivedSamples.AddRange(samples); _nextSequenceToWrite++; // Flush any contiguous pending packets while (_pendingPackets.TryGetValue(_nextSequenceToWrite, out var nextSamples)) { _receivedSamples.AddRange(nextSamples); _pendingPackets.Remove(_nextSequenceToWrite); _nextSequenceToWrite++; } } else { // Store for later when gap is filled _pendingPackets[sequence] = samples; } } } private void TrySaveReceivedAudioAsync() { if (_saveInProgress) return; short[] dataToSave; DateTime sessionStart; lock (_audioBufferLock) { if (_receivedSamples == null || _receivedSamples.Count == 0) { if (enableDebugLogging) ConvaiLogger.Info("No received audio to save.", ConvaiLogger.LogCategory.Character); return; } dataToSave = _receivedSamples.ToArray(); _receivedSamples.Clear(); _pendingPackets.Clear(); sessionStart = _sessionStartTime; } _saveInProgress = true; Task.Run(() => { try { string timestamp = sessionStart.ToLocalTime().ToString("yyyyMMdd_HHmmss"); string fileName = $"{outputFilePrefix}_{timestamp}.wav"; string dir = _persistentDataPath; string path = Path.Combine(dir, fileName); WriteWav(path, dataToSave, receivedSampleRate, 1); ConvaiLogger.Info($"Saved received audio to: {path}", ConvaiLogger.LogCategory.Character); } catch (Exception ex) { ConvaiLogger.Error($"Failed to save received audio: {ex.Message}", ConvaiLogger.LogCategory.Character); } finally { _saveInProgress = false; } }); } private void WriteWav(string path, short[] samples, int sampleRate, int channels) { using (var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None)) using (var writer = new BinaryWriter(fs)) { int bitsPerSample = 16; int byteRate = sampleRate * channels * (bitsPerSample / 8); int blockAlign = channels * (bitsPerSample / 8); int dataSize = samples.Length * (bitsPerSample / 8); int fileSize = 44 - 8 + dataSize; // RIFF header writer.Write(System.Text.Encoding.ASCII.GetBytes("RIFF")); writer.Write(fileSize); writer.Write(System.Text.Encoding.ASCII.GetBytes("WAVE")); // fmt chunk writer.Write(System.Text.Encoding.ASCII.GetBytes("fmt ")); writer.Write(16); // Subchunk1Size for PCM writer.Write((short)1); // AudioFormat = PCM writer.Write((short)channels); // NumChannels writer.Write(sampleRate); // SampleRate writer.Write(byteRate); // ByteRate writer.Write((short)blockAlign); // BlockAlign writer.Write((short)bitsPerSample); // BitsPerSample // data chunk writer.Write(System.Text.Encoding.ASCII.GetBytes("data")); writer.Write(dataSize); for (int i = 0; i < samples.Length; i++) { writer.Write(samples[i]); } } } // Event handler for when NPC becomes active private void HandleActiveNPCChanged(ConvaiNPC newActiveNPC) { if (useActiveNPC && newActiveNPC != null) { targetNPC = newActiveNPC; ConvaiLogger.Info($"UDP Audio Receiver V2 updated target NPC to: {targetNPC.characterName}", ConvaiLogger.LogCategory.Character); } } // Public properties for debugging public bool IsListening => _isListening; public bool IsReceivingAudio => _isReceivingAudio; public ConvaiNPC TargetNPC => targetNPC; // Debug methods public void ShowNetworkStatus() { ConvaiLogger.Info($"=== Audio Receiver V2 Status ===", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Listening: {_isListening} on port {listenPort}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Receiving Audio: {_isReceivingAudio}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Target NPC: {(targetNPC?.characterName ?? "None")}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Expected Sequence: {_expectedSequence}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Last Packet Time: {_lastPacketTime}", ConvaiLogger.LogCategory.Character); } } }