using System; using System.Collections.Generic; using System.Net; using System.Threading; using System.Threading.Tasks; using Convai.Scripts.Runtime.Core; using Convai.Scripts.Runtime.LoggerSystem; using Convai.Scripts.Runtime.Utils; using UnityEngine; using System.IO; namespace Convai.Scripts.Runtime.Multiplayer { /// /// Simple UDP Audio Receiver V2 - Simulates microphone input by triggering normal Convai flow /// This approach is much simpler and more reliable than trying to replicate gRPC calls /// public class ConvaiSimpleUDPAudioReceiver : MonoBehaviour { [Header("Network Configuration")] [SerializeField] private bool enableDebugLogging = true; [Header("NPC Target")] [SerializeField] private bool useActiveNPC = true; [SerializeField] private ConvaiNPC targetNPC; // Events public Action OnAudioReceiving; // Metrics for debug UI private int _totalPacketsReceived = 0; private DateTime _lastPacketReceivedTime; public int TotalPacketsReceived => _totalPacketsReceived; public float TimeSinceLastReceive => _lastPacketReceivedTime != default ? (float)(DateTime.UtcNow - _lastPacketReceivedTime).TotalSeconds : -1f; public int ListenPort => listenPort; // Network components private IPEndPoint _remoteEndPoint; private bool _isListening = false; private int listenPort; private CancellationTokenSource _cancellationTokenSource; // Audio state tracking private bool _isReceivingAudio = false; private int _expectedSequence = 0; private const uint MAGIC_NUMBER = 0xC0A1; // Simple magic number for packet validation // Timing for auto-stop private float _lastPacketTime; private const float AUTO_STOP_DELAY = 1.0f; // Stop listening after 1 second of no packets // SIMPLIFIED Packet structure (matching ConvaiSimpleUDPAudioSender) private struct AudioPacketData { public uint magicNumber; public int sequence; public int sampleCount; public short[] audioSamples; } [Header("Recording Storage")] [SerializeField] private bool saveReceivedAudio = true; [SerializeField] private int receivedSampleRate = 16000; // Should match sender [SerializeField] private string outputFilePrefix = "received_audio"; private readonly object _audioBufferLock = new object(); private List _receivedSamples = new List(64 * 1024); private Dictionary _pendingPackets = new Dictionary(); private int _nextSequenceToWrite = 0; private DateTime _sessionStartTime; private bool _saveInProgress = false; private string _persistentDataPath; private void Start() { _cancellationTokenSource = new CancellationTokenSource(); _persistentDataPath = Application.persistentDataPath; // Get network config from global instance var cfg = NetworkConfig.Instance; if (cfg != null) { listenPort = cfg.port; } else { Debug.LogError("NetworkConfig not found! Please ensure NetworkConfig.asset exists in Resources folder."); listenPort = 1221; } InitializeNetwork(); InitializeConvai(); // Subscribe to NPC manager events to handle late NPC activation if (ConvaiNPCManager.Instance != null) { ConvaiNPCManager.Instance.OnActiveNPCChanged += HandleActiveNPCChanged; } } private void OnEnable() { // When re-enabled, ensure listener is running if (_cancellationTokenSource == null) { _cancellationTokenSource = new CancellationTokenSource(); } StartCoroutine(WaitAndSubscribe()); // Immediately try to assign an enabled NPC if (useActiveNPC && targetNPC == null) { var currentActiveNPC = FindEnabledConvaiNPC(); if (currentActiveNPC != null) { targetNPC = currentActiveNPC; ConvaiLogger.Info($"🔄 UDP Audio Receiver assigned target NPC on enable: {targetNPC.characterName} (on {targetNPC.gameObject.name})", ConvaiLogger.LogCategory.Character); } } } private void OnDestroy() { // Unsubscribe from events if (ConvaiNPCManager.Instance != null) { ConvaiNPCManager.Instance.OnActiveNPCChanged -= HandleActiveNPCChanged; } StopListening(); _cancellationTokenSource?.Cancel(); _cancellationTokenSource?.Dispose(); } private void OnDisable() { // Free the UDP port when this NPC gets disabled StopListening(); } private void Update() { // Auto-stop listening if no packets received for a while if (_isReceivingAudio && Time.time - _lastPacketTime > AUTO_STOP_DELAY) { StopTalkingSimulation(); } // Continuously update target NPC if using active NPC mode if (useActiveNPC) { var currentActiveNPC = FindEnabledConvaiNPC(); // Update whenever the active NPC changes (including null → NPC or NPC → different NPC) if (currentActiveNPC != targetNPC) { targetNPC = currentActiveNPC; if (targetNPC != null) { ConvaiLogger.Info($"🔄 UDP Audio Receiver updated target NPC to: {targetNPC.characterName} (on {targetNPC.gameObject.name})", ConvaiLogger.LogCategory.Character); } else { ConvaiLogger.Info($"🔄 UDP Audio Receiver cleared target NPC", ConvaiLogger.LogCategory.Character); } } } } /// /// Finds an enabled ConvaiNPC in the scene (doesn't rely on ConvaiNPCManager raycasting) /// private ConvaiNPC FindEnabledConvaiNPC() { // Find all ConvaiNPC components in the scene (including inactive GameObjects) var allNPCs = FindObjectsOfType(true); // Return the first one that's on an active GameObject foreach (var npc in allNPCs) { if (npc.gameObject.activeInHierarchy && npc.enabled) { return npc; } } return null; } private void InitializeNetwork() { try { StartListening(); } catch (Exception ex) { ConvaiLogger.Error($"Failed to initialize UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character); } } private void InitializeConvai() { // Get target NPC by finding enabled NPCs in the scene if (useActiveNPC) { targetNPC = FindEnabledConvaiNPC(); } if (targetNPC == null) { ConvaiLogger.Warn("No target NPC found yet, will wait for NPC to become active", ConvaiLogger.LogCategory.Character); } else { ConvaiLogger.Info($"UDP Audio Receiver V2 initialized with NPC: {targetNPC.characterName} (on {targetNPC.gameObject.name})", ConvaiLogger.LogCategory.Character); } } public void StartListening() { if (_isListening || _cancellationTokenSource == null) return; try { // Subscribe to shared listener SharedUDPListener.Instance.OnPacketReceived += HandlePacketReceived; _isListening = true; ConvaiLogger.Info($"✅ Audio Receiver subscribed to shared listener, listening for magic 0x{MAGIC_NUMBER:X}", ConvaiLogger.LogCategory.Character); } catch (Exception ex) { ConvaiLogger.Error($"❌ FAILED to subscribe Audio Receiver: {ex.Message}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Error($"Stack trace: {ex.StackTrace}", ConvaiLogger.LogCategory.Character); } } private System.Collections.IEnumerator WaitAndSubscribe() { float timeout = 3f; while (SharedUDPListener.Instance == null && timeout > 0f) { timeout -= Time.unscaledDeltaTime; yield return null; } if (SharedUDPListener.Instance == null) { ConvaiLogger.Error("SharedUDPListener not ready after wait.", ConvaiLogger.LogCategory.Character); yield break; } StartListening(); } public void StopListening() { if (!_isListening) return; _isListening = false; // Unsubscribe from shared listener if (SharedUDPListener.Instance != null) { SharedUDPListener.Instance.OnPacketReceived -= HandlePacketReceived; } // Stop any ongoing simulation StopTalkingSimulation(); ConvaiLogger.Info("Stopped UDP Audio Receiver V2", ConvaiLogger.LogCategory.Character); } private void HandlePacketReceived(byte[] data, IPEndPoint senderEndPoint) { // Check if this is an audio packet (by magic number) if (data.Length < 12) return; uint magic = BitConverter.ToUInt32(data, 0); if (magic != MAGIC_NUMBER) return; // Update remote endpoint _remoteEndPoint = senderEndPoint; // Process audio packet _ = ProcessReceivedPacket(data, senderEndPoint); } private Task ProcessReceivedPacket(byte[] data, IPEndPoint sender) { try { var packetData = ParseSimpleAudioPacket(data); if (packetData.HasValue) { var packet = packetData.Value; _lastPacketTime = Time.time; // Update metrics _totalPacketsReceived++; _lastPacketReceivedTime = DateTime.UtcNow; // SIMPLIFIED: Check for end signal (sampleCount == -1) bool isEndSignal = (packet.sampleCount == -1); if (enableDebugLogging && packet.sequence % 50 == 0) { ConvaiLogger.DebugLog($"📥 Received audio packet #{packet.sequence} (magic: 0x{packet.magicNumber:X}) from {sender}, {packet.sampleCount} samples", ConvaiLogger.LogCategory.Character); } if (isEndSignal) { ConvaiLogger.Info($"📥 Received END signal from {sender}", ConvaiLogger.LogCategory.Character); StopTalkingSimulation(); OnAudioReceiving?.Invoke(false); } else { // If this is the first packet, start the talking simulation if (packet.sequence == 0 && !_isReceivingAudio) { ConvaiLogger.Info($"📥 Received FIRST audio packet from {sender}, starting simulation", ConvaiLogger.LogCategory.Character); StartTalkingSimulation(); } // Buffer audio samples for saving if (packet.audioSamples != null && packet.audioSamples.Length > 0) { BufferAudioPacket(packet.sequence, packet.audioSamples); } OnAudioReceiving?.Invoke(true); } } else { // Not our audio packet format, might be a test message string message = System.Text.Encoding.UTF8.GetString(data); if (enableDebugLogging) ConvaiLogger.Info($"Received test message from {sender}: {message}", ConvaiLogger.LogCategory.Character); } } catch (Exception ex) { ConvaiLogger.Error($"Error processing received packet: {ex.Message}", ConvaiLogger.LogCategory.Character); } return Task.CompletedTask; } private void StartTalkingSimulation() { if (_isReceivingAudio) return; MainThreadDispatcher.Instance.RunOnMainThread(() => { // Update target NPC by finding enabled NPCs in the scene if (useActiveNPC) { targetNPC = FindEnabledConvaiNPC(); } if (targetNPC == null) { ConvaiLogger.Warn("No target NPC available for audio simulation", ConvaiLogger.LogCategory.Character); return; } _isReceivingAudio = true; _expectedSequence = 0; _nextSequenceToWrite = 0; _sessionStartTime = DateTime.UtcNow; lock (_audioBufferLock) { _receivedSamples.Clear(); _pendingPackets.Clear(); } // This is the KEY! Simulate a talk key press to trigger normal Convai flow ConvaiInputManager.Instance.talkKeyInteract?.Invoke(true); ConvaiLogger.Info($"🎤 Started talking simulation for {targetNPC.characterName} (on {targetNPC.gameObject.name}) (remote player audio)", ConvaiLogger.LogCategory.Character); }); } private void StopTalkingSimulation() { if (!_isReceivingAudio) return; MainThreadDispatcher.Instance.RunOnMainThread(() => { _isReceivingAudio = false; // Simulate talk key release to stop recording ConvaiInputManager.Instance.talkKeyInteract?.Invoke(false); ConvaiLogger.Info($"🎤 Stopped talking simulation for {targetNPC?.characterName ?? "NPC"} (remote player audio)", ConvaiLogger.LogCategory.Character); if (saveReceivedAudio) { TrySaveReceivedAudioAsync(); } }); } private AudioPacketData? ParseSimpleAudioPacket(byte[] data) { if (data.Length < 12) // SIMPLIFIED: Minimum header size (4+4+4) return null; try { int offset = 0; // Read magic number uint magic = BitConverter.ToUInt32(data, offset); offset += 4; if (magic != MAGIC_NUMBER) { if (enableDebugLogging) ConvaiLogger.Warn($"❌ Invalid magic number: expected 0x{MAGIC_NUMBER:X}, got 0x{magic:X}", ConvaiLogger.LogCategory.Character); return null; } // SIMPLIFIED header (12 bytes total) int sequence = BitConverter.ToInt32(data, offset); offset += 4; int sampleCount = BitConverter.ToInt32(data, offset); offset += 4; // Read audio data (if not end signal) short[] audioSamples = null; if (sampleCount > 0) { int audioDataSize = sampleCount * sizeof(short); if (data.Length >= offset + audioDataSize) { audioSamples = new short[sampleCount]; Buffer.BlockCopy(data, offset, audioSamples, 0, audioDataSize); } else { ConvaiLogger.Warn($"⚠️ Packet too small: expected {offset + audioDataSize} bytes, got {data.Length}", ConvaiLogger.LogCategory.Character); } } return new AudioPacketData { magicNumber = magic, sequence = sequence, sampleCount = sampleCount, audioSamples = audioSamples }; } catch (Exception ex) { ConvaiLogger.Error($"Error parsing audio packet: {ex.Message}", ConvaiLogger.LogCategory.Character); return null; } } private void BufferAudioPacket(int sequence, short[] samples) { if (samples == null || samples.Length == 0) return; lock (_audioBufferLock) { if (sequence < _nextSequenceToWrite) { return; // old/duplicate packet } if (sequence == _nextSequenceToWrite) { _receivedSamples.AddRange(samples); _nextSequenceToWrite++; // Flush any contiguous pending packets while (_pendingPackets.TryGetValue(_nextSequenceToWrite, out var nextSamples)) { _receivedSamples.AddRange(nextSamples); _pendingPackets.Remove(_nextSequenceToWrite); _nextSequenceToWrite++; } } else { // Store for later when gap is filled _pendingPackets[sequence] = samples; } } } private void TrySaveReceivedAudioAsync() { if (_saveInProgress) return; short[] dataToSave; DateTime sessionStart; lock (_audioBufferLock) { if (_receivedSamples == null || _receivedSamples.Count == 0) { if (enableDebugLogging) ConvaiLogger.Info("No received audio to save.", ConvaiLogger.LogCategory.Character); return; } dataToSave = _receivedSamples.ToArray(); _receivedSamples.Clear(); _pendingPackets.Clear(); sessionStart = _sessionStartTime; } _saveInProgress = true; Task.Run(() => { try { string timestamp = sessionStart.ToLocalTime().ToString("yyyyMMdd_HHmmss"); string fileName = $"{outputFilePrefix}_{timestamp}.wav"; string dir = _persistentDataPath; string path = Path.Combine(dir, fileName); WriteWav(path, dataToSave, receivedSampleRate, 1); ConvaiLogger.Info($"Saved received audio to: {path}", ConvaiLogger.LogCategory.Character); } catch (Exception ex) { ConvaiLogger.Error($"Failed to save received audio: {ex.Message}", ConvaiLogger.LogCategory.Character); } finally { _saveInProgress = false; } }); } private void WriteWav(string path, short[] samples, int sampleRate, int channels) { using (var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None)) using (var writer = new BinaryWriter(fs)) { int bitsPerSample = 16; int byteRate = sampleRate * channels * (bitsPerSample / 8); int blockAlign = channels * (bitsPerSample / 8); int dataSize = samples.Length * (bitsPerSample / 8); int fileSize = 44 - 8 + dataSize; // RIFF header writer.Write(System.Text.Encoding.ASCII.GetBytes("RIFF")); writer.Write(fileSize); writer.Write(System.Text.Encoding.ASCII.GetBytes("WAVE")); // fmt chunk writer.Write(System.Text.Encoding.ASCII.GetBytes("fmt ")); writer.Write(16); // Subchunk1Size for PCM writer.Write((short)1); // AudioFormat = PCM writer.Write((short)channels); // NumChannels writer.Write(sampleRate); // SampleRate writer.Write(byteRate); // ByteRate writer.Write((short)blockAlign); // BlockAlign writer.Write((short)bitsPerSample); // BitsPerSample // data chunk writer.Write(System.Text.Encoding.ASCII.GetBytes("data")); writer.Write(dataSize); for (int i = 0; i < samples.Length; i++) { writer.Write(samples[i]); } } } // Event handler for when NPC becomes active private void HandleActiveNPCChanged(ConvaiNPC newActiveNPC) { if (useActiveNPC && newActiveNPC != null) { targetNPC = newActiveNPC; ConvaiLogger.Info($"UDP Audio Receiver V2 updated target NPC to: {targetNPC.characterName}", ConvaiLogger.LogCategory.Character); } } // Public properties for debugging public bool IsListening => _isListening; public bool IsReceivingAudio => _isReceivingAudio; public ConvaiNPC TargetNPC => targetNPC; // Debug methods public void ShowNetworkStatus() { ConvaiLogger.Info($"=== Audio Receiver V2 Status ===", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Listening: {_isListening} on port {listenPort}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Receiving Audio: {_isReceivingAudio}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Target NPC: {(targetNPC?.characterName ?? "None")}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Expected Sequence: {_expectedSequence}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Last Packet Time: {_lastPacketTime}", ConvaiLogger.LogCategory.Character); } } }