using System; using System.Collections.Generic; using System.Net; using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; using Convai.Scripts.Runtime.Core; using Convai.Scripts.Runtime.LoggerSystem; using Convai.Scripts.Runtime.Utils; using UnityEngine; namespace Convai.Scripts.Runtime.Multiplayer { /// /// Simple UDP Audio Receiver - Simulates microphone input by triggering normal Convai flow /// This approach is much simpler and more reliable than trying to replicate gRPC calls /// public class ConvaiSimpleUDPAudioReceiver : MonoBehaviour { [Header("Network Configuration")] [SerializeField] private int listenPort = 12345; [SerializeField] private bool enableDebugLogging = true; [SerializeField] private bool useGlobalNetworkConfig = true; [SerializeField] private NetworkConfig networkConfigAsset; [Header("NPC Target")] [SerializeField] private bool useActiveNPC = true; [SerializeField] private ConvaiNPC targetNPC; // Events public Action OnAudioReceiving; // Network components private UdpClient _udpListener; private IPEndPoint _remoteEndPoint; private bool _isListening = false; private CancellationTokenSource _cancellationTokenSource; // Audio state tracking private bool _isReceivingAudio = false; private int _expectedSequence = 0; private const uint MAGIC_NUMBER = 0xC0A1; // Simple magic number for packet validation // Timing for auto-stop private float _lastPacketTime; private const float AUTO_STOP_DELAY = 1.0f; // Stop listening after 1 second of no packets // Packet structure (matching ConvaiSimpleUDPAudioSender) private struct AudioPacketData { public uint magicNumber; public int sequence; public int sampleCount; public int microphonePosition; public bool isEndSignal; public short[] audioSamples; public long timestamp; } private void Start() { _cancellationTokenSource = new CancellationTokenSource(); // Apply global config if enabled if (useGlobalNetworkConfig) { var cfg = networkConfigAsset != null ? networkConfigAsset : NetworkConfig.Instance; if (cfg != null) { listenPort = cfg.multiplayerAudioPort; } } InitializeNetwork(); InitializeConvai(); // Subscribe to NPC manager events to handle late NPC activation if (ConvaiNPCManager.Instance != null) { ConvaiNPCManager.Instance.OnActiveNPCChanged += HandleActiveNPCChanged; } } private void OnEnable() { // When re-enabled, ensure listener is running if (_cancellationTokenSource == null) { _cancellationTokenSource = new CancellationTokenSource(); } StartListening(); } private void OnDestroy() { // Unsubscribe from events if (ConvaiNPCManager.Instance != null) { ConvaiNPCManager.Instance.OnActiveNPCChanged -= HandleActiveNPCChanged; } StopListening(); _cancellationTokenSource?.Cancel(); _cancellationTokenSource?.Dispose(); } private void OnDisable() { // Free the UDP port when this NPC gets disabled StopListening(); } private void Update() { // Auto-stop listening if no packets received for a while if (_isReceivingAudio && Time.time - _lastPacketTime > AUTO_STOP_DELAY) { StopTalkingSimulation(); } } private void InitializeNetwork() { try { StartListening(); } catch (Exception ex) { ConvaiLogger.Error($"Failed to initialize UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character); } } private void InitializeConvai() { // Prefer local ConvaiNPC on the same GameObject, then fall back to active NPC var localNPC = GetComponent(); if (localNPC != null) { targetNPC = localNPC; } else if (useActiveNPC) { targetNPC = ConvaiNPCManager.Instance?.GetActiveConvaiNPC(); } if (targetNPC == null) { ConvaiLogger.Warn("No target NPC found yet, will wait for NPC to become active", ConvaiLogger.LogCategory.Character); } else { ConvaiLogger.Info($"UDP Audio Receiver V2 initialized with NPC: {targetNPC.characterName}", ConvaiLogger.LogCategory.Character); } } public void StartListening() { if (_isListening || _cancellationTokenSource == null) return; try { _udpListener = new UdpClient(listenPort); _isListening = true; ConvaiLogger.Info($"Simple UDP Audio Receiver V2 listening on port {listenPort}", ConvaiLogger.LogCategory.Character); // Start listening for incoming packets _ = ListenForAudioPackets(_cancellationTokenSource.Token); } catch (Exception ex) { ConvaiLogger.Error($"Failed to start UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Error($"Stack trace: {ex.StackTrace}", ConvaiLogger.LogCategory.Character); } } public void StopListening() { if (!_isListening) return; _isListening = false; _udpListener?.Close(); _udpListener?.Dispose(); _udpListener = null; // Stop any ongoing simulation StopTalkingSimulation(); ConvaiLogger.Info("Stopped UDP Audio Receiver V2", ConvaiLogger.LogCategory.Character); } private async Task ListenForAudioPackets(CancellationToken cancellationToken) { try { while (_isListening && !cancellationToken.IsCancellationRequested) { var result = await _udpListener.ReceiveAsync(); _remoteEndPoint = result.RemoteEndPoint; await ProcessReceivedPacket(result.Buffer, result.RemoteEndPoint); } } catch (ObjectDisposedException) { // Normal when stopping } catch (Exception ex) { ConvaiLogger.Error($"Error in UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character); } } private async Task ProcessReceivedPacket(byte[] data, IPEndPoint sender) { try { var packetData = ParseSimpleAudioPacket(data); if (packetData.HasValue) { var packet = packetData.Value; _lastPacketTime = Time.time; if (enableDebugLogging) { if (packet.isEndSignal) ConvaiLogger.DebugLog($"Received end signal from {sender}", ConvaiLogger.LogCategory.Character); else ConvaiLogger.DebugLog($"Received audio packet {packet.sequence} with {packet.sampleCount} samples", ConvaiLogger.LogCategory.Character); } if (packet.isEndSignal) { StopTalkingSimulation(); OnAudioReceiving?.Invoke(false); } else { // If this is the first packet, start the talking simulation if (packet.sequence == 0 && !_isReceivingAudio) { StartTalkingSimulation(); } OnAudioReceiving?.Invoke(true); } } else { // Not our audio packet format, might be a test message string message = System.Text.Encoding.UTF8.GetString(data); if (enableDebugLogging) ConvaiLogger.Info($"Received test message from {sender}: {message}", ConvaiLogger.LogCategory.Character); } } catch (Exception ex) { ConvaiLogger.Error($"Error processing received packet: {ex.Message}", ConvaiLogger.LogCategory.Character); } } private void StartTalkingSimulation() { if (_isReceivingAudio) return; MainThreadDispatcher.Instance.RunOnMainThread(() => { // Update target NPC if using active NPC if (useActiveNPC) { targetNPC = ConvaiNPCManager.Instance?.GetActiveConvaiNPC(); } if (targetNPC == null) { ConvaiLogger.Warn("No target NPC available for audio simulation", ConvaiLogger.LogCategory.Character); return; } _isReceivingAudio = true; _expectedSequence = 0; // This is the KEY! Simulate a talk key press to trigger normal Convai flow ConvaiInputManager.Instance.talkKeyInteract?.Invoke(true); ConvaiLogger.Info($"🎤 Started talking simulation for {targetNPC.characterName} (remote player audio)", ConvaiLogger.LogCategory.Character); }); } private void StopTalkingSimulation() { if (!_isReceivingAudio) return; MainThreadDispatcher.Instance.RunOnMainThread(() => { _isReceivingAudio = false; // Simulate talk key release to stop recording ConvaiInputManager.Instance.talkKeyInteract?.Invoke(false); ConvaiLogger.Info($"🎤 Stopped talking simulation for {targetNPC?.characterName ?? "NPC"} (remote player audio)", ConvaiLogger.LogCategory.Character); }); } private AudioPacketData? ParseSimpleAudioPacket(byte[] data) { if (data.Length < 24) // Minimum header size return null; try { int offset = 0; // Read magic number uint magic = BitConverter.ToUInt32(data, offset); offset += 4; if (magic != MAGIC_NUMBER) return null; // Read header int sequence = BitConverter.ToInt32(data, offset); offset += 4; int sampleCount = BitConverter.ToInt32(data, offset); offset += 4; int microphonePosition = BitConverter.ToInt32(data, offset); offset += 4; bool isEndSignal = BitConverter.ToBoolean(data, offset); offset += 1; // Skip padding offset += 3; long timestamp = BitConverter.ToInt64(data, offset); offset += 8; // Read audio data short[] audioSamples = null; if (!isEndSignal && sampleCount > 0) { int audioDataSize = sampleCount * sizeof(short); if (data.Length >= offset + audioDataSize) { audioSamples = new short[sampleCount]; Buffer.BlockCopy(data, offset, audioSamples, 0, audioDataSize); } } return new AudioPacketData { magicNumber = magic, sequence = sequence, sampleCount = sampleCount, microphonePosition = microphonePosition, isEndSignal = isEndSignal, audioSamples = audioSamples, timestamp = timestamp }; } catch (Exception ex) { ConvaiLogger.Error($"Error parsing audio packet: {ex.Message}", ConvaiLogger.LogCategory.Character); return null; } } // Event handler for when NPC becomes active private void HandleActiveNPCChanged(ConvaiNPC newActiveNPC) { if (useActiveNPC && newActiveNPC != null) { targetNPC = newActiveNPC; ConvaiLogger.Info($"UDP Audio Receiver V2 updated target NPC to: {targetNPC.characterName}", ConvaiLogger.LogCategory.Character); } } // Public properties for debugging public bool IsListening => _isListening; public bool IsReceivingAudio => _isReceivingAudio; public ConvaiNPC TargetNPC => targetNPC; // Debug methods public void ShowNetworkStatus() { ConvaiLogger.Info($"=== Audio Receiver V2 Status ===", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Listening: {_isListening} on port {listenPort}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Receiving Audio: {_isReceivingAudio}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Target NPC: {(targetNPC?.characterName ?? "None")}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Expected Sequence: {_expectedSequence}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Last Packet Time: {_lastPacketTime}", ConvaiLogger.LogCategory.Character); } } }