using System; using System.Net; using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; using Convai.Scripts.Runtime.LoggerSystem; using Convai.Scripts.Runtime.UI; using UnityEngine; namespace Convai.Scripts.Runtime.Multiplayer { /// /// Simplified version of UDP Audio Sender that avoids complex chunking /// This version sends smaller, more frequent packets to avoid array bounds issues /// public class ConvaiSimpleUDPAudioSender : MonoBehaviour { [Header("Network Settings")] [SerializeField] private string targetIP = "127.0.0.1"; [SerializeField] private int targetPort = 12345; [SerializeField] private bool useGlobalNetworkConfig = true; [SerializeField] private NetworkConfig networkConfigAsset; [Header("Audio Settings")] [SerializeField] private int recordingFrequency = 16000; [SerializeField] private int recordingLength = 10; [SerializeField] private int samplesPerPacket = 1024; // Number of audio samples per packet (not bytes) [Header("UI")] [SerializeField] private KeyCode talkKey = KeyCode.T; [SerializeField] private bool useHoldToTalk = true; [SerializeField] private KeyCode controllerTalkButton = KeyCode.JoystickButton0; // A button on most controllers [Header("Debug")] [SerializeField] private bool enableDebugLogging = true; [SerializeField] private KeyCode testConnectionKey = KeyCode.C; private UdpClient _udpClient; private IPEndPoint _targetEndPoint; private AudioClip _audioClip; private bool _isRecording = false; private CancellationTokenSource _cancellationTokenSource; private int _lastMicrophonePosition = 0; private float[] _audioBuffer; private string _selectedMicrophone; private int _packetSequence = 0; public event Action OnRecordingStateChanged; private void Start() { // Apply global config if enabled if (useGlobalNetworkConfig) { var cfg = networkConfigAsset != null ? networkConfigAsset : NetworkConfig.Instance; if (cfg != null) { targetIP = cfg.ipAddress; targetPort = cfg.multiplayerAudioPort; } } InitializeNetwork(); InitializeAudio(); _cancellationTokenSource = new CancellationTokenSource(); } private void Update() { HandleInput(); } private void OnDestroy() { StopRecording(); _cancellationTokenSource?.Cancel(); _cancellationTokenSource?.Dispose(); _udpClient?.Close(); } private void InitializeNetwork() { try { _udpClient = new UdpClient(); _targetEndPoint = new IPEndPoint(IPAddress.Parse(targetIP), targetPort); ConvaiLogger.Info($"Simple UDP Audio Sender initialized. Target: {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character); } catch (Exception ex) { ConvaiLogger.Error($"Failed to initialize UDP client: {ex.Message}", ConvaiLogger.LogCategory.Character); } } private void InitializeAudio() { try { // Try to get selected microphone from Convai's UI system _selectedMicrophone = MicrophoneManager.Instance?.SelectedMicrophoneName; } catch (Exception ex) { // If UISaveLoadSystem / MicrophoneManager isn't initialized yet, fall back to first available device ConvaiLogger.Warn($"MicrophoneManager not available; falling back to default device. {ex.Message}", ConvaiLogger.LogCategory.Character); _selectedMicrophone = null; } // Fallback: pick the first available microphone if none selected or manager unavailable if (string.IsNullOrEmpty(_selectedMicrophone)) { var devices = Microphone.devices; if (devices != null && devices.Length > 0) { _selectedMicrophone = devices[0]; ConvaiLogger.Info($"Using default microphone: {_selectedMicrophone}", ConvaiLogger.LogCategory.Character); } } _audioBuffer = new float[recordingFrequency * recordingLength]; if (string.IsNullOrEmpty(_selectedMicrophone)) { ConvaiLogger.Error("No microphone available or selected for UDP audio sender", ConvaiLogger.LogCategory.Character); } } private void HandleInput() { // Handle talk key if (useHoldToTalk) { if ((Input.GetKeyDown(talkKey) || Input.GetKeyDown(controllerTalkButton)) && !_isRecording) { StartRecording(); } else if ((Input.GetKeyUp(talkKey) || Input.GetKeyUp(controllerTalkButton)) && _isRecording) { StopRecording(); } } else { if (Input.GetKeyDown(talkKey) || Input.GetKeyDown(controllerTalkButton)) { if (_isRecording) StopRecording(); else StartRecording(); } } // Handle test connection key if (Input.GetKeyDown(testConnectionKey)) { TestConnection(); } } public void StartRecording() { if (_isRecording || string.IsNullOrEmpty(_selectedMicrophone)) return; try { // Use looping clip so we can handle ring-buffer wrap-around reliably _audioClip = Microphone.Start(_selectedMicrophone, true, recordingLength, recordingFrequency); _isRecording = true; _lastMicrophonePosition = 0; _packetSequence = 0; ConvaiLogger.Info("Started recording for UDP transmission (Simple)", ConvaiLogger.LogCategory.Character); OnRecordingStateChanged?.Invoke(true); // Start continuous audio processing _ = ProcessAudioContinuously(_cancellationTokenSource.Token); } catch (Exception ex) { ConvaiLogger.Error($"Failed to start recording: {ex.Message}", ConvaiLogger.LogCategory.Character); } } public void StopRecording() { if (!_isRecording) return; try { Microphone.End(_selectedMicrophone); _isRecording = false; ConvaiLogger.Info("Stopped recording for UDP transmission (Simple)", ConvaiLogger.LogCategory.Character); OnRecordingStateChanged?.Invoke(false); // Send end-of-recording signal SendEndOfRecordingSignal(); } catch (Exception ex) { ConvaiLogger.Error($"Failed to stop recording: {ex.Message}", ConvaiLogger.LogCategory.Character); } } private async Task ProcessAudioContinuously(CancellationToken cancellationToken) { while (_isRecording && !cancellationToken.IsCancellationRequested) { try { await Task.Delay(30, cancellationToken); // Process ~33 times/sec for better capture granularity if (_audioClip == null || !Microphone.IsRecording(_selectedMicrophone)) break; int currentMicrophonePosition = Microphone.GetPosition(_selectedMicrophone); int clipSamples = _audioClip.samples; if (clipSamples <= 0) continue; // Compute how many new samples are available, accounting for wrap-around int audioDataLength = currentMicrophonePosition - _lastMicrophonePosition; bool wrapped = false; if (audioDataLength < 0) { audioDataLength += clipSamples; wrapped = true; } if (audioDataLength <= 0) continue; if (!wrapped) { // Contiguous region, read exactly the new samples int segmentLen = audioDataLength; var segment = new float[segmentLen]; _audioClip.GetData(segment, _lastMicrophonePosition); await SendAudioDataInChunks(segment, segmentLen); } else { // Wrapped: send tail [lastPos .. end) then head [0 .. currentPos) int firstLen = clipSamples - _lastMicrophonePosition; if (firstLen > 0) { var firstSeg = new float[firstLen]; _audioClip.GetData(firstSeg, _lastMicrophonePosition); await SendAudioDataInChunks(firstSeg, firstLen); } int secondLen = currentMicrophonePosition; if (secondLen > 0) { var secondSeg = new float[secondLen]; _audioClip.GetData(secondSeg, 0); await SendAudioDataInChunks(secondSeg, secondLen); } } _lastMicrophonePosition = currentMicrophonePosition; } catch (Exception ex) when (!(ex is OperationCanceledException)) { ConvaiLogger.Error($"Error in audio processing: {ex.Message}", ConvaiLogger.LogCategory.Character); break; } } } private async Task SendAudioDataInChunks(float[] audioData, int totalSamples) { int processedSamples = 0; while (processedSamples < totalSamples) { try { int remainingSamples = totalSamples - processedSamples; int currentChunkSamples = Mathf.Min(samplesPerPacket, remainingSamples); // Create a simple packet structure byte[] packet = CreateSimpleAudioPacket(audioData, processedSamples, currentChunkSamples); // Send the packet await _udpClient.SendAsync(packet, packet.Length, _targetEndPoint); if (enableDebugLogging && _packetSequence % 10 == 0) // Log every 10th packet { ConvaiLogger.DebugLog($"Sent packet {_packetSequence} with {currentChunkSamples} samples", ConvaiLogger.LogCategory.Character); } processedSamples += currentChunkSamples; _packetSequence++; // Small delay to avoid overwhelming the network await Task.Delay(10); } catch (Exception ex) { ConvaiLogger.Error($"Failed to send audio chunk: {ex.Message}", ConvaiLogger.LogCategory.Character); break; } } } private byte[] CreateSimpleAudioPacket(float[] audioData, int startIndex, int sampleCount) { // Simple packet structure: // 4 bytes: Magic number (0xC0A1) // 4 bytes: Packet sequence number // 4 bytes: Sample count in this packet // 4 bytes: Start position in stream // 1 byte: Flags (0 = normal audio, 1 = end of recording) // N bytes: Audio data (converted to shorts) int headerSize = 17; // 4 + 4 + 4 + 4 + 1 int audioDataSize = sampleCount * sizeof(short); byte[] packet = new byte[headerSize + audioDataSize]; int offset = 0; // Magic number BitConverter.GetBytes((uint)0xC0A1).CopyTo(packet, offset); offset += 4; // Packet sequence BitConverter.GetBytes(_packetSequence).CopyTo(packet, offset); offset += 4; // Sample count BitConverter.GetBytes(sampleCount).CopyTo(packet, offset); offset += 4; // Start position BitConverter.GetBytes(_lastMicrophonePosition + startIndex).CopyTo(packet, offset); offset += 4; // Flags (0 for normal audio) packet[offset] = 0; offset += 1; // Convert audio samples to bytes (same as Convai approach) for (int i = 0; i < sampleCount; i++) { float sample = audioData[startIndex + i]; short shortSample = (short)(sample * short.MaxValue); byte[] shortBytes = BitConverter.GetBytes(shortSample); packet[offset] = shortBytes[0]; packet[offset + 1] = shortBytes[1]; offset += 2; } return packet; } private void SendEndOfRecordingSignal() { try { // Create end packet byte[] packet = new byte[17]; // Header only, no audio data int offset = 0; // Magic number BitConverter.GetBytes((uint)0xC0A1).CopyTo(packet, offset); offset += 4; // Packet sequence BitConverter.GetBytes(_packetSequence).CopyTo(packet, offset); offset += 4; // Sample count (0 for end signal) BitConverter.GetBytes(0).CopyTo(packet, offset); offset += 4; // Start position BitConverter.GetBytes(_lastMicrophonePosition).CopyTo(packet, offset); offset += 4; // Flags (1 for end of recording) packet[offset] = 1; _udpClient.SendAsync(packet, packet.Length, _targetEndPoint); } catch (Exception ex) { ConvaiLogger.Error($"Failed to send end signal: {ex.Message}", ConvaiLogger.LogCategory.Character); } } // Public methods for external control public void SetTargetEndpoint(string ip, int port) { targetIP = ip; targetPort = port; _targetEndPoint = new IPEndPoint(IPAddress.Parse(ip), port); } public bool IsRecording => _isRecording; // Debug and testing methods public async void TestConnection() { if (_udpClient == null) { ConvaiLogger.Error("UDP client not initialized", ConvaiLogger.LogCategory.Character); return; } try { ConvaiLogger.Info($"Testing connection to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character); // Send a simple test packet string testMessage = "CONVAI_TEST_CONNECTION"; byte[] testData = System.Text.Encoding.UTF8.GetBytes(testMessage); await _udpClient.SendAsync(testData, testData.Length, _targetEndPoint); ConvaiLogger.Info("Test packet sent successfully", ConvaiLogger.LogCategory.Character); } catch (Exception ex) { ConvaiLogger.Error($"Connection test failed: {ex.Message}", ConvaiLogger.LogCategory.Character); } } public void ShowNetworkStatus() { ConvaiLogger.Info($"=== Network Status ===", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Target: {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"UDP Client: {(_udpClient != null ? "Initialized" : "Not initialized")}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Recording: {_isRecording}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Microphone: {_selectedMicrophone}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Packets sent: {_packetSequence}", ConvaiLogger.LogCategory.Character); if (_udpClient?.Client?.LocalEndPoint != null) { ConvaiLogger.Info($"Local endpoint: {_udpClient.Client.LocalEndPoint}", ConvaiLogger.LogCategory.Character); } } } }