using System; using System.Net; using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; using Convai.Scripts.Runtime.LoggerSystem; using Convai.Scripts.Runtime.UI; using UnityEngine; namespace Convai.Scripts.Runtime.Multiplayer { /// /// Simplified version of UDP Audio Sender that avoids complex chunking /// This version sends smaller, more frequent packets to avoid array bounds issues /// public class ConvaiSimpleUDPAudioSender : MonoBehaviour { [Header("Network Settings")] [SerializeField] private string targetIP = "127.0.0.1"; [SerializeField] private int targetPort = 12345; [Header("Audio Settings")] [SerializeField] private int recordingFrequency = 16000; [SerializeField] private int recordingLength = 10; [SerializeField] private int samplesPerPacket = 1024; // Number of audio samples per packet (not bytes) [Header("UI")] [SerializeField] private KeyCode talkKey = KeyCode.T; [SerializeField] private bool useHoldToTalk = true; [Header("Debug")] [SerializeField] private bool enableDebugLogging = true; [SerializeField] private KeyCode testConnectionKey = KeyCode.C; private UdpClient _udpClient; private IPEndPoint _targetEndPoint; private AudioClip _audioClip; private bool _isRecording = false; private CancellationTokenSource _cancellationTokenSource; private int _lastMicrophonePosition = 0; private float[] _audioBuffer; private string _selectedMicrophone; private int _packetSequence = 0; public event Action OnRecordingStateChanged; private void Start() { InitializeNetwork(); InitializeAudio(); _cancellationTokenSource = new CancellationTokenSource(); } private void Update() { HandleInput(); } private void OnDestroy() { StopRecording(); _cancellationTokenSource?.Cancel(); _cancellationTokenSource?.Dispose(); _udpClient?.Close(); } private void InitializeNetwork() { try { _udpClient = new UdpClient(); _targetEndPoint = new IPEndPoint(IPAddress.Parse(targetIP), targetPort); ConvaiLogger.Info($"Simple UDP Audio Sender initialized. Target: {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character); } catch (Exception ex) { ConvaiLogger.Error($"Failed to initialize UDP client: {ex.Message}", ConvaiLogger.LogCategory.Character); } } private void InitializeAudio() { _selectedMicrophone = MicrophoneManager.Instance.SelectedMicrophoneName; _audioBuffer = new float[recordingFrequency * recordingLength]; if (string.IsNullOrEmpty(_selectedMicrophone)) { ConvaiLogger.Error("No microphone selected for UDP audio sender", ConvaiLogger.LogCategory.Character); } } private void HandleInput() { // Handle talk key if (useHoldToTalk) { if (Input.GetKeyDown(talkKey) && !_isRecording) { StartRecording(); } else if (Input.GetKeyUp(talkKey) && _isRecording) { StopRecording(); } } else { if (Input.GetKeyDown(talkKey)) { if (_isRecording) StopRecording(); else StartRecording(); } } // Handle test connection key if (Input.GetKeyDown(testConnectionKey)) { TestConnection(); } } public void StartRecording() { if (_isRecording || string.IsNullOrEmpty(_selectedMicrophone)) return; try { _audioClip = Microphone.Start(_selectedMicrophone, false, recordingLength, recordingFrequency); _isRecording = true; _lastMicrophonePosition = 0; _packetSequence = 0; ConvaiLogger.Info("Started recording for UDP transmission (Simple)", ConvaiLogger.LogCategory.Character); OnRecordingStateChanged?.Invoke(true); // Start continuous audio processing _ = ProcessAudioContinuously(_cancellationTokenSource.Token); } catch (Exception ex) { ConvaiLogger.Error($"Failed to start recording: {ex.Message}", ConvaiLogger.LogCategory.Character); } } public void StopRecording() { if (!_isRecording) return; try { Microphone.End(_selectedMicrophone); _isRecording = false; ConvaiLogger.Info("Stopped recording for UDP transmission (Simple)", ConvaiLogger.LogCategory.Character); OnRecordingStateChanged?.Invoke(false); // Send end-of-recording signal SendEndOfRecordingSignal(); } catch (Exception ex) { ConvaiLogger.Error($"Failed to stop recording: {ex.Message}", ConvaiLogger.LogCategory.Character); } } private async Task ProcessAudioContinuously(CancellationToken cancellationToken) { while (_isRecording && !cancellationToken.IsCancellationRequested) { try { await Task.Delay(100, cancellationToken); // Process every 100ms if (_audioClip == null || !Microphone.IsRecording(_selectedMicrophone)) break; int currentMicrophonePosition = Microphone.GetPosition(_selectedMicrophone); int audioDataLength = currentMicrophonePosition - _lastMicrophonePosition; if (audioDataLength > 0) { // Get audio data from the microphone clip _audioClip.GetData(_audioBuffer, _lastMicrophonePosition); // Send data in smaller chunks to avoid array bounds issues await SendAudioDataInChunks(_audioBuffer, audioDataLength); _lastMicrophonePosition = currentMicrophonePosition; } } catch (Exception ex) when (!(ex is OperationCanceledException)) { ConvaiLogger.Error($"Error in audio processing: {ex.Message}", ConvaiLogger.LogCategory.Character); break; } } } private async Task SendAudioDataInChunks(float[] audioData, int totalSamples) { int processedSamples = 0; while (processedSamples < totalSamples) { try { int remainingSamples = totalSamples - processedSamples; int currentChunkSamples = Mathf.Min(samplesPerPacket, remainingSamples); // Create a simple packet structure byte[] packet = CreateSimpleAudioPacket(audioData, processedSamples, currentChunkSamples); // Send the packet await _udpClient.SendAsync(packet, packet.Length, _targetEndPoint); if (enableDebugLogging && _packetSequence % 10 == 0) // Log every 10th packet { ConvaiLogger.DebugLog($"Sent packet {_packetSequence} with {currentChunkSamples} samples", ConvaiLogger.LogCategory.Character); } processedSamples += currentChunkSamples; _packetSequence++; // Small delay to avoid overwhelming the network await Task.Delay(10); } catch (Exception ex) { ConvaiLogger.Error($"Failed to send audio chunk: {ex.Message}", ConvaiLogger.LogCategory.Character); break; } } } private byte[] CreateSimpleAudioPacket(float[] audioData, int startIndex, int sampleCount) { // Simple packet structure: // 4 bytes: Magic number (0xC0A1) // 4 bytes: Packet sequence number // 4 bytes: Sample count in this packet // 4 bytes: Start position in stream // 1 byte: Flags (0 = normal audio, 1 = end of recording) // N bytes: Audio data (converted to shorts) int headerSize = 17; // 4 + 4 + 4 + 4 + 1 int audioDataSize = sampleCount * sizeof(short); byte[] packet = new byte[headerSize + audioDataSize]; int offset = 0; // Magic number BitConverter.GetBytes((uint)0xC0A1).CopyTo(packet, offset); offset += 4; // Packet sequence BitConverter.GetBytes(_packetSequence).CopyTo(packet, offset); offset += 4; // Sample count BitConverter.GetBytes(sampleCount).CopyTo(packet, offset); offset += 4; // Start position BitConverter.GetBytes(_lastMicrophonePosition + startIndex).CopyTo(packet, offset); offset += 4; // Flags (0 for normal audio) packet[offset] = 0; offset += 1; // Convert audio samples to bytes (same as Convai approach) for (int i = 0; i < sampleCount; i++) { float sample = audioData[startIndex + i]; short shortSample = (short)(sample * short.MaxValue); byte[] shortBytes = BitConverter.GetBytes(shortSample); packet[offset] = shortBytes[0]; packet[offset + 1] = shortBytes[1]; offset += 2; } return packet; } private void SendEndOfRecordingSignal() { try { // Create end packet byte[] packet = new byte[17]; // Header only, no audio data int offset = 0; // Magic number BitConverter.GetBytes((uint)0xC0A1).CopyTo(packet, offset); offset += 4; // Packet sequence BitConverter.GetBytes(_packetSequence).CopyTo(packet, offset); offset += 4; // Sample count (0 for end signal) BitConverter.GetBytes(0).CopyTo(packet, offset); offset += 4; // Start position BitConverter.GetBytes(_lastMicrophonePosition).CopyTo(packet, offset); offset += 4; // Flags (1 for end of recording) packet[offset] = 1; _udpClient.SendAsync(packet, packet.Length, _targetEndPoint); } catch (Exception ex) { ConvaiLogger.Error($"Failed to send end signal: {ex.Message}", ConvaiLogger.LogCategory.Character); } } // Public methods for external control public void SetTargetEndpoint(string ip, int port) { targetIP = ip; targetPort = port; _targetEndPoint = new IPEndPoint(IPAddress.Parse(ip), port); } public bool IsRecording => _isRecording; // Debug and testing methods public async void TestConnection() { if (_udpClient == null) { ConvaiLogger.Error("UDP client not initialized", ConvaiLogger.LogCategory.Character); return; } try { ConvaiLogger.Info($"Testing connection to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character); // Send a simple test packet string testMessage = "CONVAI_TEST_CONNECTION"; byte[] testData = System.Text.Encoding.UTF8.GetBytes(testMessage); await _udpClient.SendAsync(testData, testData.Length, _targetEndPoint); ConvaiLogger.Info("Test packet sent successfully", ConvaiLogger.LogCategory.Character); } catch (Exception ex) { ConvaiLogger.Error($"Connection test failed: {ex.Message}", ConvaiLogger.LogCategory.Character); } } public void ShowNetworkStatus() { ConvaiLogger.Info($"=== Network Status ===", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Target: {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"UDP Client: {(_udpClient != null ? "Initialized" : "Not initialized")}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Recording: {_isRecording}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Microphone: {_selectedMicrophone}", ConvaiLogger.LogCategory.Character); ConvaiLogger.Info($"Packets sent: {_packetSequence}", ConvaiLogger.LogCategory.Character); if (_udpClient?.Client?.LocalEndPoint != null) { ConvaiLogger.Info($"Local endpoint: {_udpClient.Client.LocalEndPoint}", ConvaiLogger.LogCategory.Character); } } } }