using System;
using System.Net;
using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
using Convai.Scripts.Runtime.LoggerSystem;
using Convai.Scripts.Runtime.UI;
using UnityEngine;
namespace Convai.Scripts.Runtime.Multiplayer
{
///
/// Simplified version of UDP Audio Sender that avoids complex chunking
/// This version sends smaller, more frequent packets to avoid array bounds issues
///
public class ConvaiSimpleUDPAudioSender : MonoBehaviour
{
[Header("Network Settings")]
[SerializeField] private string targetIP = "127.0.0.1";
[SerializeField] private int targetPort = 12345;
[SerializeField] private bool useGlobalNetworkConfig = true;
[SerializeField] private NetworkConfig networkConfigAsset;
[Header("Audio Settings")]
[SerializeField] private int recordingFrequency = 16000;
[SerializeField] private int recordingLength = 10;
[SerializeField] private int samplesPerPacket = 1024; // Number of audio samples per packet (not bytes)
[Header("UI")]
[SerializeField] private KeyCode talkKey = KeyCode.T;
[SerializeField] private bool useHoldToTalk = true;
[SerializeField] private KeyCode controllerTalkButton = KeyCode.JoystickButton0; // A button on most controllers
[Header("Debug")]
[SerializeField] private bool enableDebugLogging = true;
[SerializeField] private KeyCode testConnectionKey = KeyCode.C;
private UdpClient _udpClient;
private IPEndPoint _targetEndPoint;
private AudioClip _audioClip;
private bool _isRecording = false;
private CancellationTokenSource _cancellationTokenSource;
private int _lastMicrophonePosition = 0;
private float[] _audioBuffer;
private string _selectedMicrophone;
private int _packetSequence = 0;
public event Action OnRecordingStateChanged;
private void Start()
{
// Apply global config if enabled
if (useGlobalNetworkConfig)
{
var cfg = networkConfigAsset != null ? networkConfigAsset : NetworkConfig.Instance;
if (cfg != null)
{
targetIP = cfg.ipAddress;
targetPort = cfg.multiplayerAudioPort;
}
}
InitializeNetwork();
InitializeAudio();
_cancellationTokenSource = new CancellationTokenSource();
}
private void Update()
{
HandleInput();
}
private void OnDestroy()
{
StopRecording();
_cancellationTokenSource?.Cancel();
_cancellationTokenSource?.Dispose();
_udpClient?.Close();
}
private void InitializeNetwork()
{
try
{
_udpClient = new UdpClient();
_targetEndPoint = new IPEndPoint(IPAddress.Parse(targetIP), targetPort);
ConvaiLogger.Info($"Simple UDP Audio Sender initialized. Target: {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
}
catch (Exception ex)
{
ConvaiLogger.Error($"Failed to initialize UDP client: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
private void InitializeAudio()
{
try
{
// Try to get selected microphone from Convai's UI system
_selectedMicrophone = MicrophoneManager.Instance?.SelectedMicrophoneName;
}
catch (Exception ex)
{
// If UISaveLoadSystem / MicrophoneManager isn't initialized yet, fall back to first available device
ConvaiLogger.Warn($"MicrophoneManager not available; falling back to default device. {ex.Message}", ConvaiLogger.LogCategory.Character);
_selectedMicrophone = null;
}
// Fallback: pick the first available microphone if none selected or manager unavailable
if (string.IsNullOrEmpty(_selectedMicrophone))
{
var devices = Microphone.devices;
if (devices != null && devices.Length > 0)
{
_selectedMicrophone = devices[0];
ConvaiLogger.Info($"Using default microphone: {_selectedMicrophone}", ConvaiLogger.LogCategory.Character);
}
}
_audioBuffer = new float[recordingFrequency * recordingLength];
if (string.IsNullOrEmpty(_selectedMicrophone))
{
ConvaiLogger.Error("No microphone available or selected for UDP audio sender", ConvaiLogger.LogCategory.Character);
}
}
private void HandleInput()
{
// Handle talk key
if (useHoldToTalk)
{
if ((Input.GetKeyDown(talkKey) || Input.GetKeyDown(controllerTalkButton)) && !_isRecording)
{
StartRecording();
}
else if ((Input.GetKeyUp(talkKey) || Input.GetKeyUp(controllerTalkButton)) && _isRecording)
{
StopRecording();
}
}
else
{
if (Input.GetKeyDown(talkKey) || Input.GetKeyDown(controllerTalkButton))
{
if (_isRecording)
StopRecording();
else
StartRecording();
}
}
// Handle test connection key
if (Input.GetKeyDown(testConnectionKey))
{
TestConnection();
}
}
public void StartRecording()
{
if (_isRecording || string.IsNullOrEmpty(_selectedMicrophone))
return;
try
{
// Use looping clip so we can handle ring-buffer wrap-around reliably
_audioClip = Microphone.Start(_selectedMicrophone, true, recordingLength, recordingFrequency);
_isRecording = true;
_lastMicrophonePosition = 0;
_packetSequence = 0;
ConvaiLogger.Info("Started recording for UDP transmission (Simple)", ConvaiLogger.LogCategory.Character);
OnRecordingStateChanged?.Invoke(true);
// Start continuous audio processing
_ = ProcessAudioContinuously(_cancellationTokenSource.Token);
}
catch (Exception ex)
{
ConvaiLogger.Error($"Failed to start recording: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
public void StopRecording()
{
if (!_isRecording)
return;
try
{
Microphone.End(_selectedMicrophone);
_isRecording = false;
ConvaiLogger.Info("Stopped recording for UDP transmission (Simple)", ConvaiLogger.LogCategory.Character);
OnRecordingStateChanged?.Invoke(false);
// Send end-of-recording signal
SendEndOfRecordingSignal();
}
catch (Exception ex)
{
ConvaiLogger.Error($"Failed to stop recording: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
private async Task ProcessAudioContinuously(CancellationToken cancellationToken)
{
while (_isRecording && !cancellationToken.IsCancellationRequested)
{
try
{
await Task.Delay(30, cancellationToken); // Process ~33 times/sec for better capture granularity
if (_audioClip == null || !Microphone.IsRecording(_selectedMicrophone))
break;
int currentMicrophonePosition = Microphone.GetPosition(_selectedMicrophone);
int clipSamples = _audioClip.samples;
if (clipSamples <= 0)
continue;
// Compute how many new samples are available, accounting for wrap-around
int audioDataLength = currentMicrophonePosition - _lastMicrophonePosition;
bool wrapped = false;
if (audioDataLength < 0)
{
audioDataLength += clipSamples;
wrapped = true;
}
if (audioDataLength <= 0)
continue;
if (!wrapped)
{
// Contiguous region, read exactly the new samples
int segmentLen = audioDataLength;
var segment = new float[segmentLen];
_audioClip.GetData(segment, _lastMicrophonePosition);
await SendAudioDataInChunks(segment, segmentLen);
}
else
{
// Wrapped: send tail [lastPos .. end) then head [0 .. currentPos)
int firstLen = clipSamples - _lastMicrophonePosition;
if (firstLen > 0)
{
var firstSeg = new float[firstLen];
_audioClip.GetData(firstSeg, _lastMicrophonePosition);
await SendAudioDataInChunks(firstSeg, firstLen);
}
int secondLen = currentMicrophonePosition;
if (secondLen > 0)
{
var secondSeg = new float[secondLen];
_audioClip.GetData(secondSeg, 0);
await SendAudioDataInChunks(secondSeg, secondLen);
}
}
_lastMicrophonePosition = currentMicrophonePosition;
}
catch (Exception ex) when (!(ex is OperationCanceledException))
{
ConvaiLogger.Error($"Error in audio processing: {ex.Message}", ConvaiLogger.LogCategory.Character);
break;
}
}
}
private async Task SendAudioDataInChunks(float[] audioData, int totalSamples)
{
int processedSamples = 0;
while (processedSamples < totalSamples)
{
try
{
int remainingSamples = totalSamples - processedSamples;
int currentChunkSamples = Mathf.Min(samplesPerPacket, remainingSamples);
// Create a simple packet structure
byte[] packet = CreateSimpleAudioPacket(audioData, processedSamples, currentChunkSamples);
// Send the packet
await _udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
if (enableDebugLogging && _packetSequence % 10 == 0) // Log every 10th packet
{
ConvaiLogger.DebugLog($"Sent packet {_packetSequence} with {currentChunkSamples} samples", ConvaiLogger.LogCategory.Character);
}
processedSamples += currentChunkSamples;
_packetSequence++;
// Small delay to avoid overwhelming the network
await Task.Delay(10);
}
catch (Exception ex)
{
ConvaiLogger.Error($"Failed to send audio chunk: {ex.Message}", ConvaiLogger.LogCategory.Character);
break;
}
}
}
private byte[] CreateSimpleAudioPacket(float[] audioData, int startIndex, int sampleCount)
{
// Simple packet structure:
// 4 bytes: Magic number (0xC0A1)
// 4 bytes: Packet sequence number
// 4 bytes: Sample count in this packet
// 4 bytes: Start position in stream
// 1 byte: Flags (0 = normal audio, 1 = end of recording)
// N bytes: Audio data (converted to shorts)
int headerSize = 17; // 4 + 4 + 4 + 4 + 1
int audioDataSize = sampleCount * sizeof(short);
byte[] packet = new byte[headerSize + audioDataSize];
int offset = 0;
// Magic number
BitConverter.GetBytes((uint)0xC0A1).CopyTo(packet, offset);
offset += 4;
// Packet sequence
BitConverter.GetBytes(_packetSequence).CopyTo(packet, offset);
offset += 4;
// Sample count
BitConverter.GetBytes(sampleCount).CopyTo(packet, offset);
offset += 4;
// Start position
BitConverter.GetBytes(_lastMicrophonePosition + startIndex).CopyTo(packet, offset);
offset += 4;
// Flags (0 for normal audio)
packet[offset] = 0;
offset += 1;
// Convert audio samples to bytes (same as Convai approach)
for (int i = 0; i < sampleCount; i++)
{
float sample = audioData[startIndex + i];
short shortSample = (short)(sample * short.MaxValue);
byte[] shortBytes = BitConverter.GetBytes(shortSample);
packet[offset] = shortBytes[0];
packet[offset + 1] = shortBytes[1];
offset += 2;
}
return packet;
}
private void SendEndOfRecordingSignal()
{
try
{
// Create end packet
byte[] packet = new byte[17]; // Header only, no audio data
int offset = 0;
// Magic number
BitConverter.GetBytes((uint)0xC0A1).CopyTo(packet, offset);
offset += 4;
// Packet sequence
BitConverter.GetBytes(_packetSequence).CopyTo(packet, offset);
offset += 4;
// Sample count (0 for end signal)
BitConverter.GetBytes(0).CopyTo(packet, offset);
offset += 4;
// Start position
BitConverter.GetBytes(_lastMicrophonePosition).CopyTo(packet, offset);
offset += 4;
// Flags (1 for end of recording)
packet[offset] = 1;
_udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
}
catch (Exception ex)
{
ConvaiLogger.Error($"Failed to send end signal: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
// Public methods for external control
public void SetTargetEndpoint(string ip, int port)
{
targetIP = ip;
targetPort = port;
_targetEndPoint = new IPEndPoint(IPAddress.Parse(ip), port);
}
public bool IsRecording => _isRecording;
// Debug and testing methods
public async void TestConnection()
{
if (_udpClient == null)
{
ConvaiLogger.Error("UDP client not initialized", ConvaiLogger.LogCategory.Character);
return;
}
try
{
ConvaiLogger.Info($"Testing connection to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
// Send a simple test packet
string testMessage = "CONVAI_TEST_CONNECTION";
byte[] testData = System.Text.Encoding.UTF8.GetBytes(testMessage);
await _udpClient.SendAsync(testData, testData.Length, _targetEndPoint);
ConvaiLogger.Info("Test packet sent successfully", ConvaiLogger.LogCategory.Character);
}
catch (Exception ex)
{
ConvaiLogger.Error($"Connection test failed: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
public void ShowNetworkStatus()
{
ConvaiLogger.Info($"=== Network Status ===", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"Target: {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"UDP Client: {(_udpClient != null ? "Initialized" : "Not initialized")}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"Recording: {_isRecording}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"Microphone: {_selectedMicrophone}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"Packets sent: {_packetSequence}", ConvaiLogger.LogCategory.Character);
if (_udpClient?.Client?.LocalEndPoint != null)
{
ConvaiLogger.Info($"Local endpoint: {_udpClient.Client.LocalEndPoint}", ConvaiLogger.LogCategory.Character);
}
}
}
}