447 lines
18 KiB
C#
447 lines
18 KiB
C#
using System;
|
|
using System.Net;
|
|
using System.Net.Sockets;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using Convai.Scripts.Runtime.LoggerSystem;
|
|
using Convai.Scripts.Runtime.UI;
|
|
using UnityEngine;
|
|
|
|
namespace Convai.Scripts.Runtime.Multiplayer
|
|
{
|
|
/// <summary>
|
|
/// Simplified version of UDP Audio Sender that avoids complex chunking
|
|
/// This version sends smaller, more frequent packets to avoid array bounds issues
|
|
/// </summary>
|
|
public class ConvaiSimpleUDPAudioSender : MonoBehaviour
|
|
{
|
|
[Header("Network Settings")]
|
|
[SerializeField] private string targetIP = "127.0.0.1";
|
|
[SerializeField] private int targetPort = 12345;
|
|
[SerializeField] private bool useGlobalNetworkConfig = true;
|
|
[SerializeField] private NetworkConfig networkConfigAsset;
|
|
|
|
[Header("Audio Settings")]
|
|
[SerializeField] private int recordingFrequency = 16000;
|
|
[SerializeField] private int recordingLength = 10;
|
|
[SerializeField] private int samplesPerPacket = 1024; // Number of audio samples per packet (not bytes)
|
|
|
|
[Header("UI")]
|
|
[SerializeField] private KeyCode talkKey = KeyCode.T;
|
|
[SerializeField] private bool useHoldToTalk = true;
|
|
[SerializeField] private KeyCode controllerTalkButton = KeyCode.JoystickButton0; // A button on most controllers
|
|
|
|
[Header("Debug")]
|
|
[SerializeField] private bool enableDebugLogging = true;
|
|
[SerializeField] private KeyCode testConnectionKey = KeyCode.C;
|
|
|
|
private UdpClient _udpClient;
|
|
private IPEndPoint _targetEndPoint;
|
|
private AudioClip _audioClip;
|
|
private bool _isRecording = false;
|
|
private CancellationTokenSource _cancellationTokenSource;
|
|
|
|
private int _lastMicrophonePosition = 0;
|
|
private float[] _audioBuffer;
|
|
private string _selectedMicrophone;
|
|
private int _packetSequence = 0;
|
|
|
|
public event Action<bool> OnRecordingStateChanged;
|
|
|
|
private void Start()
|
|
{
|
|
// Apply global config if enabled
|
|
if (useGlobalNetworkConfig)
|
|
{
|
|
var cfg = networkConfigAsset != null ? networkConfigAsset : NetworkConfig.Instance;
|
|
if (cfg != null)
|
|
{
|
|
targetIP = cfg.ipAddress;
|
|
targetPort = cfg.multiplayerAudioPort;
|
|
}
|
|
}
|
|
InitializeNetwork();
|
|
InitializeAudio();
|
|
_cancellationTokenSource = new CancellationTokenSource();
|
|
}
|
|
|
|
private void Update()
|
|
{
|
|
HandleInput();
|
|
}
|
|
|
|
private void OnDestroy()
|
|
{
|
|
StopRecording();
|
|
_cancellationTokenSource?.Cancel();
|
|
_cancellationTokenSource?.Dispose();
|
|
_udpClient?.Close();
|
|
}
|
|
|
|
private void InitializeNetwork()
|
|
{
|
|
try
|
|
{
|
|
_udpClient = new UdpClient();
|
|
_targetEndPoint = new IPEndPoint(IPAddress.Parse(targetIP), targetPort);
|
|
ConvaiLogger.Info($"Simple UDP Audio Sender initialized. Target: {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
ConvaiLogger.Error($"Failed to initialize UDP client: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
}
|
|
|
|
private void InitializeAudio()
|
|
{
|
|
try
|
|
{
|
|
// Try to get selected microphone from Convai's UI system
|
|
_selectedMicrophone = MicrophoneManager.Instance?.SelectedMicrophoneName;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// If UISaveLoadSystem / MicrophoneManager isn't initialized yet, fall back to first available device
|
|
ConvaiLogger.Warn($"MicrophoneManager not available; falling back to default device. {ex.Message}", ConvaiLogger.LogCategory.Character);
|
|
_selectedMicrophone = null;
|
|
}
|
|
|
|
// Fallback: pick the first available microphone if none selected or manager unavailable
|
|
if (string.IsNullOrEmpty(_selectedMicrophone))
|
|
{
|
|
var devices = Microphone.devices;
|
|
if (devices != null && devices.Length > 0)
|
|
{
|
|
_selectedMicrophone = devices[0];
|
|
ConvaiLogger.Info($"Using default microphone: {_selectedMicrophone}", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
}
|
|
|
|
_audioBuffer = new float[recordingFrequency * recordingLength];
|
|
|
|
if (string.IsNullOrEmpty(_selectedMicrophone))
|
|
{
|
|
ConvaiLogger.Error("No microphone available or selected for UDP audio sender", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
}
|
|
|
|
private void HandleInput()
|
|
{
|
|
// Handle talk key
|
|
if (useHoldToTalk)
|
|
{
|
|
if ((Input.GetKeyDown(talkKey) || Input.GetKeyDown(controllerTalkButton)) && !_isRecording)
|
|
{
|
|
StartRecording();
|
|
}
|
|
else if ((Input.GetKeyUp(talkKey) || Input.GetKeyUp(controllerTalkButton)) && _isRecording)
|
|
{
|
|
StopRecording();
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (Input.GetKeyDown(talkKey) || Input.GetKeyDown(controllerTalkButton))
|
|
{
|
|
if (_isRecording)
|
|
StopRecording();
|
|
else
|
|
StartRecording();
|
|
}
|
|
}
|
|
|
|
// Handle test connection key
|
|
if (Input.GetKeyDown(testConnectionKey))
|
|
{
|
|
TestConnection();
|
|
}
|
|
}
|
|
|
|
public void StartRecording()
|
|
{
|
|
if (_isRecording || string.IsNullOrEmpty(_selectedMicrophone))
|
|
return;
|
|
|
|
try
|
|
{
|
|
// Use looping clip so we can handle ring-buffer wrap-around reliably
|
|
_audioClip = Microphone.Start(_selectedMicrophone, true, recordingLength, recordingFrequency);
|
|
_isRecording = true;
|
|
_lastMicrophonePosition = 0;
|
|
_packetSequence = 0;
|
|
|
|
ConvaiLogger.Info("Started recording for UDP transmission (Simple)", ConvaiLogger.LogCategory.Character);
|
|
OnRecordingStateChanged?.Invoke(true);
|
|
|
|
// Start continuous audio processing
|
|
_ = ProcessAudioContinuously(_cancellationTokenSource.Token);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
ConvaiLogger.Error($"Failed to start recording: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
}
|
|
|
|
public void StopRecording()
|
|
{
|
|
if (!_isRecording)
|
|
return;
|
|
|
|
try
|
|
{
|
|
Microphone.End(_selectedMicrophone);
|
|
_isRecording = false;
|
|
|
|
ConvaiLogger.Info("Stopped recording for UDP transmission (Simple)", ConvaiLogger.LogCategory.Character);
|
|
OnRecordingStateChanged?.Invoke(false);
|
|
|
|
// Send end-of-recording signal
|
|
SendEndOfRecordingSignal();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
ConvaiLogger.Error($"Failed to stop recording: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
}
|
|
|
|
private async Task ProcessAudioContinuously(CancellationToken cancellationToken)
|
|
{
|
|
while (_isRecording && !cancellationToken.IsCancellationRequested)
|
|
{
|
|
try
|
|
{
|
|
await Task.Delay(30, cancellationToken); // Process ~33 times/sec for better capture granularity
|
|
|
|
if (_audioClip == null || !Microphone.IsRecording(_selectedMicrophone))
|
|
break;
|
|
|
|
int currentMicrophonePosition = Microphone.GetPosition(_selectedMicrophone);
|
|
int clipSamples = _audioClip.samples;
|
|
|
|
if (clipSamples <= 0)
|
|
continue;
|
|
|
|
// Compute how many new samples are available, accounting for wrap-around
|
|
int audioDataLength = currentMicrophonePosition - _lastMicrophonePosition;
|
|
bool wrapped = false;
|
|
if (audioDataLength < 0)
|
|
{
|
|
audioDataLength += clipSamples;
|
|
wrapped = true;
|
|
}
|
|
|
|
if (audioDataLength <= 0)
|
|
continue;
|
|
|
|
if (!wrapped)
|
|
{
|
|
// Contiguous region, read exactly the new samples
|
|
int segmentLen = audioDataLength;
|
|
var segment = new float[segmentLen];
|
|
_audioClip.GetData(segment, _lastMicrophonePosition);
|
|
await SendAudioDataInChunks(segment, segmentLen);
|
|
}
|
|
else
|
|
{
|
|
// Wrapped: send tail [lastPos .. end) then head [0 .. currentPos)
|
|
int firstLen = clipSamples - _lastMicrophonePosition;
|
|
if (firstLen > 0)
|
|
{
|
|
var firstSeg = new float[firstLen];
|
|
_audioClip.GetData(firstSeg, _lastMicrophonePosition);
|
|
await SendAudioDataInChunks(firstSeg, firstLen);
|
|
}
|
|
|
|
int secondLen = currentMicrophonePosition;
|
|
if (secondLen > 0)
|
|
{
|
|
var secondSeg = new float[secondLen];
|
|
_audioClip.GetData(secondSeg, 0);
|
|
await SendAudioDataInChunks(secondSeg, secondLen);
|
|
}
|
|
}
|
|
|
|
_lastMicrophonePosition = currentMicrophonePosition;
|
|
}
|
|
catch (Exception ex) when (!(ex is OperationCanceledException))
|
|
{
|
|
ConvaiLogger.Error($"Error in audio processing: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task SendAudioDataInChunks(float[] audioData, int totalSamples)
|
|
{
|
|
int processedSamples = 0;
|
|
|
|
while (processedSamples < totalSamples)
|
|
{
|
|
try
|
|
{
|
|
int remainingSamples = totalSamples - processedSamples;
|
|
int currentChunkSamples = Mathf.Min(samplesPerPacket, remainingSamples);
|
|
|
|
// Create a simple packet structure
|
|
byte[] packet = CreateSimpleAudioPacket(audioData, processedSamples, currentChunkSamples);
|
|
|
|
// Send the packet
|
|
await _udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
|
|
|
|
if (enableDebugLogging && _packetSequence % 10 == 0) // Log every 10th packet
|
|
{
|
|
ConvaiLogger.DebugLog($"Sent packet {_packetSequence} with {currentChunkSamples} samples", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
|
|
processedSamples += currentChunkSamples;
|
|
_packetSequence++;
|
|
|
|
// Small delay to avoid overwhelming the network
|
|
await Task.Delay(10);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
ConvaiLogger.Error($"Failed to send audio chunk: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
private byte[] CreateSimpleAudioPacket(float[] audioData, int startIndex, int sampleCount)
|
|
{
|
|
// Simple packet structure:
|
|
// 4 bytes: Magic number (0xC0A1)
|
|
// 4 bytes: Packet sequence number
|
|
// 4 bytes: Sample count in this packet
|
|
// 4 bytes: Start position in stream
|
|
// 1 byte: Flags (0 = normal audio, 1 = end of recording)
|
|
// N bytes: Audio data (converted to shorts)
|
|
|
|
int headerSize = 17; // 4 + 4 + 4 + 4 + 1
|
|
int audioDataSize = sampleCount * sizeof(short);
|
|
byte[] packet = new byte[headerSize + audioDataSize];
|
|
|
|
int offset = 0;
|
|
|
|
// Magic number
|
|
BitConverter.GetBytes((uint)0xC0A1).CopyTo(packet, offset);
|
|
offset += 4;
|
|
|
|
// Packet sequence
|
|
BitConverter.GetBytes(_packetSequence).CopyTo(packet, offset);
|
|
offset += 4;
|
|
|
|
// Sample count
|
|
BitConverter.GetBytes(sampleCount).CopyTo(packet, offset);
|
|
offset += 4;
|
|
|
|
// Start position
|
|
BitConverter.GetBytes(_lastMicrophonePosition + startIndex).CopyTo(packet, offset);
|
|
offset += 4;
|
|
|
|
// Flags (0 for normal audio)
|
|
packet[offset] = 0;
|
|
offset += 1;
|
|
|
|
// Convert audio samples to bytes (same as Convai approach)
|
|
for (int i = 0; i < sampleCount; i++)
|
|
{
|
|
float sample = audioData[startIndex + i];
|
|
short shortSample = (short)(sample * short.MaxValue);
|
|
byte[] shortBytes = BitConverter.GetBytes(shortSample);
|
|
packet[offset] = shortBytes[0];
|
|
packet[offset + 1] = shortBytes[1];
|
|
offset += 2;
|
|
}
|
|
|
|
return packet;
|
|
}
|
|
|
|
private void SendEndOfRecordingSignal()
|
|
{
|
|
try
|
|
{
|
|
// Create end packet
|
|
byte[] packet = new byte[17]; // Header only, no audio data
|
|
int offset = 0;
|
|
|
|
// Magic number
|
|
BitConverter.GetBytes((uint)0xC0A1).CopyTo(packet, offset);
|
|
offset += 4;
|
|
|
|
// Packet sequence
|
|
BitConverter.GetBytes(_packetSequence).CopyTo(packet, offset);
|
|
offset += 4;
|
|
|
|
// Sample count (0 for end signal)
|
|
BitConverter.GetBytes(0).CopyTo(packet, offset);
|
|
offset += 4;
|
|
|
|
// Start position
|
|
BitConverter.GetBytes(_lastMicrophonePosition).CopyTo(packet, offset);
|
|
offset += 4;
|
|
|
|
// Flags (1 for end of recording)
|
|
packet[offset] = 1;
|
|
|
|
_udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
ConvaiLogger.Error($"Failed to send end signal: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
}
|
|
|
|
// Public methods for external control
|
|
public void SetTargetEndpoint(string ip, int port)
|
|
{
|
|
targetIP = ip;
|
|
targetPort = port;
|
|
_targetEndPoint = new IPEndPoint(IPAddress.Parse(ip), port);
|
|
}
|
|
|
|
public bool IsRecording => _isRecording;
|
|
|
|
// Debug and testing methods
|
|
public async void TestConnection()
|
|
{
|
|
if (_udpClient == null)
|
|
{
|
|
ConvaiLogger.Error("UDP client not initialized", ConvaiLogger.LogCategory.Character);
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
ConvaiLogger.Info($"Testing connection to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
|
|
|
|
// Send a simple test packet
|
|
string testMessage = "CONVAI_TEST_CONNECTION";
|
|
byte[] testData = System.Text.Encoding.UTF8.GetBytes(testMessage);
|
|
|
|
await _udpClient.SendAsync(testData, testData.Length, _targetEndPoint);
|
|
ConvaiLogger.Info("Test packet sent successfully", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
ConvaiLogger.Error($"Connection test failed: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
}
|
|
|
|
public void ShowNetworkStatus()
|
|
{
|
|
ConvaiLogger.Info($"=== Network Status ===", ConvaiLogger.LogCategory.Character);
|
|
ConvaiLogger.Info($"Target: {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
|
|
ConvaiLogger.Info($"UDP Client: {(_udpClient != null ? "Initialized" : "Not initialized")}", ConvaiLogger.LogCategory.Character);
|
|
ConvaiLogger.Info($"Recording: {_isRecording}", ConvaiLogger.LogCategory.Character);
|
|
ConvaiLogger.Info($"Microphone: {_selectedMicrophone}", ConvaiLogger.LogCategory.Character);
|
|
ConvaiLogger.Info($"Packets sent: {_packetSequence}", ConvaiLogger.LogCategory.Character);
|
|
|
|
if (_udpClient?.Client?.LocalEndPoint != null)
|
|
{
|
|
ConvaiLogger.Info($"Local endpoint: {_udpClient.Client.LocalEndPoint}", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
}
|
|
}
|
|
}
|