389 lines
15 KiB
C#
389 lines
15 KiB
C#
using System;
|
|
using System.Net;
|
|
using System.Net.Sockets;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using Convai.Scripts.Runtime.LoggerSystem;
|
|
using Convai.Scripts.Runtime.UI;
|
|
using UnityEngine;
|
|
|
|
namespace Convai.Scripts.Runtime.Multiplayer
|
|
{
|
|
/// <summary>
|
|
/// Simplified version of UDP Audio Sender that avoids complex chunking
|
|
/// This version sends smaller, more frequent packets to avoid array bounds issues
|
|
/// </summary>
|
|
public class ConvaiSimpleUDPAudioSender : MonoBehaviour
|
|
{
|
|
[Header("Network Settings")]
|
|
[SerializeField] private string targetIP = "127.0.0.1";
|
|
[SerializeField] private int targetPort = 12345;
|
|
[SerializeField] private bool useGlobalNetworkConfig = true;
|
|
[SerializeField] private NetworkConfig networkConfigAsset;
|
|
|
|
[Header("Audio Settings")]
|
|
[SerializeField] private int recordingFrequency = 16000;
|
|
[SerializeField] private int recordingLength = 10;
|
|
[SerializeField] private int samplesPerPacket = 1024; // Number of audio samples per packet (not bytes)
|
|
|
|
[Header("UI")]
|
|
[SerializeField] private KeyCode talkKey = KeyCode.T;
|
|
[SerializeField] private bool useHoldToTalk = true;
|
|
|
|
[Header("Debug")]
|
|
[SerializeField] private bool enableDebugLogging = true;
|
|
[SerializeField] private KeyCode testConnectionKey = KeyCode.C;
|
|
|
|
private UdpClient _udpClient;
|
|
private IPEndPoint _targetEndPoint;
|
|
private AudioClip _audioClip;
|
|
private bool _isRecording = false;
|
|
private CancellationTokenSource _cancellationTokenSource;
|
|
|
|
private int _lastMicrophonePosition = 0;
|
|
private float[] _audioBuffer;
|
|
private string _selectedMicrophone;
|
|
private int _packetSequence = 0;
|
|
|
|
public event Action<bool> OnRecordingStateChanged;
|
|
|
|
private void Start()
|
|
{
|
|
// Apply global config if enabled
|
|
if (useGlobalNetworkConfig)
|
|
{
|
|
var cfg = networkConfigAsset != null ? networkConfigAsset : NetworkConfig.Instance;
|
|
if (cfg != null)
|
|
{
|
|
targetIP = cfg.ipAddress;
|
|
targetPort = cfg.port;
|
|
}
|
|
}
|
|
InitializeNetwork();
|
|
InitializeAudio();
|
|
_cancellationTokenSource = new CancellationTokenSource();
|
|
}
|
|
|
|
private void Update()
|
|
{
|
|
HandleInput();
|
|
}
|
|
|
|
private void OnDestroy()
|
|
{
|
|
StopRecording();
|
|
_cancellationTokenSource?.Cancel();
|
|
_cancellationTokenSource?.Dispose();
|
|
_udpClient?.Close();
|
|
}
|
|
|
|
private void InitializeNetwork()
|
|
{
|
|
try
|
|
{
|
|
_udpClient = new UdpClient();
|
|
_targetEndPoint = new IPEndPoint(IPAddress.Parse(targetIP), targetPort);
|
|
ConvaiLogger.Info($"Simple UDP Audio Sender initialized. Target: {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
ConvaiLogger.Error($"Failed to initialize UDP client: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
}
|
|
|
|
private void InitializeAudio()
|
|
{
|
|
_selectedMicrophone = MicrophoneManager.Instance.SelectedMicrophoneName;
|
|
_audioBuffer = new float[recordingFrequency * recordingLength];
|
|
|
|
if (string.IsNullOrEmpty(_selectedMicrophone))
|
|
{
|
|
ConvaiLogger.Error("No microphone selected for UDP audio sender", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
}
|
|
|
|
private void HandleInput()
|
|
{
|
|
// Handle talk key
|
|
if (useHoldToTalk)
|
|
{
|
|
if (Input.GetKeyDown(talkKey) && !_isRecording)
|
|
{
|
|
StartRecording();
|
|
}
|
|
else if (Input.GetKeyUp(talkKey) && _isRecording)
|
|
{
|
|
StopRecording();
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (Input.GetKeyDown(talkKey))
|
|
{
|
|
if (_isRecording)
|
|
StopRecording();
|
|
else
|
|
StartRecording();
|
|
}
|
|
}
|
|
|
|
// Handle test connection key
|
|
if (Input.GetKeyDown(testConnectionKey))
|
|
{
|
|
TestConnection();
|
|
}
|
|
}
|
|
|
|
public void StartRecording()
|
|
{
|
|
if (_isRecording || string.IsNullOrEmpty(_selectedMicrophone))
|
|
return;
|
|
|
|
try
|
|
{
|
|
_audioClip = Microphone.Start(_selectedMicrophone, false, recordingLength, recordingFrequency);
|
|
_isRecording = true;
|
|
_lastMicrophonePosition = 0;
|
|
_packetSequence = 0;
|
|
|
|
ConvaiLogger.Info("Started recording for UDP transmission (Simple)", ConvaiLogger.LogCategory.Character);
|
|
OnRecordingStateChanged?.Invoke(true);
|
|
|
|
// Start continuous audio processing
|
|
_ = ProcessAudioContinuously(_cancellationTokenSource.Token);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
ConvaiLogger.Error($"Failed to start recording: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
}
|
|
|
|
public void StopRecording()
|
|
{
|
|
if (!_isRecording)
|
|
return;
|
|
|
|
try
|
|
{
|
|
Microphone.End(_selectedMicrophone);
|
|
_isRecording = false;
|
|
|
|
ConvaiLogger.Info("Stopped recording for UDP transmission (Simple)", ConvaiLogger.LogCategory.Character);
|
|
OnRecordingStateChanged?.Invoke(false);
|
|
|
|
// Send end-of-recording signal
|
|
SendEndOfRecordingSignal();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
ConvaiLogger.Error($"Failed to stop recording: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
}
|
|
|
|
private async Task ProcessAudioContinuously(CancellationToken cancellationToken)
|
|
{
|
|
while (_isRecording && !cancellationToken.IsCancellationRequested)
|
|
{
|
|
try
|
|
{
|
|
await Task.Delay(100, cancellationToken); // Process every 100ms
|
|
|
|
if (_audioClip == null || !Microphone.IsRecording(_selectedMicrophone))
|
|
break;
|
|
|
|
int currentMicrophonePosition = Microphone.GetPosition(_selectedMicrophone);
|
|
int audioDataLength = currentMicrophonePosition - _lastMicrophonePosition;
|
|
|
|
if (audioDataLength > 0)
|
|
{
|
|
// Get audio data from the microphone clip
|
|
_audioClip.GetData(_audioBuffer, _lastMicrophonePosition);
|
|
|
|
// Send data in smaller chunks to avoid array bounds issues
|
|
await SendAudioDataInChunks(_audioBuffer, audioDataLength);
|
|
|
|
_lastMicrophonePosition = currentMicrophonePosition;
|
|
}
|
|
}
|
|
catch (Exception ex) when (!(ex is OperationCanceledException))
|
|
{
|
|
ConvaiLogger.Error($"Error in audio processing: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task SendAudioDataInChunks(float[] audioData, int totalSamples)
|
|
{
|
|
int processedSamples = 0;
|
|
|
|
while (processedSamples < totalSamples)
|
|
{
|
|
try
|
|
{
|
|
int remainingSamples = totalSamples - processedSamples;
|
|
int currentChunkSamples = Mathf.Min(samplesPerPacket, remainingSamples);
|
|
|
|
// Create a simple packet structure
|
|
byte[] packet = CreateSimpleAudioPacket(audioData, processedSamples, currentChunkSamples);
|
|
|
|
// Send the packet
|
|
await _udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
|
|
|
|
if (enableDebugLogging && _packetSequence % 10 == 0) // Log every 10th packet
|
|
{
|
|
ConvaiLogger.DebugLog($"Sent packet {_packetSequence} with {currentChunkSamples} samples", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
|
|
processedSamples += currentChunkSamples;
|
|
_packetSequence++;
|
|
|
|
// Small delay to avoid overwhelming the network
|
|
await Task.Delay(10);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
ConvaiLogger.Error($"Failed to send audio chunk: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
private byte[] CreateSimpleAudioPacket(float[] audioData, int startIndex, int sampleCount)
|
|
{
|
|
// Simple packet structure:
|
|
// 4 bytes: Magic number (0xC0A1)
|
|
// 4 bytes: Packet sequence number
|
|
// 4 bytes: Sample count in this packet
|
|
// 4 bytes: Start position in stream
|
|
// 1 byte: Flags (0 = normal audio, 1 = end of recording)
|
|
// N bytes: Audio data (converted to shorts)
|
|
|
|
int headerSize = 17; // 4 + 4 + 4 + 4 + 1
|
|
int audioDataSize = sampleCount * sizeof(short);
|
|
byte[] packet = new byte[headerSize + audioDataSize];
|
|
|
|
int offset = 0;
|
|
|
|
// Magic number
|
|
BitConverter.GetBytes((uint)0xC0A1).CopyTo(packet, offset);
|
|
offset += 4;
|
|
|
|
// Packet sequence
|
|
BitConverter.GetBytes(_packetSequence).CopyTo(packet, offset);
|
|
offset += 4;
|
|
|
|
// Sample count
|
|
BitConverter.GetBytes(sampleCount).CopyTo(packet, offset);
|
|
offset += 4;
|
|
|
|
// Start position
|
|
BitConverter.GetBytes(_lastMicrophonePosition + startIndex).CopyTo(packet, offset);
|
|
offset += 4;
|
|
|
|
// Flags (0 for normal audio)
|
|
packet[offset] = 0;
|
|
offset += 1;
|
|
|
|
// Convert audio samples to bytes (same as Convai approach)
|
|
for (int i = 0; i < sampleCount; i++)
|
|
{
|
|
float sample = audioData[startIndex + i];
|
|
short shortSample = (short)(sample * short.MaxValue);
|
|
byte[] shortBytes = BitConverter.GetBytes(shortSample);
|
|
packet[offset] = shortBytes[0];
|
|
packet[offset + 1] = shortBytes[1];
|
|
offset += 2;
|
|
}
|
|
|
|
return packet;
|
|
}
|
|
|
|
private void SendEndOfRecordingSignal()
|
|
{
|
|
try
|
|
{
|
|
// Create end packet
|
|
byte[] packet = new byte[17]; // Header only, no audio data
|
|
int offset = 0;
|
|
|
|
// Magic number
|
|
BitConverter.GetBytes((uint)0xC0A1).CopyTo(packet, offset);
|
|
offset += 4;
|
|
|
|
// Packet sequence
|
|
BitConverter.GetBytes(_packetSequence).CopyTo(packet, offset);
|
|
offset += 4;
|
|
|
|
// Sample count (0 for end signal)
|
|
BitConverter.GetBytes(0).CopyTo(packet, offset);
|
|
offset += 4;
|
|
|
|
// Start position
|
|
BitConverter.GetBytes(_lastMicrophonePosition).CopyTo(packet, offset);
|
|
offset += 4;
|
|
|
|
// Flags (1 for end of recording)
|
|
packet[offset] = 1;
|
|
|
|
_udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
ConvaiLogger.Error($"Failed to send end signal: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
}
|
|
|
|
// Public methods for external control
|
|
public void SetTargetEndpoint(string ip, int port)
|
|
{
|
|
targetIP = ip;
|
|
targetPort = port;
|
|
_targetEndPoint = new IPEndPoint(IPAddress.Parse(ip), port);
|
|
}
|
|
|
|
public bool IsRecording => _isRecording;
|
|
|
|
// Debug and testing methods
|
|
public async void TestConnection()
|
|
{
|
|
if (_udpClient == null)
|
|
{
|
|
ConvaiLogger.Error("UDP client not initialized", ConvaiLogger.LogCategory.Character);
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
ConvaiLogger.Info($"Testing connection to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
|
|
|
|
// Send a simple test packet
|
|
string testMessage = "CONVAI_TEST_CONNECTION";
|
|
byte[] testData = System.Text.Encoding.UTF8.GetBytes(testMessage);
|
|
|
|
await _udpClient.SendAsync(testData, testData.Length, _targetEndPoint);
|
|
ConvaiLogger.Info("Test packet sent successfully", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
ConvaiLogger.Error($"Connection test failed: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
}
|
|
|
|
public void ShowNetworkStatus()
|
|
{
|
|
ConvaiLogger.Info($"=== Network Status ===", ConvaiLogger.LogCategory.Character);
|
|
ConvaiLogger.Info($"Target: {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
|
|
ConvaiLogger.Info($"UDP Client: {(_udpClient != null ? "Initialized" : "Not initialized")}", ConvaiLogger.LogCategory.Character);
|
|
ConvaiLogger.Info($"Recording: {_isRecording}", ConvaiLogger.LogCategory.Character);
|
|
ConvaiLogger.Info($"Microphone: {_selectedMicrophone}", ConvaiLogger.LogCategory.Character);
|
|
ConvaiLogger.Info($"Packets sent: {_packetSequence}", ConvaiLogger.LogCategory.Character);
|
|
|
|
if (_udpClient?.Client?.LocalEndPoint != null)
|
|
{
|
|
ConvaiLogger.Info($"Local endpoint: {_udpClient.Client.LocalEndPoint}", ConvaiLogger.LogCategory.Character);
|
|
}
|
|
}
|
|
}
|
|
}
|