tomh104/Master-Arbeit-Tom-Hempel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

created centralized UDP Listener

Browse Source
This commit is contained in:
tom.hempel
2025-10-25 14:58:22 +02:00
parent 4ddb89d011
commit 3f72973ff5
10 changed files with 585 additions and 534 deletions
Download Patch File Download Diff File Expand all files Collapse all files

141
Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs
View File

@ -48,25 +48,21 @@ namespace Convai.Scripts.Runtime.Multiplayer
private AudioClip _audioClip;
private bool _isRecording = false;
private CancellationTokenSource _cancellationTokenSource;
private CancellationTokenSource _ackCancellationTokenSource;
private int _lastMicrophonePosition = 0;
private float[] _audioBuffer;
private string _selectedMicrophone;
private int _packetSequence = 0;
private volatile bool _startAckReceived = false;
private bool _xrAButtonPrevPressed = false;
private InputAction _xrTalkAction;
private InputAction _xrTestAction;
private bool _usingExternalTalkAction = false;
private InputAction _externalTalkAction;
// Protocol constants
// Protocol constants - SIMPLIFIED
private const uint AUDIO_MAGIC = 0xC0A1;
private const uint ACK_MAGIC = 0xC0A2;
private const byte FLAG_AUDIO = 0;
private const byte FLAG_END = 1;
private const byte FLAG_START = 2;
public event Action<bool> OnRecordingStateChanged;
@ -118,10 +114,6 @@ namespace Convai.Scripts.Runtime.Multiplayer
InitializeAudio();
_persistentDataPath = Application.persistentDataPath;
_cancellationTokenSource = new CancellationTokenSource();
_ackCancellationTokenSource = new CancellationTokenSource();
// Start ACK listener
_ = ListenForAcks(_ackCancellationTokenSource.Token);
// Setup Input System action for XR A/primary button
if (useInputSystemXR)
@ -158,8 +150,6 @@ namespace Convai.Scripts.Runtime.Multiplayer
StopRecording();
_cancellationTokenSource?.Cancel();
_cancellationTokenSource?.Dispose();
_ackCancellationTokenSource?.Cancel();
_ackCancellationTokenSource?.Dispose();
if (_usingExternalTalkAction)
TeardownExternalTalkInputAction();
else
@ -476,7 +466,6 @@ namespace Convai.Scripts.Runtime.Multiplayer
_isRecording = true;
_lastMicrophonePosition = 0;
_packetSequence = 0;
_startAckReceived = false;
_localSessionStart = DateTime.UtcNow;
lock (_localAudioLock)
{
@ -485,11 +474,8 @@ namespace Convai.Scripts.Runtime.Multiplayer
ConvaiLogger.Info($"🎤 Started recording for UDP transmission to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
OnRecordingStateChanged?.Invoke(true);
// Send START control and wait briefly for ACK to ensure receiver is ready
_ = SendStartOfRecordingSignalAndAwaitAck();
// Start continuous audio processing
// Start continuous audio processing immediately (SIMPLIFIED - no ACK waiting)
_ = ProcessAudioContinuously(_cancellationTokenSource.Token);
}
catch (Exception ex)
@ -640,15 +626,13 @@ namespace Convai.Scripts.Runtime.Multiplayer
private byte[] CreateSimpleAudioPacket(float[] audioData, int startIndex, int sampleCount)
{
// Simple packet structure:
// SIMPLIFIED packet structure:
// 4 bytes: Magic number (0xC0A1)
// 4 bytes: Packet sequence number
// 4 bytes: Sample count in this packet
// 4 bytes: Start position in stream
// 1 byte: Flags (0 = normal audio, 1 = end of recording)
// N bytes: Audio data (converted to shorts)
int headerSize = 17; // 4 + 4 + 4 + 4 + 1
int headerSize = 12; // 4 + 4 + 4 (SIMPLIFIED from 17)
int audioDataSize = sampleCount * sizeof(short);
byte[] packet = new byte[headerSize + audioDataSize];
@ -666,15 +650,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
BitConverter.GetBytes(sampleCount).CopyTo(packet, offset);
offset += 4;
// Start position
BitConverter.GetBytes(_lastMicrophonePosition + startIndex).CopyTo(packet, offset);
offset += 4;
// Flags (0 for normal audio)
packet[offset] = FLAG_AUDIO;
offset += 1;
// Convert audio samples to bytes (same as Convai approach)
// Convert audio samples to bytes
for (int i = 0; i < sampleCount; i++)
{
float sample = audioData[startIndex + i];
@ -685,6 +661,11 @@ namespace Convai.Scripts.Runtime.Multiplayer
offset += 2;
}
if (enableDebugLogging && _packetSequence % 50 == 0)
{
ConvaiLogger.DebugLog($"📤 Sent audio packet #{_packetSequence} (magic: 0x{AUDIO_MAGIC:X}) to {targetIP}:{targetPort}, {sampleCount} samples", ConvaiLogger.LogCategory.Character);
}
return packet;
}
@ -692,8 +673,8 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
try
{
// Create end packet
byte[] packet = new byte[17]; // Header only, no audio data
// SIMPLIFIED end packet
byte[] packet = new byte[12]; // Header only, no audio data
int offset = 0;
// Magic number
@ -704,18 +685,12 @@ namespace Convai.Scripts.Runtime.Multiplayer
BitConverter.GetBytes(_packetSequence).CopyTo(packet, offset);
offset += 4;
// Sample count (0 for end signal)
BitConverter.GetBytes(0).CopyTo(packet, offset);
offset += 4;
// Start position
BitConverter.GetBytes(_lastMicrophonePosition).CopyTo(packet, offset);
offset += 4;
// Flags (1 for end of recording)
packet[offset] = FLAG_END;
// Sample count (-1 for end signal)
BitConverter.GetBytes(-1).CopyTo(packet, offset);
_udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
ConvaiLogger.Info($"📤 Sent END signal (magic: 0x{AUDIO_MAGIC:X}) to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
}
catch (Exception ex)
{
@ -817,90 +792,6 @@ namespace Convai.Scripts.Runtime.Multiplayer
}
}
private async Task SendStartOfRecordingSignalAndAwaitAck()
{
try
{
const int maxAttempts = 3;
const int ackTimeoutMs = 250;
for (int attempt = 1; attempt <= maxAttempts && !_startAckReceived; attempt++)
{
// Build START control packet (no audio, special flag)
byte[] packet = new byte[17];
int offset = 0;
BitConverter.GetBytes(AUDIO_MAGIC).CopyTo(packet, offset);
offset += 4;
// Use -1 as the special sequence for START control
BitConverter.GetBytes(-1).CopyTo(packet, offset);
offset += 4;
BitConverter.GetBytes(0).CopyTo(packet, offset);
offset += 4;
BitConverter.GetBytes(_lastMicrophonePosition).CopyTo(packet, offset);
offset += 4;
packet[offset] = FLAG_START;
await _udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
// Wait for ACK
int waited = 0;
while (!_startAckReceived && waited < ackTimeoutMs)
{
await Task.Delay(10);
waited += 10;
}
if (_startAckReceived)
{
if (enableDebugLogging)
ConvaiLogger.DebugLog("Received START ACK from receiver", ConvaiLogger.LogCategory.Character);
break;
}
else if (enableDebugLogging)
{
ConvaiLogger.Warn($"No START ACK (attempt {attempt}/{maxAttempts}), retrying...", ConvaiLogger.LogCategory.Character);
}
}
}
catch (Exception ex)
{
ConvaiLogger.Warn($"Error during START ACK process: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
private async Task ListenForAcks(CancellationToken token)
{
// Use the same UDP client; acks will be sent back to our ephemeral local port
while (!token.IsCancellationRequested)
{
try
{
var result = await _udpClient.ReceiveAsync();
var data = result.Buffer;
if (data == null || data.Length < 8)
continue;
uint magic = BitConverter.ToUInt32(data, 0);
if (magic != ACK_MAGIC)
continue;
int seq = BitConverter.ToInt32(data, 4);
if (seq == -1)
{
_startAckReceived = true;
}
}
catch (ObjectDisposedException)
{
break;
}
catch (Exception)
{
// Ignore and keep listening
}
}
}
// Public methods for external control
public void SetTargetEndpoint(string ip, int port)
{