tomh104/Master-Arbeit-Tom-Hempel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

further work on bug fixing

Browse Source
This commit is contained in:
tom.hempel
2025-09-26 10:13:33 +02:00
parent 7aeb173789
commit 7d65d1b799
6 changed files with 1492 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioReceiver.cs
View File

@ -39,7 +39,8 @@ namespace Convai.Scripts.Runtime.Multiplayer
// Audio state tracking
private bool _isReceivingAudio = false;
private int _expectedSequence = 0;
private const uint MAGIC_NUMBER = 0xC0A1; // Simple magic number for packet validation
private const uint AUDIO_MAGIC = 0xC0A1; // Audio packet magic
private const uint ACK_MAGIC = 0xC0A2; // Ack packet magic
// Timing for auto-stop
private float _lastPacketTime;
@ -48,13 +49,12 @@ namespace Convai.Scripts.Runtime.Multiplayer
// Packet structure (matching ConvaiSimpleUDPAudioSender)
private struct AudioPacketData
{
public uint magicNumber;
public int sequence;
public int sampleCount;
public int microphonePosition;
public bool isEndSignal;
public bool isStartSignal;
public short[] audioSamples;
public long timestamp;
}
private void Start()
@ -216,9 +216,9 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
try
{
var packetData = ParseSimpleAudioPacket(data);
var packetData = ParseSimpleAudioPacket(data, sender);
if (packetData.HasValue)
if (packetData.HasValue)
{
var packet = packetData.Value;
_lastPacketTime = Time.time;
@ -231,13 +231,17 @@ namespace Convai.Scripts.Runtime.Multiplayer
ConvaiLogger.DebugLog($"Received audio packet {packet.sequence} with {packet.sampleCount} samples", ConvaiLogger.LogCategory.Character);
}
if (packet.isEndSignal)
if (packet.isEndSignal)
{
StopTalkingSimulation();
OnAudioReceiving?.Invoke(false);
}
else
{
if (packet.isStartSignal)
{
// START packet acknowledged earlier
}
// If this is the first packet, start the talking simulation
if (packet.sequence == 0 && !_isReceivingAudio)
{
@ -251,8 +255,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
{
// Not our audio packet format, might be a test message
string message = System.Text.Encoding.UTF8.GetString(data);
if (enableDebugLogging)
ConvaiLogger.Info($"Received test message from {sender}: {message}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"Received test message from {sender}: {message}", ConvaiLogger.LogCategory.Character);
}
}
catch (Exception ex)
@ -302,44 +305,41 @@ namespace Convai.Scripts.Runtime.Multiplayer
});
}
private AudioPacketData? ParseSimpleAudioPacket(byte[] data)
private AudioPacketData? ParseSimpleAudioPacket(byte[] data, IPEndPoint sender)
{
if (data.Length < 24) // Minimum header size
// Sender uses a 17-byte header (no timestamp/padding). We also support older 24+ byte format gracefully.
if (data.Length < 17)
return null;
try
{
int offset = 0;
// Read magic number
uint magic = BitConverter.ToUInt32(data, offset);
offset += 4;
if (magic != MAGIC_NUMBER)
if (magic != AUDIO_MAGIC)
{
// Might be a test message or something else
return null;
// Read header
}
int sequence = BitConverter.ToInt32(data, offset);
offset += 4;
int sampleCount = BitConverter.ToInt32(data, offset);
offset += 4;
int microphonePosition = BitConverter.ToInt32(data, offset);
offset += 4;
bool isEndSignal = BitConverter.ToBoolean(data, offset);
byte flag = data[offset];
offset += 1;
// Skip padding
offset += 3;
long timestamp = BitConverter.ToInt64(data, offset);
offset += 8;
// Read audio data
bool isEndSignal = (flag == 1);
bool isStartSignal = (flag == 2);
// Send ACK immediately (for START and audio packets)
SendAck(sender, sequence);
short[] audioSamples = null;
if (!isEndSignal && sampleCount > 0)
if (!isEndSignal && !isStartSignal && sampleCount > 0)
{
int audioDataSize = sampleCount * sizeof(short);
if (data.Length >= offset + audioDataSize)
@ -348,16 +348,15 @@ namespace Convai.Scripts.Runtime.Multiplayer
Buffer.BlockCopy(data, offset, audioSamples, 0, audioDataSize);
}
}
return new AudioPacketData
{
magicNumber = magic,
sequence = sequence,
sampleCount = sampleCount,
microphonePosition = microphonePosition,
isEndSignal = isEndSignal,
audioSamples = audioSamples,
timestamp = timestamp
isStartSignal = isStartSignal,
audioSamples = audioSamples
};
}
catch (Exception ex)
@ -366,6 +365,24 @@ namespace Convai.Scripts.Runtime.Multiplayer
return null;
}
}
private void SendAck(IPEndPoint recipient, int sequence)
{
try
{
using (var client = new UdpClient())
{
byte[] ack = new byte[8];
Buffer.BlockCopy(BitConverter.GetBytes(ACK_MAGIC), 0, ack, 0, 4);
Buffer.BlockCopy(BitConverter.GetBytes(sequence), 0, ack, 4, 4);
client.Send(ack, ack.Length, recipient);
}
}
catch (Exception ex)
{
ConvaiLogger.Warn($"Failed to send ACK: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
// Event handler for when NPC becomes active
private void HandleActiveNPCChanged(ConvaiNPC newActiveNPC)