Files
Master-Arbeit-Tom-Hempel/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioReceiver.cs

641 lines
25 KiB
C#

using System;
using System.Collections.Generic;
using System.Net;
using System.Net.Sockets;
using System.Threading;
using System.Threading.Tasks;
using Convai.Scripts.Runtime.Core;
using Convai.Scripts.Runtime.LoggerSystem;
using Convai.Scripts.Runtime.Utils;
using UnityEngine;
using System.IO;
namespace Convai.Scripts.Runtime.Multiplayer
{
/// <summary>
/// Simple UDP Audio Receiver V2 - Simulates microphone input by triggering normal Convai flow
/// This approach is much simpler and more reliable than trying to replicate gRPC calls
/// </summary>
public class ConvaiSimpleUDPAudioReceiver : MonoBehaviour
{
[Header("Network Configuration")]
[SerializeField] private bool enableDebugLogging = true;
[Header("NPC Target")]
[SerializeField] private bool useActiveNPC = true;
[SerializeField] private ConvaiNPC targetNPC;
// Events
public Action<bool> OnAudioReceiving;
// Metrics for debug UI
private int _totalPacketsReceived = 0;
private DateTime _lastPacketReceivedTime;
public int TotalPacketsReceived => _totalPacketsReceived;
public float TimeSinceLastReceive => _lastPacketReceivedTime != default ?
(float)(DateTime.UtcNow - _lastPacketReceivedTime).TotalSeconds : -1f;
public int ListenPort => listenPort;
// Network components
private UdpClient _udpListener;
private IPEndPoint _remoteEndPoint;
private bool _isListening = false;
private int listenPort;
private CancellationTokenSource _cancellationTokenSource;
// Audio state tracking
private bool _isReceivingAudio = false;
private int _expectedSequence = 0;
private const uint MAGIC_NUMBER = 0xC0A1; // Simple magic number for packet validation
private const uint ACK_MAGIC = 0xC0A2; // ACK magic to confirm START control
// Timing for auto-stop
private float _lastPacketTime;
private const float AUTO_STOP_DELAY = 1.0f; // Stop listening after 1 second of no packets
// Packet structure (matching ConvaiSimpleUDPAudioSender)
private struct AudioPacketData
{
public uint magicNumber;
public int sequence;
public int sampleCount;
public int microphonePosition;
public bool isEndSignal;
public bool isStartSignal;
public short[] audioSamples;
public long timestamp;
}
[Header("Recording Storage")]
[SerializeField] private bool saveReceivedAudio = true;
[SerializeField] private int receivedSampleRate = 16000; // Should match sender
[SerializeField] private string outputFilePrefix = "received_audio";
private readonly object _audioBufferLock = new object();
private List<short> _receivedSamples = new List<short>(64 * 1024);
private Dictionary<int, short[]> _pendingPackets = new Dictionary<int, short[]>();
private int _nextSequenceToWrite = 0;
private DateTime _sessionStartTime;
private bool _saveInProgress = false;
private string _persistentDataPath;
private void Start()
{
_cancellationTokenSource = new CancellationTokenSource();
_persistentDataPath = Application.persistentDataPath;
// Get network config from global instance
var cfg = NetworkConfig.Instance;
if (cfg != null)
{
listenPort = cfg.port;
}
else
{
Debug.LogError("NetworkConfig not found! Please ensure NetworkConfig.asset exists in Resources folder.");
listenPort = 1221;
}
InitializeNetwork();
InitializeConvai();
// Subscribe to NPC manager events to handle late NPC activation
if (ConvaiNPCManager.Instance != null)
{
ConvaiNPCManager.Instance.OnActiveNPCChanged += HandleActiveNPCChanged;
}
}
private void OnEnable()
{
// When re-enabled, ensure listener is running
if (_cancellationTokenSource == null)
{
_cancellationTokenSource = new CancellationTokenSource();
}
StartListening();
// Immediately try to assign an enabled NPC
if (useActiveNPC && targetNPC == null)
{
var currentActiveNPC = FindEnabledConvaiNPC();
if (currentActiveNPC != null)
{
targetNPC = currentActiveNPC;
ConvaiLogger.Info($"🔄 UDP Audio Receiver assigned target NPC on enable: {targetNPC.characterName} (on {targetNPC.gameObject.name})", ConvaiLogger.LogCategory.Character);
}
}
}
private void OnDestroy()
{
// Unsubscribe from events
if (ConvaiNPCManager.Instance != null)
{
ConvaiNPCManager.Instance.OnActiveNPCChanged -= HandleActiveNPCChanged;
}
StopListening();
_cancellationTokenSource?.Cancel();
_cancellationTokenSource?.Dispose();
}
private void OnDisable()
{
// Free the UDP port when this NPC gets disabled
StopListening();
}
private void Update()
{
// Auto-stop listening if no packets received for a while
if (_isReceivingAudio && Time.time - _lastPacketTime > AUTO_STOP_DELAY)
{
StopTalkingSimulation();
}
// Continuously update target NPC if using active NPC mode
if (useActiveNPC)
{
var currentActiveNPC = FindEnabledConvaiNPC();
// Update whenever the active NPC changes (including null → NPC or NPC → different NPC)
if (currentActiveNPC != targetNPC)
{
targetNPC = currentActiveNPC;
if (targetNPC != null)
{
ConvaiLogger.Info($"🔄 UDP Audio Receiver updated target NPC to: {targetNPC.characterName} (on {targetNPC.gameObject.name})", ConvaiLogger.LogCategory.Character);
}
else
{
ConvaiLogger.Info($"🔄 UDP Audio Receiver cleared target NPC", ConvaiLogger.LogCategory.Character);
}
}
}
}
/// <summary>
/// Finds an enabled ConvaiNPC in the scene (doesn't rely on ConvaiNPCManager raycasting)
/// </summary>
private ConvaiNPC FindEnabledConvaiNPC()
{
// Find all ConvaiNPC components in the scene (including inactive GameObjects)
var allNPCs = FindObjectsOfType<ConvaiNPC>(true);
// Return the first one that's on an active GameObject
foreach (var npc in allNPCs)
{
if (npc.gameObject.activeInHierarchy && npc.enabled)
{
return npc;
}
}
return null;
}
private void InitializeNetwork()
{
try
{
StartListening();
}
catch (Exception ex)
{
ConvaiLogger.Error($"Failed to initialize UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
private void InitializeConvai()
{
// Get target NPC by finding enabled NPCs in the scene
if (useActiveNPC)
{
targetNPC = FindEnabledConvaiNPC();
}
if (targetNPC == null)
{
ConvaiLogger.Warn("No target NPC found yet, will wait for NPC to become active", ConvaiLogger.LogCategory.Character);
}
else
{
ConvaiLogger.Info($"UDP Audio Receiver V2 initialized with NPC: {targetNPC.characterName} (on {targetNPC.gameObject.name})", ConvaiLogger.LogCategory.Character);
}
}
public void StartListening()
{
if (_isListening || _cancellationTokenSource == null)
return;
try
{
// Create UDP client with port reuse to allow sharing with UDPPeerDiscovery
_udpListener = new UdpClient();
_udpListener.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_udpListener.Client.Bind(new IPEndPoint(IPAddress.Any, listenPort));
_isListening = true;
ConvaiLogger.Info($"Simple UDP Audio Receiver V2 listening on port {listenPort} (shared)", ConvaiLogger.LogCategory.Character);
// Start listening for incoming packets
_ = ListenForAudioPackets(_cancellationTokenSource.Token);
}
catch (Exception ex)
{
ConvaiLogger.Error($"Failed to start UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Error($"Stack trace: {ex.StackTrace}", ConvaiLogger.LogCategory.Character);
}
}
public void StopListening()
{
if (!_isListening)
return;
_isListening = false;
_udpListener?.Close();
_udpListener?.Dispose();
_udpListener = null;
// Stop any ongoing simulation
StopTalkingSimulation();
ConvaiLogger.Info("Stopped UDP Audio Receiver V2", ConvaiLogger.LogCategory.Character);
}
private async Task ListenForAudioPackets(CancellationToken cancellationToken)
{
try
{
while (_isListening && !cancellationToken.IsCancellationRequested)
{
var result = await _udpListener.ReceiveAsync();
_remoteEndPoint = result.RemoteEndPoint;
await ProcessReceivedPacket(result.Buffer, result.RemoteEndPoint);
}
}
catch (ObjectDisposedException)
{
// Normal when stopping
}
catch (Exception ex)
{
ConvaiLogger.Error($"Error in UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
private async Task ProcessReceivedPacket(byte[] data, IPEndPoint sender)
{
try
{
var packetData = ParseSimpleAudioPacket(data);
if (packetData.HasValue)
{
var packet = packetData.Value;
_lastPacketTime = Time.time;
// Update metrics
_totalPacketsReceived++;
_lastPacketReceivedTime = DateTime.UtcNow;
if (enableDebugLogging)
{
if (packet.isEndSignal)
ConvaiLogger.DebugLog($"Received end signal from {sender}", ConvaiLogger.LogCategory.Character);
else
ConvaiLogger.DebugLog($"Received audio packet {packet.sequence} with {packet.sampleCount} samples", ConvaiLogger.LogCategory.Character);
}
// Handle START control: acknowledge and begin simulation
if (packet.isStartSignal)
{
SendStartAck(sender);
if (!_isReceivingAudio)
{
StartTalkingSimulation();
}
OnAudioReceiving?.Invoke(true);
return;
}
if (packet.isEndSignal)
{
StopTalkingSimulation();
OnAudioReceiving?.Invoke(false);
}
else
{
// If this is the first packet, start the talking simulation
if (packet.sequence == 0 && !_isReceivingAudio)
{
StartTalkingSimulation();
}
// Buffer audio samples for saving
if (packet.audioSamples != null && packet.audioSamples.Length > 0)
{
BufferAudioPacket(packet.sequence, packet.audioSamples);
}
OnAudioReceiving?.Invoke(true);
}
}
else
{
// Not our audio packet format, might be a test message
string message = System.Text.Encoding.UTF8.GetString(data);
if (enableDebugLogging)
ConvaiLogger.Info($"Received test message from {sender}: {message}", ConvaiLogger.LogCategory.Character);
}
}
catch (Exception ex)
{
ConvaiLogger.Error($"Error processing received packet: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
private void StartTalkingSimulation()
{
if (_isReceivingAudio) return;
MainThreadDispatcher.Instance.RunOnMainThread(() => {
// Update target NPC by finding enabled NPCs in the scene
if (useActiveNPC)
{
targetNPC = FindEnabledConvaiNPC();
}
if (targetNPC == null)
{
ConvaiLogger.Warn("No target NPC available for audio simulation", ConvaiLogger.LogCategory.Character);
return;
}
_isReceivingAudio = true;
_expectedSequence = 0;
_nextSequenceToWrite = 0;
_sessionStartTime = DateTime.UtcNow;
lock (_audioBufferLock)
{
_receivedSamples.Clear();
_pendingPackets.Clear();
}
// This is the KEY! Simulate a talk key press to trigger normal Convai flow
ConvaiInputManager.Instance.talkKeyInteract?.Invoke(true);
ConvaiLogger.Info($"🎤 Started talking simulation for {targetNPC.characterName} (on {targetNPC.gameObject.name}) (remote player audio)", ConvaiLogger.LogCategory.Character);
});
}
private void StopTalkingSimulation()
{
if (!_isReceivingAudio) return;
MainThreadDispatcher.Instance.RunOnMainThread(() => {
_isReceivingAudio = false;
// Simulate talk key release to stop recording
ConvaiInputManager.Instance.talkKeyInteract?.Invoke(false);
ConvaiLogger.Info($"🎤 Stopped talking simulation for {targetNPC?.characterName ?? "NPC"} (remote player audio)", ConvaiLogger.LogCategory.Character);
if (saveReceivedAudio)
{
TrySaveReceivedAudioAsync();
}
});
}
private AudioPacketData? ParseSimpleAudioPacket(byte[] data)
{
if (data.Length < 17) // Minimum header size to match sender
return null;
try
{
int offset = 0;
// Read magic number
uint magic = BitConverter.ToUInt32(data, offset);
offset += 4;
if (magic != MAGIC_NUMBER)
return null;
// Read header (matching sender's 17-byte format)
int sequence = BitConverter.ToInt32(data, offset);
offset += 4;
int sampleCount = BitConverter.ToInt32(data, offset);
offset += 4;
int microphonePosition = BitConverter.ToInt32(data, offset);
offset += 4;
byte flags = data[offset];
offset += 1;
bool isEndSignal = (flags == 1);
bool isStartSignal = (flags == 2);
// Read audio data
short[] audioSamples = null;
if (!isEndSignal && !isStartSignal && sampleCount > 0)
{
int audioDataSize = sampleCount * sizeof(short);
if (data.Length >= offset + audioDataSize)
{
audioSamples = new short[sampleCount];
Buffer.BlockCopy(data, offset, audioSamples, 0, audioDataSize);
}
}
return new AudioPacketData
{
magicNumber = magic,
sequence = sequence,
sampleCount = sampleCount,
microphonePosition = microphonePosition,
isEndSignal = isEndSignal,
isStartSignal = isStartSignal,
audioSamples = audioSamples,
timestamp = 0 // Not provided in sender format
};
}
catch (Exception ex)
{
ConvaiLogger.Error($"Error parsing audio packet: {ex.Message}", ConvaiLogger.LogCategory.Character);
return null;
}
}
private void SendStartAck(IPEndPoint sender)
{
try
{
if (_udpListener == null || sender == null)
return;
byte[] ack = new byte[8];
BitConverter.GetBytes(ACK_MAGIC).CopyTo(ack, 0);
BitConverter.GetBytes(-1).CopyTo(ack, 4);
_udpListener.SendAsync(ack, ack.Length, sender);
if (enableDebugLogging)
ConvaiLogger.DebugLog($"Sent START ACK to {sender}", ConvaiLogger.LogCategory.Character);
}
catch (Exception ex)
{
ConvaiLogger.Warn($"Failed to send START ACK: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
private void BufferAudioPacket(int sequence, short[] samples)
{
if (samples == null || samples.Length == 0)
return;
lock (_audioBufferLock)
{
if (sequence < _nextSequenceToWrite)
{
return; // old/duplicate packet
}
if (sequence == _nextSequenceToWrite)
{
_receivedSamples.AddRange(samples);
_nextSequenceToWrite++;
// Flush any contiguous pending packets
while (_pendingPackets.TryGetValue(_nextSequenceToWrite, out var nextSamples))
{
_receivedSamples.AddRange(nextSamples);
_pendingPackets.Remove(_nextSequenceToWrite);
_nextSequenceToWrite++;
}
}
else
{
// Store for later when gap is filled
_pendingPackets[sequence] = samples;
}
}
}
private void TrySaveReceivedAudioAsync()
{
if (_saveInProgress)
return;
short[] dataToSave;
DateTime sessionStart;
lock (_audioBufferLock)
{
if (_receivedSamples == null || _receivedSamples.Count == 0)
{
if (enableDebugLogging)
ConvaiLogger.Info("No received audio to save.", ConvaiLogger.LogCategory.Character);
return;
}
dataToSave = _receivedSamples.ToArray();
_receivedSamples.Clear();
_pendingPackets.Clear();
sessionStart = _sessionStartTime;
}
_saveInProgress = true;
Task.Run(() =>
{
try
{
string timestamp = sessionStart.ToLocalTime().ToString("yyyyMMdd_HHmmss");
string fileName = $"{outputFilePrefix}_{timestamp}.wav";
string dir = _persistentDataPath;
string path = Path.Combine(dir, fileName);
WriteWav(path, dataToSave, receivedSampleRate, 1);
ConvaiLogger.Info($"Saved received audio to: {path}", ConvaiLogger.LogCategory.Character);
}
catch (Exception ex)
{
ConvaiLogger.Error($"Failed to save received audio: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
finally
{
_saveInProgress = false;
}
});
}
private void WriteWav(string path, short[] samples, int sampleRate, int channels)
{
using (var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.None))
using (var writer = new BinaryWriter(fs))
{
int bitsPerSample = 16;
int byteRate = sampleRate * channels * (bitsPerSample / 8);
int blockAlign = channels * (bitsPerSample / 8);
int dataSize = samples.Length * (bitsPerSample / 8);
int fileSize = 44 - 8 + dataSize;
// RIFF header
writer.Write(System.Text.Encoding.ASCII.GetBytes("RIFF"));
writer.Write(fileSize);
writer.Write(System.Text.Encoding.ASCII.GetBytes("WAVE"));
// fmt chunk
writer.Write(System.Text.Encoding.ASCII.GetBytes("fmt "));
writer.Write(16); // Subchunk1Size for PCM
writer.Write((short)1); // AudioFormat = PCM
writer.Write((short)channels); // NumChannels
writer.Write(sampleRate); // SampleRate
writer.Write(byteRate); // ByteRate
writer.Write((short)blockAlign); // BlockAlign
writer.Write((short)bitsPerSample); // BitsPerSample
// data chunk
writer.Write(System.Text.Encoding.ASCII.GetBytes("data"));
writer.Write(dataSize);
for (int i = 0; i < samples.Length; i++)
{
writer.Write(samples[i]);
}
}
}
// Event handler for when NPC becomes active
private void HandleActiveNPCChanged(ConvaiNPC newActiveNPC)
{
if (useActiveNPC && newActiveNPC != null)
{
targetNPC = newActiveNPC;
ConvaiLogger.Info($"UDP Audio Receiver V2 updated target NPC to: {targetNPC.characterName}", ConvaiLogger.LogCategory.Character);
}
}
// Public properties for debugging
public bool IsListening => _isListening;
public bool IsReceivingAudio => _isReceivingAudio;
public ConvaiNPC TargetNPC => targetNPC;
// Debug methods
public void ShowNetworkStatus()
{
ConvaiLogger.Info($"=== Audio Receiver V2 Status ===", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"Listening: {_isListening} on port {listenPort}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"Receiving Audio: {_isReceivingAudio}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"Target NPC: {(targetNPC?.characterName ?? "None")}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"Expected Sequence: {_expectedSequence}", ConvaiLogger.LogCategory.Character);
ConvaiLogger.Info($"Last Packet Time: {_lastPacketTime}", ConvaiLogger.LogCategory.Character);
}
}
}