added multiplayer scripts
This commit is contained in:
tom.hempel
@ -0,0 +1,364 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Convai.Scripts.Runtime.Core;
|
||||
using Convai.Scripts.Runtime.LoggerSystem;
|
||||
using Convai.Scripts.Runtime.Utils;
|
||||
using UnityEngine;
|
||||
|
||||
namespace Convai.Scripts.Runtime.Multiplayer
|
||||
{
|
||||
/// <summary>
|
||||
/// Simple UDP Audio Receiver V2 - Simulates microphone input by triggering normal Convai flow
|
||||
/// This approach is much simpler and more reliable than trying to replicate gRPC calls
|
||||
/// </summary>
|
||||
public class ConvaiSimpleUDPAudioReceiverV2 : MonoBehaviour
|
||||
{
|
||||
[Header("Network Configuration")]
|
||||
[SerializeField] private int listenPort = 12345;
|
||||
[SerializeField] private bool enableDebugLogging = true;
|
||||
|
||||
[Header("NPC Target")]
|
||||
[SerializeField] private bool useActiveNPC = true;
|
||||
[SerializeField] private ConvaiNPC targetNPC;
|
||||
|
||||
// Events
|
||||
public Action<bool> OnAudioReceiving;
|
||||
|
||||
// Network components
|
||||
private UdpClient _udpListener;
|
||||
private IPEndPoint _remoteEndPoint;
|
||||
private bool _isListening = false;
|
||||
private CancellationTokenSource _cancellationTokenSource;
|
||||
|
||||
// Audio state tracking
|
||||
private bool _isReceivingAudio = false;
|
||||
private int _expectedSequence = 0;
|
||||
private const uint MAGIC_NUMBER = 0xC0A1; // Simple magic number for packet validation
|
||||
|
||||
// Timing for auto-stop
|
||||
private float _lastPacketTime;
|
||||
private const float AUTO_STOP_DELAY = 1.0f; // Stop listening after 1 second of no packets
|
||||
|
||||
// Packet structure (matching ConvaiSimpleUDPAudioSender)
|
||||
private struct AudioPacketData
|
||||
{
|
||||
public uint magicNumber;
|
||||
public int sequence;
|
||||
public int sampleCount;
|
||||
public int microphonePosition;
|
||||
public bool isEndSignal;
|
||||
public short[] audioSamples;
|
||||
public long timestamp;
|
||||
}
|
||||
|
||||
private void Start()
|
||||
{
|
||||
_cancellationTokenSource = new CancellationTokenSource();
|
||||
InitializeNetwork();
|
||||
InitializeConvai();
|
||||
|
||||
// Subscribe to NPC manager events to handle late NPC activation
|
||||
if (ConvaiNPCManager.Instance != null)
|
||||
{
|
||||
ConvaiNPCManager.Instance.OnActiveNPCChanged += HandleActiveNPCChanged;
|
||||
}
|
||||
}
|
||||
|
||||
private void OnDestroy()
|
||||
{
|
||||
// Unsubscribe from events
|
||||
if (ConvaiNPCManager.Instance != null)
|
||||
{
|
||||
ConvaiNPCManager.Instance.OnActiveNPCChanged -= HandleActiveNPCChanged;
|
||||
}
|
||||
|
||||
StopListening();
|
||||
_cancellationTokenSource?.Cancel();
|
||||
_cancellationTokenSource?.Dispose();
|
||||
}
|
||||
|
||||
private void Update()
|
||||
{
|
||||
// Auto-stop listening if no packets received for a while
|
||||
if (_isReceivingAudio && Time.time - _lastPacketTime > AUTO_STOP_DELAY)
|
||||
{
|
||||
StopTalkingSimulation();
|
||||
}
|
||||
}
|
||||
|
||||
private void InitializeNetwork()
|
||||
{
|
||||
try
|
||||
{
|
||||
StartListening();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Failed to initialize UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private void InitializeConvai()
|
||||
{
|
||||
// Get target NPC
|
||||
if (useActiveNPC)
|
||||
{
|
||||
targetNPC = ConvaiNPCManager.Instance?.GetActiveConvaiNPC();
|
||||
}
|
||||
|
||||
if (targetNPC == null)
|
||||
{
|
||||
ConvaiLogger.Warn("No target NPC found yet, will wait for NPC to become active", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
else
|
||||
{
|
||||
ConvaiLogger.Info($"UDP Audio Receiver V2 initialized with NPC: {targetNPC.characterName}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
public void StartListening()
|
||||
{
|
||||
if (_isListening || _cancellationTokenSource == null)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
_udpListener = new UdpClient(listenPort);
|
||||
_isListening = true;
|
||||
|
||||
ConvaiLogger.Info($"Simple UDP Audio Receiver V2 listening on port {listenPort}", ConvaiLogger.LogCategory.Character);
|
||||
|
||||
// Start listening for incoming packets
|
||||
_ = ListenForAudioPackets(_cancellationTokenSource.Token);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Failed to start UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Error($"Stack trace: {ex.StackTrace}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
public void StopListening()
|
||||
{
|
||||
if (!_isListening)
|
||||
return;
|
||||
|
||||
_isListening = false;
|
||||
_udpListener?.Close();
|
||||
_udpListener?.Dispose();
|
||||
_udpListener = null;
|
||||
|
||||
// Stop any ongoing simulation
|
||||
StopTalkingSimulation();
|
||||
|
||||
ConvaiLogger.Info("Stopped UDP Audio Receiver V2", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
|
||||
private async Task ListenForAudioPackets(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
while (_isListening && !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var result = await _udpListener.ReceiveAsync();
|
||||
_remoteEndPoint = result.RemoteEndPoint;
|
||||
|
||||
await ProcessReceivedPacket(result.Buffer, result.RemoteEndPoint);
|
||||
}
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// Normal when stopping
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Error in UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessReceivedPacket(byte[] data, IPEndPoint sender)
|
||||
{
|
||||
try
|
||||
{
|
||||
var packetData = ParseSimpleAudioPacket(data);
|
||||
|
||||
if (packetData.HasValue)
|
||||
{
|
||||
var packet = packetData.Value;
|
||||
_lastPacketTime = Time.time;
|
||||
|
||||
if (enableDebugLogging)
|
||||
{
|
||||
if (packet.isEndSignal)
|
||||
ConvaiLogger.DebugLog($"Received end signal from {sender}", ConvaiLogger.LogCategory.Character);
|
||||
else
|
||||
ConvaiLogger.DebugLog($"Received audio packet {packet.sequence} with {packet.sampleCount} samples", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
|
||||
if (packet.isEndSignal)
|
||||
{
|
||||
StopTalkingSimulation();
|
||||
OnAudioReceiving?.Invoke(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
// If this is the first packet, start the talking simulation
|
||||
if (packet.sequence == 0 && !_isReceivingAudio)
|
||||
{
|
||||
StartTalkingSimulation();
|
||||
}
|
||||
|
||||
OnAudioReceiving?.Invoke(true);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Not our audio packet format, might be a test message
|
||||
string message = System.Text.Encoding.UTF8.GetString(data);
|
||||
if (enableDebugLogging)
|
||||
ConvaiLogger.Info($"Received test message from {sender}: {message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Error processing received packet: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private void StartTalkingSimulation()
|
||||
{
|
||||
if (_isReceivingAudio) return;
|
||||
|
||||
MainThreadDispatcher.Instance.RunOnMainThread(() => {
|
||||
// Update target NPC if using active NPC
|
||||
if (useActiveNPC)
|
||||
{
|
||||
targetNPC = ConvaiNPCManager.Instance?.GetActiveConvaiNPC();
|
||||
}
|
||||
|
||||
if (targetNPC == null)
|
||||
{
|
||||
ConvaiLogger.Warn("No target NPC available for audio simulation", ConvaiLogger.LogCategory.Character);
|
||||
return;
|
||||
}
|
||||
|
||||
_isReceivingAudio = true;
|
||||
_expectedSequence = 0;
|
||||
|
||||
// This is the KEY! Simulate a talk key press to trigger normal Convai flow
|
||||
ConvaiInputManager.Instance.talkKeyInteract?.Invoke(true);
|
||||
|
||||
ConvaiLogger.Info($"🎤 Started talking simulation for {targetNPC.characterName} (remote player audio)", ConvaiLogger.LogCategory.Character);
|
||||
});
|
||||
}
|
||||
|
||||
private void StopTalkingSimulation()
|
||||
{
|
||||
if (!_isReceivingAudio) return;
|
||||
|
||||
MainThreadDispatcher.Instance.RunOnMainThread(() => {
|
||||
_isReceivingAudio = false;
|
||||
|
||||
// Simulate talk key release to stop recording
|
||||
ConvaiInputManager.Instance.talkKeyInteract?.Invoke(false);
|
||||
|
||||
ConvaiLogger.Info($"🎤 Stopped talking simulation for {targetNPC?.characterName ?? "NPC"} (remote player audio)", ConvaiLogger.LogCategory.Character);
|
||||
});
|
||||
}
|
||||
|
||||
private AudioPacketData? ParseSimpleAudioPacket(byte[] data)
|
||||
{
|
||||
if (data.Length < 24) // Minimum header size
|
||||
return null;
|
||||
|
||||
try
|
||||
{
|
||||
int offset = 0;
|
||||
|
||||
// Read magic number
|
||||
uint magic = BitConverter.ToUInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
if (magic != MAGIC_NUMBER)
|
||||
return null;
|
||||
|
||||
// Read header
|
||||
int sequence = BitConverter.ToInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
int sampleCount = BitConverter.ToInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
int microphonePosition = BitConverter.ToInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
bool isEndSignal = BitConverter.ToBoolean(data, offset);
|
||||
offset += 1;
|
||||
|
||||
// Skip padding
|
||||
offset += 3;
|
||||
|
||||
long timestamp = BitConverter.ToInt64(data, offset);
|
||||
offset += 8;
|
||||
|
||||
// Read audio data
|
||||
short[] audioSamples = null;
|
||||
if (!isEndSignal && sampleCount > 0)
|
||||
{
|
||||
int audioDataSize = sampleCount * sizeof(short);
|
||||
if (data.Length >= offset + audioDataSize)
|
||||
{
|
||||
audioSamples = new short[sampleCount];
|
||||
Buffer.BlockCopy(data, offset, audioSamples, 0, audioDataSize);
|
||||
}
|
||||
}
|
||||
|
||||
return new AudioPacketData
|
||||
{
|
||||
magicNumber = magic,
|
||||
sequence = sequence,
|
||||
sampleCount = sampleCount,
|
||||
microphonePosition = microphonePosition,
|
||||
isEndSignal = isEndSignal,
|
||||
audioSamples = audioSamples,
|
||||
timestamp = timestamp
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Error parsing audio packet: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Event handler for when NPC becomes active
|
||||
private void HandleActiveNPCChanged(ConvaiNPC newActiveNPC)
|
||||
{
|
||||
if (useActiveNPC && newActiveNPC != null)
|
||||
{
|
||||
targetNPC = newActiveNPC;
|
||||
ConvaiLogger.Info($"UDP Audio Receiver V2 updated target NPC to: {targetNPC.characterName}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
// Public properties for debugging
|
||||
public bool IsListening => _isListening;
|
||||
public bool IsReceivingAudio => _isReceivingAudio;
|
||||
public ConvaiNPC TargetNPC => targetNPC;
|
||||
|
||||
// Debug methods
|
||||
public void ShowNetworkStatus()
|
||||
{
|
||||
ConvaiLogger.Info($"=== Audio Receiver V2 Status ===", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Listening: {_isListening} on port {listenPort}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Receiving Audio: {_isReceivingAudio}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Target NPC: {(targetNPC?.characterName ?? "None")}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Expected Sequence: {_expectedSequence}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Last Packet Time: {_lastPacketTime}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user