added multiplayer scripts
This commit is contained in:
tom.hempel
8
Unity-Master/Assets/Scripts/Multiplayer.meta
Normal file
8
Unity-Master/Assets/Scripts/Multiplayer.meta
Normal file
@ -0,0 +1,8 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 7bf18a6dad33703489c04336f7f81a3d
|
||||
folderAsset: yes
|
||||
DefaultImporter:
|
||||
externalObjects: {}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
@ -0,0 +1,364 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Convai.Scripts.Runtime.Core;
|
||||
using Convai.Scripts.Runtime.LoggerSystem;
|
||||
using Convai.Scripts.Runtime.Utils;
|
||||
using UnityEngine;
|
||||
|
||||
namespace Convai.Scripts.Runtime.Multiplayer
|
||||
{
|
||||
/// <summary>
|
||||
/// Simple UDP Audio Receiver V2 - Simulates microphone input by triggering normal Convai flow
|
||||
/// This approach is much simpler and more reliable than trying to replicate gRPC calls
|
||||
/// </summary>
|
||||
public class ConvaiSimpleUDPAudioReceiverV2 : MonoBehaviour
|
||||
{
|
||||
[Header("Network Configuration")]
|
||||
[SerializeField] private int listenPort = 12345;
|
||||
[SerializeField] private bool enableDebugLogging = true;
|
||||
|
||||
[Header("NPC Target")]
|
||||
[SerializeField] private bool useActiveNPC = true;
|
||||
[SerializeField] private ConvaiNPC targetNPC;
|
||||
|
||||
// Events
|
||||
public Action<bool> OnAudioReceiving;
|
||||
|
||||
// Network components
|
||||
private UdpClient _udpListener;
|
||||
private IPEndPoint _remoteEndPoint;
|
||||
private bool _isListening = false;
|
||||
private CancellationTokenSource _cancellationTokenSource;
|
||||
|
||||
// Audio state tracking
|
||||
private bool _isReceivingAudio = false;
|
||||
private int _expectedSequence = 0;
|
||||
private const uint MAGIC_NUMBER = 0xC0A1; // Simple magic number for packet validation
|
||||
|
||||
// Timing for auto-stop
|
||||
private float _lastPacketTime;
|
||||
private const float AUTO_STOP_DELAY = 1.0f; // Stop listening after 1 second of no packets
|
||||
|
||||
// Packet structure (matching ConvaiSimpleUDPAudioSender)
|
||||
private struct AudioPacketData
|
||||
{
|
||||
public uint magicNumber;
|
||||
public int sequence;
|
||||
public int sampleCount;
|
||||
public int microphonePosition;
|
||||
public bool isEndSignal;
|
||||
public short[] audioSamples;
|
||||
public long timestamp;
|
||||
}
|
||||
|
||||
private void Start()
|
||||
{
|
||||
_cancellationTokenSource = new CancellationTokenSource();
|
||||
InitializeNetwork();
|
||||
InitializeConvai();
|
||||
|
||||
// Subscribe to NPC manager events to handle late NPC activation
|
||||
if (ConvaiNPCManager.Instance != null)
|
||||
{
|
||||
ConvaiNPCManager.Instance.OnActiveNPCChanged += HandleActiveNPCChanged;
|
||||
}
|
||||
}
|
||||
|
||||
private void OnDestroy()
|
||||
{
|
||||
// Unsubscribe from events
|
||||
if (ConvaiNPCManager.Instance != null)
|
||||
{
|
||||
ConvaiNPCManager.Instance.OnActiveNPCChanged -= HandleActiveNPCChanged;
|
||||
}
|
||||
|
||||
StopListening();
|
||||
_cancellationTokenSource?.Cancel();
|
||||
_cancellationTokenSource?.Dispose();
|
||||
}
|
||||
|
||||
private void Update()
|
||||
{
|
||||
// Auto-stop listening if no packets received for a while
|
||||
if (_isReceivingAudio && Time.time - _lastPacketTime > AUTO_STOP_DELAY)
|
||||
{
|
||||
StopTalkingSimulation();
|
||||
}
|
||||
}
|
||||
|
||||
private void InitializeNetwork()
|
||||
{
|
||||
try
|
||||
{
|
||||
StartListening();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Failed to initialize UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private void InitializeConvai()
|
||||
{
|
||||
// Get target NPC
|
||||
if (useActiveNPC)
|
||||
{
|
||||
targetNPC = ConvaiNPCManager.Instance?.GetActiveConvaiNPC();
|
||||
}
|
||||
|
||||
if (targetNPC == null)
|
||||
{
|
||||
ConvaiLogger.Warn("No target NPC found yet, will wait for NPC to become active", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
else
|
||||
{
|
||||
ConvaiLogger.Info($"UDP Audio Receiver V2 initialized with NPC: {targetNPC.characterName}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
public void StartListening()
|
||||
{
|
||||
if (_isListening || _cancellationTokenSource == null)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
_udpListener = new UdpClient(listenPort);
|
||||
_isListening = true;
|
||||
|
||||
ConvaiLogger.Info($"Simple UDP Audio Receiver V2 listening on port {listenPort}", ConvaiLogger.LogCategory.Character);
|
||||
|
||||
// Start listening for incoming packets
|
||||
_ = ListenForAudioPackets(_cancellationTokenSource.Token);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Failed to start UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Error($"Stack trace: {ex.StackTrace}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
public void StopListening()
|
||||
{
|
||||
if (!_isListening)
|
||||
return;
|
||||
|
||||
_isListening = false;
|
||||
_udpListener?.Close();
|
||||
_udpListener?.Dispose();
|
||||
_udpListener = null;
|
||||
|
||||
// Stop any ongoing simulation
|
||||
StopTalkingSimulation();
|
||||
|
||||
ConvaiLogger.Info("Stopped UDP Audio Receiver V2", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
|
||||
private async Task ListenForAudioPackets(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
while (_isListening && !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var result = await _udpListener.ReceiveAsync();
|
||||
_remoteEndPoint = result.RemoteEndPoint;
|
||||
|
||||
await ProcessReceivedPacket(result.Buffer, result.RemoteEndPoint);
|
||||
}
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// Normal when stopping
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Error in UDP listener: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessReceivedPacket(byte[] data, IPEndPoint sender)
|
||||
{
|
||||
try
|
||||
{
|
||||
var packetData = ParseSimpleAudioPacket(data);
|
||||
|
||||
if (packetData.HasValue)
|
||||
{
|
||||
var packet = packetData.Value;
|
||||
_lastPacketTime = Time.time;
|
||||
|
||||
if (enableDebugLogging)
|
||||
{
|
||||
if (packet.isEndSignal)
|
||||
ConvaiLogger.DebugLog($"Received end signal from {sender}", ConvaiLogger.LogCategory.Character);
|
||||
else
|
||||
ConvaiLogger.DebugLog($"Received audio packet {packet.sequence} with {packet.sampleCount} samples", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
|
||||
if (packet.isEndSignal)
|
||||
{
|
||||
StopTalkingSimulation();
|
||||
OnAudioReceiving?.Invoke(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
// If this is the first packet, start the talking simulation
|
||||
if (packet.sequence == 0 && !_isReceivingAudio)
|
||||
{
|
||||
StartTalkingSimulation();
|
||||
}
|
||||
|
||||
OnAudioReceiving?.Invoke(true);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Not our audio packet format, might be a test message
|
||||
string message = System.Text.Encoding.UTF8.GetString(data);
|
||||
if (enableDebugLogging)
|
||||
ConvaiLogger.Info($"Received test message from {sender}: {message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Error processing received packet: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private void StartTalkingSimulation()
|
||||
{
|
||||
if (_isReceivingAudio) return;
|
||||
|
||||
MainThreadDispatcher.Instance.RunOnMainThread(() => {
|
||||
// Update target NPC if using active NPC
|
||||
if (useActiveNPC)
|
||||
{
|
||||
targetNPC = ConvaiNPCManager.Instance?.GetActiveConvaiNPC();
|
||||
}
|
||||
|
||||
if (targetNPC == null)
|
||||
{
|
||||
ConvaiLogger.Warn("No target NPC available for audio simulation", ConvaiLogger.LogCategory.Character);
|
||||
return;
|
||||
}
|
||||
|
||||
_isReceivingAudio = true;
|
||||
_expectedSequence = 0;
|
||||
|
||||
// This is the KEY! Simulate a talk key press to trigger normal Convai flow
|
||||
ConvaiInputManager.Instance.talkKeyInteract?.Invoke(true);
|
||||
|
||||
ConvaiLogger.Info($"🎤 Started talking simulation for {targetNPC.characterName} (remote player audio)", ConvaiLogger.LogCategory.Character);
|
||||
});
|
||||
}
|
||||
|
||||
private void StopTalkingSimulation()
|
||||
{
|
||||
if (!_isReceivingAudio) return;
|
||||
|
||||
MainThreadDispatcher.Instance.RunOnMainThread(() => {
|
||||
_isReceivingAudio = false;
|
||||
|
||||
// Simulate talk key release to stop recording
|
||||
ConvaiInputManager.Instance.talkKeyInteract?.Invoke(false);
|
||||
|
||||
ConvaiLogger.Info($"🎤 Stopped talking simulation for {targetNPC?.characterName ?? "NPC"} (remote player audio)", ConvaiLogger.LogCategory.Character);
|
||||
});
|
||||
}
|
||||
|
||||
private AudioPacketData? ParseSimpleAudioPacket(byte[] data)
|
||||
{
|
||||
if (data.Length < 24) // Minimum header size
|
||||
return null;
|
||||
|
||||
try
|
||||
{
|
||||
int offset = 0;
|
||||
|
||||
// Read magic number
|
||||
uint magic = BitConverter.ToUInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
if (magic != MAGIC_NUMBER)
|
||||
return null;
|
||||
|
||||
// Read header
|
||||
int sequence = BitConverter.ToInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
int sampleCount = BitConverter.ToInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
int microphonePosition = BitConverter.ToInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
bool isEndSignal = BitConverter.ToBoolean(data, offset);
|
||||
offset += 1;
|
||||
|
||||
// Skip padding
|
||||
offset += 3;
|
||||
|
||||
long timestamp = BitConverter.ToInt64(data, offset);
|
||||
offset += 8;
|
||||
|
||||
// Read audio data
|
||||
short[] audioSamples = null;
|
||||
if (!isEndSignal && sampleCount > 0)
|
||||
{
|
||||
int audioDataSize = sampleCount * sizeof(short);
|
||||
if (data.Length >= offset + audioDataSize)
|
||||
{
|
||||
audioSamples = new short[sampleCount];
|
||||
Buffer.BlockCopy(data, offset, audioSamples, 0, audioDataSize);
|
||||
}
|
||||
}
|
||||
|
||||
return new AudioPacketData
|
||||
{
|
||||
magicNumber = magic,
|
||||
sequence = sequence,
|
||||
sampleCount = sampleCount,
|
||||
microphonePosition = microphonePosition,
|
||||
isEndSignal = isEndSignal,
|
||||
audioSamples = audioSamples,
|
||||
timestamp = timestamp
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Error parsing audio packet: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Event handler for when NPC becomes active
|
||||
private void HandleActiveNPCChanged(ConvaiNPC newActiveNPC)
|
||||
{
|
||||
if (useActiveNPC && newActiveNPC != null)
|
||||
{
|
||||
targetNPC = newActiveNPC;
|
||||
ConvaiLogger.Info($"UDP Audio Receiver V2 updated target NPC to: {targetNPC.characterName}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
// Public properties for debugging
|
||||
public bool IsListening => _isListening;
|
||||
public bool IsReceivingAudio => _isReceivingAudio;
|
||||
public ConvaiNPC TargetNPC => targetNPC;
|
||||
|
||||
// Debug methods
|
||||
public void ShowNetworkStatus()
|
||||
{
|
||||
ConvaiLogger.Info($"=== Audio Receiver V2 Status ===", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Listening: {_isListening} on port {listenPort}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Receiving Audio: {_isReceivingAudio}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Target NPC: {(targetNPC?.characterName ?? "None")}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Expected Sequence: {_expectedSequence}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Last Packet Time: {_lastPacketTime}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: fa35a6fc55fc4ca44b29b3636484bfd2
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
@ -0,0 +1,376 @@
|
||||
using System;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Convai.Scripts.Runtime.LoggerSystem;
|
||||
using Convai.Scripts.Runtime.UI;
|
||||
using UnityEngine;
|
||||
|
||||
namespace Convai.Scripts.Runtime.Multiplayer
|
||||
{
|
||||
/// <summary>
|
||||
/// Simplified version of UDP Audio Sender that avoids complex chunking
|
||||
/// This version sends smaller, more frequent packets to avoid array bounds issues
|
||||
/// </summary>
|
||||
public class ConvaiSimpleUDPAudioSender : MonoBehaviour
|
||||
{
|
||||
[Header("Network Settings")]
|
||||
[SerializeField] private string targetIP = "127.0.0.1";
|
||||
[SerializeField] private int targetPort = 12345;
|
||||
|
||||
[Header("Audio Settings")]
|
||||
[SerializeField] private int recordingFrequency = 16000;
|
||||
[SerializeField] private int recordingLength = 10;
|
||||
[SerializeField] private int samplesPerPacket = 1024; // Number of audio samples per packet (not bytes)
|
||||
|
||||
[Header("UI")]
|
||||
[SerializeField] private KeyCode talkKey = KeyCode.T;
|
||||
[SerializeField] private bool useHoldToTalk = true;
|
||||
|
||||
[Header("Debug")]
|
||||
[SerializeField] private bool enableDebugLogging = true;
|
||||
[SerializeField] private KeyCode testConnectionKey = KeyCode.C;
|
||||
|
||||
private UdpClient _udpClient;
|
||||
private IPEndPoint _targetEndPoint;
|
||||
private AudioClip _audioClip;
|
||||
private bool _isRecording = false;
|
||||
private CancellationTokenSource _cancellationTokenSource;
|
||||
|
||||
private int _lastMicrophonePosition = 0;
|
||||
private float[] _audioBuffer;
|
||||
private string _selectedMicrophone;
|
||||
private int _packetSequence = 0;
|
||||
|
||||
public event Action<bool> OnRecordingStateChanged;
|
||||
|
||||
private void Start()
|
||||
{
|
||||
InitializeNetwork();
|
||||
InitializeAudio();
|
||||
_cancellationTokenSource = new CancellationTokenSource();
|
||||
}
|
||||
|
||||
private void Update()
|
||||
{
|
||||
HandleInput();
|
||||
}
|
||||
|
||||
private void OnDestroy()
|
||||
{
|
||||
StopRecording();
|
||||
_cancellationTokenSource?.Cancel();
|
||||
_cancellationTokenSource?.Dispose();
|
||||
_udpClient?.Close();
|
||||
}
|
||||
|
||||
private void InitializeNetwork()
|
||||
{
|
||||
try
|
||||
{
|
||||
_udpClient = new UdpClient();
|
||||
_targetEndPoint = new IPEndPoint(IPAddress.Parse(targetIP), targetPort);
|
||||
ConvaiLogger.Info($"Simple UDP Audio Sender initialized. Target: {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Failed to initialize UDP client: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private void InitializeAudio()
|
||||
{
|
||||
_selectedMicrophone = MicrophoneManager.Instance.SelectedMicrophoneName;
|
||||
_audioBuffer = new float[recordingFrequency * recordingLength];
|
||||
|
||||
if (string.IsNullOrEmpty(_selectedMicrophone))
|
||||
{
|
||||
ConvaiLogger.Error("No microphone selected for UDP audio sender", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleInput()
|
||||
{
|
||||
// Handle talk key
|
||||
if (useHoldToTalk)
|
||||
{
|
||||
if (Input.GetKeyDown(talkKey) && !_isRecording)
|
||||
{
|
||||
StartRecording();
|
||||
}
|
||||
else if (Input.GetKeyUp(talkKey) && _isRecording)
|
||||
{
|
||||
StopRecording();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (Input.GetKeyDown(talkKey))
|
||||
{
|
||||
if (_isRecording)
|
||||
StopRecording();
|
||||
else
|
||||
StartRecording();
|
||||
}
|
||||
}
|
||||
|
||||
// Handle test connection key
|
||||
if (Input.GetKeyDown(testConnectionKey))
|
||||
{
|
||||
TestConnection();
|
||||
}
|
||||
}
|
||||
|
||||
public void StartRecording()
|
||||
{
|
||||
if (_isRecording || string.IsNullOrEmpty(_selectedMicrophone))
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
_audioClip = Microphone.Start(_selectedMicrophone, false, recordingLength, recordingFrequency);
|
||||
_isRecording = true;
|
||||
_lastMicrophonePosition = 0;
|
||||
_packetSequence = 0;
|
||||
|
||||
ConvaiLogger.Info("Started recording for UDP transmission (Simple)", ConvaiLogger.LogCategory.Character);
|
||||
OnRecordingStateChanged?.Invoke(true);
|
||||
|
||||
// Start continuous audio processing
|
||||
_ = ProcessAudioContinuously(_cancellationTokenSource.Token);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Failed to start recording: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
public void StopRecording()
|
||||
{
|
||||
if (!_isRecording)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
Microphone.End(_selectedMicrophone);
|
||||
_isRecording = false;
|
||||
|
||||
ConvaiLogger.Info("Stopped recording for UDP transmission (Simple)", ConvaiLogger.LogCategory.Character);
|
||||
OnRecordingStateChanged?.Invoke(false);
|
||||
|
||||
// Send end-of-recording signal
|
||||
SendEndOfRecordingSignal();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Failed to stop recording: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessAudioContinuously(CancellationToken cancellationToken)
|
||||
{
|
||||
while (_isRecording && !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(100, cancellationToken); // Process every 100ms
|
||||
|
||||
if (_audioClip == null || !Microphone.IsRecording(_selectedMicrophone))
|
||||
break;
|
||||
|
||||
int currentMicrophonePosition = Microphone.GetPosition(_selectedMicrophone);
|
||||
int audioDataLength = currentMicrophonePosition - _lastMicrophonePosition;
|
||||
|
||||
if (audioDataLength > 0)
|
||||
{
|
||||
// Get audio data from the microphone clip
|
||||
_audioClip.GetData(_audioBuffer, _lastMicrophonePosition);
|
||||
|
||||
// Send data in smaller chunks to avoid array bounds issues
|
||||
await SendAudioDataInChunks(_audioBuffer, audioDataLength);
|
||||
|
||||
_lastMicrophonePosition = currentMicrophonePosition;
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (!(ex is OperationCanceledException))
|
||||
{
|
||||
ConvaiLogger.Error($"Error in audio processing: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SendAudioDataInChunks(float[] audioData, int totalSamples)
|
||||
{
|
||||
int processedSamples = 0;
|
||||
|
||||
while (processedSamples < totalSamples)
|
||||
{
|
||||
try
|
||||
{
|
||||
int remainingSamples = totalSamples - processedSamples;
|
||||
int currentChunkSamples = Mathf.Min(samplesPerPacket, remainingSamples);
|
||||
|
||||
// Create a simple packet structure
|
||||
byte[] packet = CreateSimpleAudioPacket(audioData, processedSamples, currentChunkSamples);
|
||||
|
||||
// Send the packet
|
||||
await _udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
|
||||
|
||||
if (enableDebugLogging && _packetSequence % 10 == 0) // Log every 10th packet
|
||||
{
|
||||
ConvaiLogger.DebugLog($"Sent packet {_packetSequence} with {currentChunkSamples} samples", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
|
||||
processedSamples += currentChunkSamples;
|
||||
_packetSequence++;
|
||||
|
||||
// Small delay to avoid overwhelming the network
|
||||
await Task.Delay(10);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Failed to send audio chunk: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] CreateSimpleAudioPacket(float[] audioData, int startIndex, int sampleCount)
|
||||
{
|
||||
// Simple packet structure:
|
||||
// 4 bytes: Magic number (0xC0A1)
|
||||
// 4 bytes: Packet sequence number
|
||||
// 4 bytes: Sample count in this packet
|
||||
// 4 bytes: Start position in stream
|
||||
// 1 byte: Flags (0 = normal audio, 1 = end of recording)
|
||||
// N bytes: Audio data (converted to shorts)
|
||||
|
||||
int headerSize = 17; // 4 + 4 + 4 + 4 + 1
|
||||
int audioDataSize = sampleCount * sizeof(short);
|
||||
byte[] packet = new byte[headerSize + audioDataSize];
|
||||
|
||||
int offset = 0;
|
||||
|
||||
// Magic number
|
||||
BitConverter.GetBytes((uint)0xC0A1).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
// Packet sequence
|
||||
BitConverter.GetBytes(_packetSequence).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
// Sample count
|
||||
BitConverter.GetBytes(sampleCount).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
// Start position
|
||||
BitConverter.GetBytes(_lastMicrophonePosition + startIndex).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
// Flags (0 for normal audio)
|
||||
packet[offset] = 0;
|
||||
offset += 1;
|
||||
|
||||
// Convert audio samples to bytes (same as Convai approach)
|
||||
for (int i = 0; i < sampleCount; i++)
|
||||
{
|
||||
float sample = audioData[startIndex + i];
|
||||
short shortSample = (short)(sample * short.MaxValue);
|
||||
byte[] shortBytes = BitConverter.GetBytes(shortSample);
|
||||
packet[offset] = shortBytes[0];
|
||||
packet[offset + 1] = shortBytes[1];
|
||||
offset += 2;
|
||||
}
|
||||
|
||||
return packet;
|
||||
}
|
||||
|
||||
private void SendEndOfRecordingSignal()
|
||||
{
|
||||
try
|
||||
{
|
||||
// Create end packet
|
||||
byte[] packet = new byte[17]; // Header only, no audio data
|
||||
int offset = 0;
|
||||
|
||||
// Magic number
|
||||
BitConverter.GetBytes((uint)0xC0A1).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
// Packet sequence
|
||||
BitConverter.GetBytes(_packetSequence).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
// Sample count (0 for end signal)
|
||||
BitConverter.GetBytes(0).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
// Start position
|
||||
BitConverter.GetBytes(_lastMicrophonePosition).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
// Flags (1 for end of recording)
|
||||
packet[offset] = 1;
|
||||
|
||||
_udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Failed to send end signal: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
// Public methods for external control
|
||||
public void SetTargetEndpoint(string ip, int port)
|
||||
{
|
||||
targetIP = ip;
|
||||
targetPort = port;
|
||||
_targetEndPoint = new IPEndPoint(IPAddress.Parse(ip), port);
|
||||
}
|
||||
|
||||
public bool IsRecording => _isRecording;
|
||||
|
||||
// Debug and testing methods
|
||||
public async void TestConnection()
|
||||
{
|
||||
if (_udpClient == null)
|
||||
{
|
||||
ConvaiLogger.Error("UDP client not initialized", ConvaiLogger.LogCategory.Character);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
ConvaiLogger.Info($"Testing connection to {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
|
||||
|
||||
// Send a simple test packet
|
||||
string testMessage = "CONVAI_TEST_CONNECTION";
|
||||
byte[] testData = System.Text.Encoding.UTF8.GetBytes(testMessage);
|
||||
|
||||
await _udpClient.SendAsync(testData, testData.Length, _targetEndPoint);
|
||||
ConvaiLogger.Info("Test packet sent successfully", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Connection test failed: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
public void ShowNetworkStatus()
|
||||
{
|
||||
ConvaiLogger.Info($"=== Network Status ===", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Target: {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"UDP Client: {(_udpClient != null ? "Initialized" : "Not initialized")}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Recording: {_isRecording}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Microphone: {_selectedMicrophone}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Packets sent: {_packetSequence}", ConvaiLogger.LogCategory.Character);
|
||||
|
||||
if (_udpClient?.Client?.LocalEndPoint != null)
|
||||
{
|
||||
ConvaiLogger.Info($"Local endpoint: {_udpClient.Client.LocalEndPoint}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: fa5cc94311721d04f8e8821151ffb737
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
@ -0,0 +1,639 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Convai.Scripts.Runtime.LoggerSystem;
|
||||
using Convai.Scripts.Runtime.Utils;
|
||||
using UnityEngine;
|
||||
|
||||
namespace Convai.Scripts.Runtime.Multiplayer
|
||||
{
|
||||
/// <summary>
|
||||
/// UDP Speech Receiver - Receives high-quality Convai speech with proper buffering
|
||||
/// This version reconstructs the original AudioClip objects for seamless playback
|
||||
/// </summary>
|
||||
public class ConvaiUDPSpeechReceiver : MonoBehaviour
|
||||
{
|
||||
[Header("Network Configuration")]
|
||||
[SerializeField] private int listenPort = 12346;
|
||||
[SerializeField] private bool enableDebugLogging = true;
|
||||
|
||||
[Header("Audio Playback")]
|
||||
[SerializeField] private AudioSource speechAudioSource;
|
||||
[SerializeField] private bool createAudioSourceIfMissing = true;
|
||||
[SerializeField] private float audioVolume = 1.0f;
|
||||
[SerializeField] private bool spatialAudio = false;
|
||||
|
||||
[Header("UI")]
|
||||
[SerializeField] private bool showTranscripts = true;
|
||||
|
||||
// Network components
|
||||
private UdpClient _udpListener;
|
||||
private IPEndPoint _remoteEndPoint;
|
||||
private bool _isListening = false;
|
||||
private CancellationTokenSource _cancellationTokenSource;
|
||||
|
||||
// Audio reconstruction
|
||||
private Dictionary<int, IncomingAudioClip> _incomingClips = new Dictionary<int, IncomingAudioClip>();
|
||||
private Queue<ReconstructedAudioClip> _playbackQueue = new Queue<ReconstructedAudioClip>();
|
||||
private bool _isPlayingSequence = false;
|
||||
private int _currentSequence = 0;
|
||||
|
||||
// Packet constants (matching sender V3)
|
||||
private const uint MAGIC_NUMBER = 0xC0A3;
|
||||
private const byte PACKET_TYPE_AUDIO_START = 0x01;
|
||||
private const byte PACKET_TYPE_AUDIO_CHUNK = 0x02;
|
||||
private const byte PACKET_TYPE_AUDIO_END = 0x03;
|
||||
private const byte PACKET_TYPE_TRANSCRIPT = 0x04;
|
||||
private const byte PACKET_TYPE_FINAL = 0x05;
|
||||
|
||||
// Events
|
||||
public Action<bool> OnSpeechReceiving;
|
||||
public Action<string> OnTranscriptReceived;
|
||||
public Action<AudioClip> OnAudioClipReceived;
|
||||
|
||||
// Data structures
|
||||
private struct SpeechPacket
|
||||
{
|
||||
public uint magicNumber;
|
||||
public byte packetType;
|
||||
public int sequence;
|
||||
public int totalSamples;
|
||||
public int sampleRate;
|
||||
public int channels;
|
||||
public int startSample;
|
||||
public int chunkSampleCount;
|
||||
public short[] audioSamples;
|
||||
public string transcript;
|
||||
}
|
||||
|
||||
private class IncomingAudioClip
|
||||
{
|
||||
public int totalSamples;
|
||||
public int sampleRate;
|
||||
public int channels;
|
||||
public string transcript;
|
||||
public float[] audioData;
|
||||
public bool isComplete;
|
||||
public bool hasStart;
|
||||
public bool hasEnd;
|
||||
public int receivedSamples;
|
||||
|
||||
public IncomingAudioClip(int totalSamples, int sampleRate, int channels, string transcript)
|
||||
{
|
||||
this.totalSamples = totalSamples;
|
||||
this.sampleRate = sampleRate;
|
||||
this.channels = channels;
|
||||
this.transcript = transcript;
|
||||
this.audioData = new float[totalSamples];
|
||||
this.isComplete = false;
|
||||
this.hasStart = false;
|
||||
this.hasEnd = false;
|
||||
this.receivedSamples = 0;
|
||||
}
|
||||
}
|
||||
|
||||
private struct ReconstructedAudioClip
|
||||
{
|
||||
public AudioClip audioClip;
|
||||
public string transcript;
|
||||
public bool isFinal;
|
||||
}
|
||||
|
||||
private void Start()
|
||||
{
|
||||
_cancellationTokenSource = new CancellationTokenSource();
|
||||
InitializeAudio();
|
||||
InitializeNetwork();
|
||||
}
|
||||
|
||||
private void OnDestroy()
|
||||
{
|
||||
StopListening();
|
||||
_cancellationTokenSource?.Cancel();
|
||||
_cancellationTokenSource?.Dispose();
|
||||
}
|
||||
|
||||
private void Update()
|
||||
{
|
||||
// Process playback queue
|
||||
ProcessPlaybackQueue();
|
||||
}
|
||||
|
||||
private void InitializeAudio()
|
||||
{
|
||||
if (speechAudioSource == null)
|
||||
{
|
||||
speechAudioSource = GetComponent<AudioSource>();
|
||||
|
||||
if (speechAudioSource == null && createAudioSourceIfMissing)
|
||||
{
|
||||
speechAudioSource = gameObject.AddComponent<AudioSource>();
|
||||
ConvaiLogger.Info("Created AudioSource for speech playback", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
if (speechAudioSource != null)
|
||||
{
|
||||
speechAudioSource.volume = audioVolume;
|
||||
speechAudioSource.playOnAwake = false;
|
||||
speechAudioSource.spatialBlend = spatialAudio ? 1.0f : 0.0f;
|
||||
}
|
||||
else
|
||||
{
|
||||
ConvaiLogger.Error("No AudioSource available for speech playback", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private void InitializeNetwork()
|
||||
{
|
||||
try
|
||||
{
|
||||
StartListening();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Failed to initialize UDP speech receiver: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
public void StartListening()
|
||||
{
|
||||
if (_isListening || _cancellationTokenSource == null)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
_udpListener = new UdpClient(listenPort);
|
||||
_isListening = true;
|
||||
|
||||
ConvaiLogger.Info($"UDP Speech Receiver listening on port {listenPort}", ConvaiLogger.LogCategory.Character);
|
||||
|
||||
// Start listening for incoming packets
|
||||
_ = ListenForSpeechPackets(_cancellationTokenSource.Token);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Failed to start UDP speech receiver: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
public void StopListening()
|
||||
{
|
||||
if (!_isListening)
|
||||
return;
|
||||
|
||||
_isListening = false;
|
||||
_udpListener?.Close();
|
||||
_udpListener?.Dispose();
|
||||
_udpListener = null;
|
||||
|
||||
// Stop any ongoing playback
|
||||
StopSpeechPlayback();
|
||||
|
||||
ConvaiLogger.Info("Stopped UDP Speech Receiver", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
|
||||
private async Task ListenForSpeechPackets(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
while (_isListening && !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var result = await _udpListener.ReceiveAsync();
|
||||
_remoteEndPoint = result.RemoteEndPoint;
|
||||
|
||||
await ProcessReceivedPacket(result.Buffer, result.RemoteEndPoint);
|
||||
}
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// Normal when stopping
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Error in UDP speech listener: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessReceivedPacket(byte[] data, IPEndPoint sender)
|
||||
{
|
||||
try
|
||||
{
|
||||
var packetData = ParseSpeechPacket(data);
|
||||
|
||||
if (packetData.HasValue)
|
||||
{
|
||||
var packet = packetData.Value;
|
||||
|
||||
if (enableDebugLogging)
|
||||
{
|
||||
string typeStr = packet.packetType switch
|
||||
{
|
||||
PACKET_TYPE_AUDIO_START => "start",
|
||||
PACKET_TYPE_AUDIO_CHUNK => "chunk",
|
||||
PACKET_TYPE_AUDIO_END => "end",
|
||||
PACKET_TYPE_TRANSCRIPT => "transcript",
|
||||
PACKET_TYPE_FINAL => "final",
|
||||
_ => "unknown"
|
||||
};
|
||||
ConvaiLogger.DebugLog($"📥 Received {typeStr} packet {packet.sequence} from {sender}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
|
||||
switch (packet.packetType)
|
||||
{
|
||||
case PACKET_TYPE_AUDIO_START:
|
||||
HandleAudioStartPacket(packet);
|
||||
break;
|
||||
|
||||
case PACKET_TYPE_AUDIO_CHUNK:
|
||||
HandleAudioChunkPacket(packet);
|
||||
break;
|
||||
|
||||
case PACKET_TYPE_AUDIO_END:
|
||||
HandleAudioEndPacket(packet);
|
||||
break;
|
||||
|
||||
case PACKET_TYPE_TRANSCRIPT:
|
||||
HandleTranscriptPacket(packet);
|
||||
break;
|
||||
|
||||
case PACKET_TYPE_FINAL:
|
||||
HandleFinalPacket();
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (enableDebugLogging)
|
||||
{
|
||||
// Check if it's a different magic number
|
||||
if (data.Length >= 4)
|
||||
{
|
||||
uint receivedMagic = BitConverter.ToUInt32(data, 0);
|
||||
ConvaiLogger.Warn($"❌ Invalid speech packet from {sender}. Expected magic: 0x{MAGIC_NUMBER:X}, Got: 0x{receivedMagic:X}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
else
|
||||
{
|
||||
ConvaiLogger.Warn($"❌ Packet too small from {sender}: {data.Length} bytes", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Error processing speech packet: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleAudioStartPacket(SpeechPacket packet)
|
||||
{
|
||||
// Start new speech sequence if this is the first start packet
|
||||
if (packet.sequence == 0 && !_isPlayingSequence)
|
||||
{
|
||||
StartSpeechReception();
|
||||
}
|
||||
|
||||
// Create new incoming audio clip
|
||||
var incomingClip = new IncomingAudioClip(packet.totalSamples, packet.sampleRate, packet.channels, packet.transcript);
|
||||
incomingClip.hasStart = true;
|
||||
|
||||
_incomingClips[packet.sequence] = incomingClip;
|
||||
|
||||
if (enableDebugLogging)
|
||||
ConvaiLogger.DebugLog($"🎵 Started receiving audio clip {packet.sequence}: {packet.totalSamples} samples, '{packet.transcript}'", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
|
||||
private void HandleAudioChunkPacket(SpeechPacket packet)
|
||||
{
|
||||
if (!_incomingClips.ContainsKey(packet.sequence)) return;
|
||||
|
||||
var incomingClip = _incomingClips[packet.sequence];
|
||||
|
||||
// Convert short samples back to float and copy to the correct position
|
||||
if (packet.audioSamples != null && packet.startSample + packet.chunkSampleCount <= incomingClip.totalSamples)
|
||||
{
|
||||
for (int i = 0; i < packet.chunkSampleCount; i++)
|
||||
{
|
||||
int targetIndex = packet.startSample + i;
|
||||
if (targetIndex < incomingClip.audioData.Length)
|
||||
{
|
||||
incomingClip.audioData[targetIndex] = packet.audioSamples[i] / (float)short.MaxValue;
|
||||
}
|
||||
}
|
||||
|
||||
incomingClip.receivedSamples += packet.chunkSampleCount;
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleAudioEndPacket(SpeechPacket packet)
|
||||
{
|
||||
if (!_incomingClips.ContainsKey(packet.sequence)) return;
|
||||
|
||||
var incomingClip = _incomingClips[packet.sequence];
|
||||
incomingClip.hasEnd = true;
|
||||
|
||||
// Check if the clip is complete (has start, end, and all samples)
|
||||
if (incomingClip.hasStart && incomingClip.hasEnd)
|
||||
{
|
||||
incomingClip.isComplete = true;
|
||||
|
||||
// Create the AudioClip
|
||||
CreateAndQueueAudioClip(incomingClip, packet.sequence);
|
||||
|
||||
// Remove from incoming clips
|
||||
_incomingClips.Remove(packet.sequence);
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleTranscriptPacket(SpeechPacket packet)
|
||||
{
|
||||
if (showTranscripts && !string.IsNullOrEmpty(packet.transcript))
|
||||
{
|
||||
MainThreadDispatcher.Instance.RunOnMainThread(() => {
|
||||
OnTranscriptReceived?.Invoke(packet.transcript);
|
||||
|
||||
if (enableDebugLogging)
|
||||
ConvaiLogger.Info($"📝 Remote NPC said: '{packet.transcript}'", ConvaiLogger.LogCategory.Character);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleFinalPacket()
|
||||
{
|
||||
// Process any remaining incomplete clips
|
||||
ProcessIncompleteClips();
|
||||
|
||||
// Add final marker to queue
|
||||
_playbackQueue.Enqueue(new ReconstructedAudioClip
|
||||
{
|
||||
audioClip = null,
|
||||
transcript = "",
|
||||
isFinal = true
|
||||
});
|
||||
|
||||
StopSpeechReception();
|
||||
}
|
||||
|
||||
private void ProcessIncompleteClips()
|
||||
{
|
||||
// Try to create AudioClips from any clips that might be mostly complete
|
||||
var keysToRemove = new List<int>();
|
||||
|
||||
foreach (var kvp in _incomingClips)
|
||||
{
|
||||
var incomingClip = kvp.Value;
|
||||
|
||||
// If we received a reasonable amount of data, try to create the clip
|
||||
if (incomingClip.receivedSamples > incomingClip.totalSamples * 0.8f) // 80% received
|
||||
{
|
||||
CreateAndQueueAudioClip(incomingClip, kvp.Key);
|
||||
keysToRemove.Add(kvp.Key);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var key in keysToRemove)
|
||||
{
|
||||
_incomingClips.Remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
private void CreateAndQueueAudioClip(IncomingAudioClip incomingClip, int sequence)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Create AudioClip
|
||||
AudioClip clip = AudioClip.Create($"RemoteSpeech_{sequence}",
|
||||
incomingClip.totalSamples, incomingClip.channels, incomingClip.sampleRate, false);
|
||||
clip.SetData(incomingClip.audioData, 0);
|
||||
|
||||
// Queue for playback
|
||||
_playbackQueue.Enqueue(new ReconstructedAudioClip
|
||||
{
|
||||
audioClip = clip,
|
||||
transcript = incomingClip.transcript,
|
||||
isFinal = false
|
||||
});
|
||||
|
||||
OnAudioClipReceived?.Invoke(clip);
|
||||
|
||||
if (enableDebugLogging)
|
||||
ConvaiLogger.DebugLog($"✅ Reconstructed audio clip {sequence}: {clip.length:F2}s, '{incomingClip.transcript}'", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Failed to create audio clip from sequence {sequence}: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private void ProcessPlaybackQueue()
|
||||
{
|
||||
// If not currently playing and we have queued clips, start playing
|
||||
if (!_isPlayingSequence && _playbackQueue.Count > 0 && speechAudioSource != null)
|
||||
{
|
||||
PlayNextAudioClip();
|
||||
}
|
||||
|
||||
// Check if current clip finished playing
|
||||
if (_isPlayingSequence && speechAudioSource != null && !speechAudioSource.isPlaying)
|
||||
{
|
||||
// Current clip finished, play next one if available
|
||||
if (_playbackQueue.Count > 0)
|
||||
{
|
||||
PlayNextAudioClip();
|
||||
}
|
||||
else
|
||||
{
|
||||
_isPlayingSequence = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void PlayNextAudioClip()
|
||||
{
|
||||
if (_playbackQueue.Count == 0 || speechAudioSource == null) return;
|
||||
|
||||
var reconstructedClip = _playbackQueue.Dequeue();
|
||||
|
||||
if (reconstructedClip.isFinal)
|
||||
{
|
||||
_isPlayingSequence = false;
|
||||
ConvaiLogger.Info("🔊 Finished playing remote speech sequence", ConvaiLogger.LogCategory.Character);
|
||||
return;
|
||||
}
|
||||
|
||||
if (reconstructedClip.audioClip != null)
|
||||
{
|
||||
speechAudioSource.clip = reconstructedClip.audioClip;
|
||||
speechAudioSource.Play();
|
||||
_isPlayingSequence = true;
|
||||
|
||||
if (enableDebugLogging)
|
||||
ConvaiLogger.DebugLog($"🔊 Playing remote speech: {reconstructedClip.audioClip.length:F2}s, '{reconstructedClip.transcript}'", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private void StartSpeechReception()
|
||||
{
|
||||
_isPlayingSequence = false;
|
||||
_currentSequence = 0;
|
||||
_incomingClips.Clear();
|
||||
_playbackQueue.Clear();
|
||||
|
||||
OnSpeechReceiving?.Invoke(true);
|
||||
|
||||
ConvaiLogger.Info("🔊 Started receiving remote NPC speech", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
|
||||
private void StopSpeechReception()
|
||||
{
|
||||
OnSpeechReceiving?.Invoke(false);
|
||||
|
||||
ConvaiLogger.Info("🔊 Stopped receiving remote NPC speech", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
|
||||
private void StopSpeechPlayback()
|
||||
{
|
||||
if (speechAudioSource != null && speechAudioSource.isPlaying)
|
||||
{
|
||||
speechAudioSource.Stop();
|
||||
}
|
||||
|
||||
_isPlayingSequence = false;
|
||||
_playbackQueue.Clear();
|
||||
_incomingClips.Clear();
|
||||
}
|
||||
|
||||
private SpeechPacket? ParseSpeechPacket(byte[] data)
|
||||
{
|
||||
if (data.Length < 13) // Minimum header size
|
||||
return null;
|
||||
|
||||
try
|
||||
{
|
||||
int offset = 0;
|
||||
|
||||
// Read magic number
|
||||
uint magic = BitConverter.ToUInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
if (magic != MAGIC_NUMBER)
|
||||
return null;
|
||||
|
||||
// Read packet type
|
||||
byte packetType = data[offset];
|
||||
offset += 1;
|
||||
|
||||
// Read sequence
|
||||
int sequence = BitConverter.ToInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
var packet = new SpeechPacket
|
||||
{
|
||||
magicNumber = magic,
|
||||
packetType = packetType,
|
||||
sequence = sequence
|
||||
};
|
||||
|
||||
// Parse based on packet type
|
||||
switch (packetType)
|
||||
{
|
||||
case PACKET_TYPE_AUDIO_START:
|
||||
if (data.Length < offset + 16) return null; // Need additional fields
|
||||
|
||||
packet.totalSamples = BitConverter.ToInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
packet.sampleRate = BitConverter.ToInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
packet.channels = BitConverter.ToInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
int transcriptLength = BitConverter.ToInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
if (transcriptLength > 0 && data.Length >= offset + transcriptLength)
|
||||
{
|
||||
packet.transcript = System.Text.Encoding.UTF8.GetString(data, offset, transcriptLength);
|
||||
}
|
||||
break;
|
||||
|
||||
case PACKET_TYPE_AUDIO_CHUNK:
|
||||
if (data.Length < offset + 8) return null; // Need start sample + count
|
||||
|
||||
packet.startSample = BitConverter.ToInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
packet.chunkSampleCount = BitConverter.ToInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
// Read audio data
|
||||
if (packet.chunkSampleCount > 0 && data.Length >= offset + packet.chunkSampleCount * 2)
|
||||
{
|
||||
packet.audioSamples = new short[packet.chunkSampleCount];
|
||||
for (int i = 0; i < packet.chunkSampleCount; i++)
|
||||
{
|
||||
packet.audioSamples[i] = BitConverter.ToInt16(data, offset);
|
||||
offset += 2;
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case PACKET_TYPE_AUDIO_END:
|
||||
case PACKET_TYPE_FINAL:
|
||||
// These packets have no additional data beyond the header
|
||||
break;
|
||||
|
||||
case PACKET_TYPE_TRANSCRIPT:
|
||||
// Similar to start packet transcript handling
|
||||
if (data.Length >= offset + 4)
|
||||
{
|
||||
int transcriptLen = BitConverter.ToInt32(data, offset);
|
||||
offset += 4;
|
||||
|
||||
if (transcriptLen > 0 && data.Length >= offset + transcriptLen)
|
||||
{
|
||||
packet.transcript = System.Text.Encoding.UTF8.GetString(data, offset, transcriptLen);
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
|
||||
return packet;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Error parsing speech packet V2: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Public properties for debugging
|
||||
public bool IsListening => _isListening;
|
||||
public bool IsPlayingSequence => _isPlayingSequence;
|
||||
public int QueuedClipCount => _playbackQueue.Count;
|
||||
public int IncomingClipCount => _incomingClips.Count;
|
||||
|
||||
// Debug methods
|
||||
public void ShowNetworkStatus()
|
||||
{
|
||||
ConvaiLogger.Info($"=== Speech Receiver Status ===", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Listening: {_isListening} on port {listenPort}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Playing Sequence: {_isPlayingSequence}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Current Sequence: {_currentSequence}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Queued Clips: {_playbackQueue.Count}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Incoming Clips: {_incomingClips.Count}", ConvaiLogger.LogCategory.Character);
|
||||
|
||||
if (speechAudioSource != null)
|
||||
{
|
||||
ConvaiLogger.Info($"Audio Source: {speechAudioSource.name} (Volume: {speechAudioSource.volume})", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 8fe3e5348bc484f44be079d6aaf6e17e
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
479
Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechSender.cs
Normal file
479
Unity-Master/Assets/Scripts/Multiplayer/ConvaiUDPSpeechSender.cs
Normal file
@ -0,0 +1,479 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Threading.Tasks;
|
||||
using Convai.Scripts.Runtime.Core;
|
||||
using Convai.Scripts.Runtime.LoggerSystem;
|
||||
using Convai.Scripts.Runtime.Utils;
|
||||
using UnityEngine;
|
||||
using System.Collections;
|
||||
|
||||
namespace Convai.Scripts.Runtime.Multiplayer
|
||||
{
|
||||
/// <summary>
|
||||
/// UDP Speech Sender - Simple and reliable approach using events
|
||||
/// Hooks into AudioManager events to capture when clips are about to be played
|
||||
/// </summary>
|
||||
public class ConvaiUDPSpeechSender : MonoBehaviour
|
||||
{
|
||||
[Header("Network Configuration")]
|
||||
[SerializeField] private string targetIP = "127.0.0.1";
|
||||
[SerializeField] private int targetPort = 12346;
|
||||
[SerializeField] private bool enableDebugLogging = true;
|
||||
|
||||
[Header("NPC Source")]
|
||||
[SerializeField] private bool useActiveNPC = true;
|
||||
[SerializeField] private ConvaiNPC sourceNPC;
|
||||
|
||||
[Header("Audio Settings")]
|
||||
[SerializeField] private int maxSamplesPerPacket = 8192;
|
||||
[SerializeField] private bool sendTranscripts = true;
|
||||
|
||||
// Network components
|
||||
private UdpClient _udpClient;
|
||||
private IPEndPoint _targetEndPoint;
|
||||
private bool _isInitialized = false;
|
||||
|
||||
// Speech tracking
|
||||
private int _speechSequence = 0;
|
||||
private bool _isSendingSpeech = false;
|
||||
private HashSet<AudioClip> _sentClips = new HashSet<AudioClip>();
|
||||
|
||||
// Packet constants
|
||||
private const uint MAGIC_NUMBER = 0xC0A3; // V3 magic number
|
||||
private const byte PACKET_TYPE_AUDIO_START = 0x01;
|
||||
private const byte PACKET_TYPE_AUDIO_CHUNK = 0x02;
|
||||
private const byte PACKET_TYPE_AUDIO_END = 0x03;
|
||||
private const byte PACKET_TYPE_FINAL = 0x05;
|
||||
|
||||
// Events
|
||||
public Action<bool> OnSpeechTransmission;
|
||||
public Action<string> OnSpeechSent;
|
||||
|
||||
private void Start()
|
||||
{
|
||||
InitializeNetwork();
|
||||
InitializeConvai();
|
||||
}
|
||||
|
||||
private void OnDestroy()
|
||||
{
|
||||
CleanupNPCSubscriptions();
|
||||
CleanupNetwork();
|
||||
}
|
||||
|
||||
private void InitializeNetwork()
|
||||
{
|
||||
try
|
||||
{
|
||||
_udpClient = new UdpClient();
|
||||
_targetEndPoint = new IPEndPoint(IPAddress.Parse(targetIP), targetPort);
|
||||
_isInitialized = true;
|
||||
|
||||
ConvaiLogger.Info($"UDP Speech Sender initialized. Target: {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Failed to initialize UDP speech sender: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private void InitializeConvai()
|
||||
{
|
||||
// Get target NPC
|
||||
if (useActiveNPC)
|
||||
{
|
||||
sourceNPC = ConvaiNPCManager.Instance?.GetActiveConvaiNPC();
|
||||
}
|
||||
|
||||
SubscribeToNPCEvents();
|
||||
|
||||
// Subscribe to NPC manager events for late NPC activation
|
||||
if (ConvaiNPCManager.Instance != null)
|
||||
{
|
||||
ConvaiNPCManager.Instance.OnActiveNPCChanged += HandleActiveNPCChanged;
|
||||
}
|
||||
}
|
||||
|
||||
private void SubscribeToNPCEvents()
|
||||
{
|
||||
if (sourceNPC?.AudioManager != null)
|
||||
{
|
||||
// Hook into the character talking events
|
||||
sourceNPC.AudioManager.OnCharacterTalkingChanged += HandleCharacterTalkingChanged;
|
||||
sourceNPC.AudioManager.OnAudioTranscriptAvailable += HandleTranscriptAvailable;
|
||||
|
||||
ConvaiLogger.Info($"UDP Speech Sender subscribed to NPC: {sourceNPC.characterName}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
else
|
||||
{
|
||||
ConvaiLogger.Warn("No source NPC available for speech transmission", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleCharacterTalkingChanged(bool isTalking)
|
||||
{
|
||||
if (!_isInitialized) return;
|
||||
|
||||
if (isTalking)
|
||||
{
|
||||
// Start monitoring for audio clips
|
||||
StartCoroutine(MonitorAudioClips());
|
||||
}
|
||||
else
|
||||
{
|
||||
// End speech transmission
|
||||
_ = SendFinalPacket();
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleTranscriptAvailable(string transcript)
|
||||
{
|
||||
if (enableDebugLogging && !string.IsNullOrEmpty(transcript))
|
||||
{
|
||||
ConvaiLogger.DebugLog($"📝 NPC transcript: '{transcript}'", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerator MonitorAudioClips()
|
||||
{
|
||||
if (sourceNPC?.AudioManager == null) yield break;
|
||||
|
||||
AudioSource audioSource = sourceNPC.AudioManager.GetComponent<AudioSource>();
|
||||
AudioClip lastClip = null;
|
||||
|
||||
while (sourceNPC.IsCharacterTalking)
|
||||
{
|
||||
if (audioSource?.clip != null && audioSource.clip != lastClip)
|
||||
{
|
||||
// New clip detected!
|
||||
lastClip = audioSource.clip;
|
||||
|
||||
// Only send if we haven't sent this clip before
|
||||
if (!_sentClips.Contains(lastClip))
|
||||
{
|
||||
_sentClips.Add(lastClip);
|
||||
|
||||
// Get the transcript from the most recent available transcript
|
||||
string transcript = GetRecentTranscript();
|
||||
|
||||
// Send this clip
|
||||
_ = TransmitAudioClip(lastClip, transcript);
|
||||
}
|
||||
}
|
||||
|
||||
yield return new WaitForSeconds(0.1f); // Check every 100ms
|
||||
}
|
||||
|
||||
// Clear sent clips when done
|
||||
_sentClips.Clear();
|
||||
}
|
||||
|
||||
private string GetRecentTranscript()
|
||||
{
|
||||
// Try to get transcript from the NPC's recent activity
|
||||
// This is a simple approach - in a more complex setup you might want to match clips to transcripts
|
||||
return ""; // Transcripts come via the transcript event
|
||||
}
|
||||
|
||||
private async Task TransmitAudioClip(AudioClip audioClip, string transcript)
|
||||
{
|
||||
if (!_isInitialized || audioClip == null) return;
|
||||
|
||||
try
|
||||
{
|
||||
// Start transmission if not already started
|
||||
if (!_isSendingSpeech)
|
||||
{
|
||||
_isSendingSpeech = true;
|
||||
OnSpeechTransmission?.Invoke(true);
|
||||
|
||||
ConvaiLogger.Info($"🔊 Starting speech transmission", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
|
||||
// Use the current speech sequence for this entire clip
|
||||
int clipSequence = _speechSequence;
|
||||
|
||||
// Send start packet with metadata
|
||||
await SendAudioStartPacket(audioClip, transcript, clipSequence);
|
||||
|
||||
// Send audio data in chunks (all with the same sequence)
|
||||
await SendAudioClipInChunks(audioClip, clipSequence);
|
||||
|
||||
// Send end packet for this clip (with the same sequence)
|
||||
await SendAudioEndPacket(clipSequence);
|
||||
|
||||
// Only increment sequence after the entire clip is sent
|
||||
_speechSequence++;
|
||||
|
||||
OnSpeechSent?.Invoke(transcript);
|
||||
|
||||
if (enableDebugLogging)
|
||||
ConvaiLogger.DebugLog($"✅ Transmitted speech clip: {audioClip.length:F2}s (sequence {clipSequence})", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Failed to transmit AudioClip: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SendAudioStartPacket(AudioClip audioClip, string transcript, int sequence)
|
||||
{
|
||||
byte[] packet = CreateAudioStartPacket(audioClip, transcript, sequence);
|
||||
await _udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
|
||||
|
||||
if (enableDebugLogging)
|
||||
ConvaiLogger.DebugLog($"📤 Sent start packet {sequence}: {audioClip.samples} samples", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
|
||||
private async Task SendAudioClipInChunks(AudioClip audioClip, int sequence)
|
||||
{
|
||||
// Get all audio data
|
||||
float[] audioData = new float[audioClip.samples];
|
||||
audioClip.GetData(audioData, 0);
|
||||
|
||||
// Send in chunks
|
||||
int totalSamples = audioData.Length;
|
||||
int processedSamples = 0;
|
||||
int chunkCount = 0;
|
||||
|
||||
while (processedSamples < totalSamples)
|
||||
{
|
||||
int remainingSamples = totalSamples - processedSamples;
|
||||
int currentChunkSize = Mathf.Min(maxSamplesPerPacket, remainingSamples);
|
||||
|
||||
float[] chunkData = new float[currentChunkSize];
|
||||
Array.Copy(audioData, processedSamples, chunkData, 0, currentChunkSize);
|
||||
|
||||
byte[] packet = CreateAudioChunkPacket(chunkData, audioClip.frequency, processedSamples, sequence);
|
||||
await _udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
|
||||
|
||||
processedSamples += currentChunkSize;
|
||||
chunkCount++;
|
||||
|
||||
if (enableDebugLogging && chunkCount % 10 == 0)
|
||||
ConvaiLogger.DebugLog($"📤 Sent chunk {chunkCount} for sequence {sequence}", ConvaiLogger.LogCategory.Character);
|
||||
|
||||
// Small delay to avoid overwhelming the network
|
||||
await Task.Delay(5);
|
||||
}
|
||||
|
||||
if (enableDebugLogging)
|
||||
ConvaiLogger.DebugLog($"📤 Sent {chunkCount} audio chunks for sequence {sequence}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
|
||||
private async Task SendAudioEndPacket(int sequence)
|
||||
{
|
||||
byte[] packet = CreateAudioEndPacket(sequence);
|
||||
await _udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
|
||||
|
||||
if (enableDebugLogging)
|
||||
ConvaiLogger.DebugLog($"📤 Sent end packet for sequence {sequence}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
|
||||
private async Task SendFinalPacket()
|
||||
{
|
||||
if (!_isSendingSpeech) return;
|
||||
|
||||
try
|
||||
{
|
||||
byte[] packet = CreateFinalPacket();
|
||||
await _udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
|
||||
|
||||
_isSendingSpeech = false;
|
||||
OnSpeechTransmission?.Invoke(false);
|
||||
|
||||
ConvaiLogger.Info("🔊 Speech transmission completed", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ConvaiLogger.Error($"Failed to send final packet: {ex.Message}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] CreateAudioStartPacket(AudioClip audioClip, string transcript, int sequence)
|
||||
{
|
||||
byte[] transcriptBytes = System.Text.Encoding.UTF8.GetBytes(transcript ?? "");
|
||||
|
||||
// Packet structure:
|
||||
// 4 bytes: Magic number
|
||||
// 1 byte: Packet type (0x01 = audio start)
|
||||
// 4 bytes: Sequence number
|
||||
// 4 bytes: Total samples in clip
|
||||
// 4 bytes: Sample rate
|
||||
// 4 bytes: Channels
|
||||
// 4 bytes: Transcript length
|
||||
// N bytes: Transcript (UTF-8)
|
||||
|
||||
int headerSize = 25;
|
||||
byte[] packet = new byte[headerSize + transcriptBytes.Length];
|
||||
|
||||
int offset = 0;
|
||||
|
||||
BitConverter.GetBytes(MAGIC_NUMBER).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
packet[offset] = PACKET_TYPE_AUDIO_START;
|
||||
offset += 1;
|
||||
|
||||
BitConverter.GetBytes(sequence).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
BitConverter.GetBytes(audioClip.samples).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
BitConverter.GetBytes(audioClip.frequency).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
BitConverter.GetBytes(audioClip.channels).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
BitConverter.GetBytes(transcriptBytes.Length).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
transcriptBytes.CopyTo(packet, offset);
|
||||
|
||||
return packet;
|
||||
}
|
||||
|
||||
private byte[] CreateAudioChunkPacket(float[] audioData, int frequency, int startSample, int sequence)
|
||||
{
|
||||
// Packet structure:
|
||||
// 4 bytes: Magic number
|
||||
// 1 byte: Packet type (0x02 = audio chunk)
|
||||
// 4 bytes: Sequence number
|
||||
// 4 bytes: Start sample position
|
||||
// 4 bytes: Sample count in this chunk
|
||||
// N bytes: Audio data (as 16-bit PCM)
|
||||
|
||||
int headerSize = 17;
|
||||
int audioDataSize = audioData.Length * sizeof(short);
|
||||
byte[] packet = new byte[headerSize + audioDataSize];
|
||||
|
||||
int offset = 0;
|
||||
|
||||
BitConverter.GetBytes(MAGIC_NUMBER).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
packet[offset] = PACKET_TYPE_AUDIO_CHUNK;
|
||||
offset += 1;
|
||||
|
||||
BitConverter.GetBytes(sequence).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
BitConverter.GetBytes(startSample).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
BitConverter.GetBytes(audioData.Length).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
// Convert float samples to 16-bit PCM
|
||||
for (int i = 0; i < audioData.Length; i++)
|
||||
{
|
||||
short sample = (short)(Mathf.Clamp(audioData[i], -1f, 1f) * short.MaxValue);
|
||||
BitConverter.GetBytes(sample).CopyTo(packet, offset);
|
||||
offset += 2;
|
||||
}
|
||||
|
||||
return packet;
|
||||
}
|
||||
|
||||
private byte[] CreateAudioEndPacket(int sequence)
|
||||
{
|
||||
byte[] packet = new byte[13]; // Header only
|
||||
|
||||
int offset = 0;
|
||||
|
||||
BitConverter.GetBytes(MAGIC_NUMBER).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
packet[offset] = PACKET_TYPE_AUDIO_END;
|
||||
offset += 1;
|
||||
|
||||
BitConverter.GetBytes(sequence).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
BitConverter.GetBytes(0).CopyTo(packet, offset); // No additional data
|
||||
|
||||
return packet;
|
||||
}
|
||||
|
||||
private byte[] CreateFinalPacket()
|
||||
{
|
||||
byte[] packet = new byte[13]; // Header only
|
||||
|
||||
int offset = 0;
|
||||
|
||||
BitConverter.GetBytes(MAGIC_NUMBER).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
packet[offset] = PACKET_TYPE_FINAL;
|
||||
offset += 1;
|
||||
|
||||
BitConverter.GetBytes(_speechSequence).CopyTo(packet, offset);
|
||||
offset += 4;
|
||||
|
||||
BitConverter.GetBytes(0).CopyTo(packet, offset); // No additional data
|
||||
|
||||
return packet;
|
||||
}
|
||||
|
||||
private void CleanupNPCSubscriptions()
|
||||
{
|
||||
if (sourceNPC?.AudioManager != null)
|
||||
{
|
||||
sourceNPC.AudioManager.OnCharacterTalkingChanged -= HandleCharacterTalkingChanged;
|
||||
sourceNPC.AudioManager.OnAudioTranscriptAvailable -= HandleTranscriptAvailable;
|
||||
}
|
||||
|
||||
if (ConvaiNPCManager.Instance != null)
|
||||
{
|
||||
ConvaiNPCManager.Instance.OnActiveNPCChanged -= HandleActiveNPCChanged;
|
||||
}
|
||||
}
|
||||
|
||||
private void CleanupNetwork()
|
||||
{
|
||||
_udpClient?.Close();
|
||||
_udpClient?.Dispose();
|
||||
_udpClient = null;
|
||||
}
|
||||
|
||||
private void HandleActiveNPCChanged(ConvaiNPC newActiveNPC)
|
||||
{
|
||||
if (!useActiveNPC) return;
|
||||
|
||||
// Cleanup old subscriptions
|
||||
CleanupNPCSubscriptions();
|
||||
|
||||
// Update to new NPC
|
||||
sourceNPC = newActiveNPC;
|
||||
SubscribeToNPCEvents();
|
||||
}
|
||||
|
||||
// Public methods for external control
|
||||
public void SetTargetEndpoint(string ip, int port)
|
||||
{
|
||||
targetIP = ip;
|
||||
targetPort = port;
|
||||
_targetEndPoint = new IPEndPoint(IPAddress.Parse(ip), port);
|
||||
}
|
||||
|
||||
public bool IsSendingSpeech => _isSendingSpeech;
|
||||
public bool IsInitialized => _isInitialized;
|
||||
public ConvaiNPC SourceNPC => sourceNPC;
|
||||
|
||||
// Debug methods
|
||||
public void ShowNetworkStatus()
|
||||
{
|
||||
ConvaiLogger.Info($"=== Speech Sender Status ===", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Target: {targetIP}:{targetPort}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Initialized: {_isInitialized}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Sending Speech: {_isSendingSpeech}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Source NPC: {(sourceNPC?.characterName ?? "None")}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Packets sent: {_speechSequence}", ConvaiLogger.LogCategory.Character);
|
||||
ConvaiLogger.Info($"Sent clips: {_sentClips.Count}", ConvaiLogger.LogCategory.Character);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: f903e03686cf216469fb4bf1e6c027d0
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
Reference in New Issue
Block a user