tomh104/Master-Arbeit-Tom-Hempel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

further work on bug fixing

Browse Source
This commit is contained in:
tom.hempel
2025-09-26 10:13:33 +02:00
parent 7aeb173789
commit 7d65d1b799
6 changed files with 1492 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

355
Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs
View File

@ -6,6 +6,9 @@ using System.Threading.Tasks;
using Convai.Scripts.Runtime.LoggerSystem;
using Convai.Scripts.Runtime.UI;
using UnityEngine;
using UnityEngine.XR;
using UnityEngine.InputSystem;
using UnityEngine.InputSystem.XR;
namespace Convai.Scripts.Runtime.Multiplayer
{
@ -30,6 +33,11 @@ namespace Convai.Scripts.Runtime.Multiplayer
[SerializeField] private KeyCode talkKey = KeyCode.T;
[SerializeField] private bool useHoldToTalk = true;
[SerializeField] private KeyCode controllerTalkButton = KeyCode.JoystickButton0; // A button on most controllers
[SerializeField] private bool useXRControllerAButton = true; // OpenXR primaryButton
[SerializeField] private XRNode xrControllerNode = XRNode.RightHand; // Quest A button is on right hand
[SerializeField] private bool useInputSystemXR = true; // Use new Input System for XR button
[SerializeField] private bool enableBButtonTest = true; // Press B/secondary to send test packet
[SerializeField] private InputActionReference sendVoiceActionReference; // Optional external action (e.g., QuestPro primaryButton)
[Header("Debug")]
[SerializeField] private bool enableDebugLogging = true;
@ -40,11 +48,25 @@ namespace Convai.Scripts.Runtime.Multiplayer
private AudioClip _audioClip;
private bool _isRecording = false;
private CancellationTokenSource _cancellationTokenSource;
private CancellationTokenSource _ackCancellationTokenSource;
private int _lastMicrophonePosition = 0;
private float[] _audioBuffer;
private string _selectedMicrophone;
private int _packetSequence = 0;
private volatile bool _startAckReceived = false;
private bool _xrAButtonPrevPressed = false;
private InputAction _xrTalkAction;
private InputAction _xrTestAction;
private bool _usingExternalTalkAction = false;
private InputAction _externalTalkAction;
// Protocol constants
private const uint AUDIO_MAGIC = 0xC0A1;
private const uint ACK_MAGIC = 0xC0A2;
private const byte FLAG_AUDIO = 0;
private const byte FLAG_END = 1;
private const byte FLAG_START = 2;
public event Action<bool> OnRecordingStateChanged;
@ -63,6 +85,27 @@ namespace Convai.Scripts.Runtime.Multiplayer
InitializeNetwork();
InitializeAudio();
_cancellationTokenSource = new CancellationTokenSource();
_ackCancellationTokenSource = new CancellationTokenSource();
// Start ACK listener
_ = ListenForAcks(_ackCancellationTokenSource.Token);
// Setup Input System action for XR A/primary button
if (useInputSystemXR)
{
if (sendVoiceActionReference != null && sendVoiceActionReference.action != null)
{
SetupExternalTalkInputAction(sendVoiceActionReference.action);
}
else
{
SetupXRTalkInputAction();
}
if (enableBButtonTest)
{
SetupXRTestInputAction();
}
}
}
private void Update()
@ -75,6 +118,13 @@ namespace Convai.Scripts.Runtime.Multiplayer
StopRecording();
_cancellationTokenSource?.Cancel();
_cancellationTokenSource?.Dispose();
_ackCancellationTokenSource?.Cancel();
_ackCancellationTokenSource?.Dispose();
if (_usingExternalTalkAction)
TeardownExternalTalkInputAction();
else
TeardownXRTalkInputAction();
TeardownXRTestInputAction();
_udpClient?.Close();
}
@ -127,21 +177,34 @@ namespace Convai.Scripts.Runtime.Multiplayer
private void HandleInput()
{
// Input System XR events handle XR button press/release.
// Here we keep keyboard/legacy controller as fallback.
bool xrDown = false;
bool xrUp = false;
if (useXRControllerAButton && !useInputSystemXR)
{
bool xrPressed = GetXRPrimaryButtonPressed(xrControllerNode);
xrDown = xrPressed && !_xrAButtonPrevPressed;
xrUp = !xrPressed && _xrAButtonPrevPressed;
_xrAButtonPrevPressed = xrPressed;
}
// Handle talk key
if (useHoldToTalk)
{
if ((Input.GetKeyDown(talkKey) || Input.GetKeyDown(controllerTalkButton)) && !_isRecording)
if ((Input.GetKeyDown(talkKey) || Input.GetKeyDown(controllerTalkButton) || xrDown) && !_isRecording)
{
StartRecording();
}
else if ((Input.GetKeyUp(talkKey) || Input.GetKeyUp(controllerTalkButton)) && _isRecording)
else if ((Input.GetKeyUp(talkKey) || Input.GetKeyUp(controllerTalkButton) || xrUp) && _isRecording)
{
StopRecording();
}
}
else
{
if (Input.GetKeyDown(talkKey) || Input.GetKeyDown(controllerTalkButton))
if (Input.GetKeyDown(talkKey) || Input.GetKeyDown(controllerTalkButton) || xrDown)
{
if (_isRecording)
StopRecording();
@ -156,6 +219,196 @@ namespace Convai.Scripts.Runtime.Multiplayer
TestConnection();
}
}
private void SetupXRTalkInputAction()
{
try
{
// Create button action
_xrTalkAction = new InputAction("XRTalk", InputActionType.Button);
string handTag = xrControllerNode == XRNode.LeftHand ? "{LeftHand}" : "{RightHand}";
// Bind to common XR controller primary/A button paths
_xrTalkAction.AddBinding($"<XRController>{handTag}/primaryButton");
_xrTalkAction.AddBinding($"<OculusTouchController>{handTag}/primaryButton");
_xrTalkAction.AddBinding($"<MetaTouchController>{handTag}/primaryButton");
_xrTalkAction.AddBinding($"<QuestProTouchController>{handTag}/primaryButton");
_xrTalkAction.AddBinding($"<XRController>{handTag}/buttonSouth");
// Gamepad A as additional fallback (useful in editor)
_xrTalkAction.AddBinding("<Gamepad>/buttonSouth");
if (useHoldToTalk)
{
_xrTalkAction.started += ctx => { if (!_isRecording) StartRecording(); };
_xrTalkAction.canceled += ctx => { if (_isRecording) StopRecording(); };
}
else
{
_xrTalkAction.started += ctx => { if (_isRecording) StopRecording(); else StartRecording(); };
}
_xrTalkAction.Enable();
}
catch (Exception ex)
{
ConvaiLogger.Warn($"Failed to setup XR InputAction: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
private void TeardownXRTalkInputAction()
{
try
{
if (_xrTalkAction != null)
{
_xrTalkAction.Disable();
_xrTalkAction.Dispose();
_xrTalkAction = null;
}
}
catch (Exception)
{
// ignore
}
}
private void SetupExternalTalkInputAction(InputAction action)
{
try
{
_externalTalkAction = action;
_usingExternalTalkAction = true;
if (useHoldToTalk)
{
_externalTalkAction.started += ctx => { if (!_isRecording) StartRecording(); };
_externalTalkAction.canceled += ctx => { if (_isRecording) StopRecording(); };
}
else
{
_externalTalkAction.started += ctx => { if (_isRecording) StopRecording(); else StartRecording(); };
}
if (!_externalTalkAction.enabled)
_externalTalkAction.Enable();
}
catch (Exception ex)
{
ConvaiLogger.Warn($"Failed to setup external talk action: {ex.Message}", ConvaiLogger.LogCategory.Character);
_usingExternalTalkAction = false;
_externalTalkAction = null;
// Fallback to programmatic setup
SetupXRTalkInputAction();
}
}
private void TeardownExternalTalkInputAction()
{
try
{
if (_externalTalkAction != null)
{
// Remove handlers; do not disable/dispose external actions
if (useHoldToTalk)
{
_externalTalkAction.started -= ctx => { if (!_isRecording) StartRecording(); };
_externalTalkAction.canceled -= ctx => { if (_isRecording) StopRecording(); };
}
else
{
_externalTalkAction.started -= ctx => { if (_isRecording) StopRecording(); else StartRecording(); };
}
}
}
catch (Exception)
{
// ignore
}
finally
{
_externalTalkAction = null;
_usingExternalTalkAction = false;
}
}
private void SetupXRTestInputAction()
{
try
{
_xrTestAction = new InputAction("XRTest", InputActionType.Button);
string handTag = xrControllerNode == XRNode.LeftHand ? "{LeftHand}" : "{RightHand}";
// B button is typically secondaryButton or buttonEast
_xrTestAction.AddBinding($"<XRController>{handTag}/secondaryButton");
_xrTestAction.AddBinding($"<OculusTouchController>{handTag}/secondaryButton");
_xrTestAction.AddBinding($"<MetaTouchController>{handTag}/secondaryButton");
_xrTestAction.AddBinding($"<XRController>{handTag}/buttonEast");
_xrTestAction.AddBinding("<Gamepad>/buttonEast");
_xrTestAction.started += ctx => { SendTestPacket(); };
_xrTestAction.Enable();
}
catch (Exception ex)
{
ConvaiLogger.Warn($"Failed to setup XR Test InputAction: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
private void TeardownXRTestInputAction()
{
try
{
if (_xrTestAction != null)
{
_xrTestAction.Disable();
_xrTestAction.Dispose();
_xrTestAction = null;
}
}
catch (Exception)
{
// ignore
}
}
private async void SendTestPacket()
{
try
{
if (_udpClient == null || _targetEndPoint == null)
{
ConvaiLogger.Error("UDP client not initialized for test packet", ConvaiLogger.LogCategory.Character);
return;
}
string testMessage = "Hello this is a Test";
byte[] data = System.Text.Encoding.UTF8.GetBytes(testMessage);
await _udpClient.SendAsync(data, data.Length, _targetEndPoint);
ConvaiLogger.Info("Sent test packet: 'Hello this is a Test'", ConvaiLogger.LogCategory.Character);
}
catch (Exception ex)
{
ConvaiLogger.Error($"Failed to send test packet: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
private bool GetXRPrimaryButtonPressed(XRNode hand)
{
try
{
var device = InputDevices.GetDeviceAtXRNode(hand);
if (!device.isValid)
return false;
if (device.TryGetFeatureValue(UnityEngine.XR.CommonUsages.primaryButton, out bool pressed))
return pressed;
}
catch (Exception)
{
// ignore errors and treat as not pressed
}
return false;
}
public void StartRecording()
{
@ -169,10 +422,14 @@ namespace Convai.Scripts.Runtime.Multiplayer
_isRecording = true;
_lastMicrophonePosition = 0;
_packetSequence = 0;
_startAckReceived = false;
ConvaiLogger.Info("Started recording for UDP transmission (Simple)", ConvaiLogger.LogCategory.Character);
OnRecordingStateChanged?.Invoke(true);
// Send START control and wait briefly for ACK to ensure receiver is ready
_ = SendStartOfRecordingSignalAndAwaitAck();
// Start continuous audio processing
_ = ProcessAudioContinuously(_cancellationTokenSource.Token);
}
@ -324,7 +581,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
int offset = 0;
// Magic number
BitConverter.GetBytes((uint)0xC0A1).CopyTo(packet, offset);
BitConverter.GetBytes(AUDIO_MAGIC).CopyTo(packet, offset);
offset += 4;
// Packet sequence
@ -340,7 +597,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
offset += 4;
// Flags (0 for normal audio)
packet[offset] = 0;
packet[offset] = FLAG_AUDIO;
offset += 1;
// Convert audio samples to bytes (same as Convai approach)
@ -366,7 +623,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
int offset = 0;
// Magic number
BitConverter.GetBytes((uint)0xC0A1).CopyTo(packet, offset);
BitConverter.GetBytes(AUDIO_MAGIC).CopyTo(packet, offset);
offset += 4;
// Packet sequence
@ -382,7 +639,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
offset += 4;
// Flags (1 for end of recording)
packet[offset] = 1;
packet[offset] = FLAG_END;
_udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
}
@ -391,6 +648,90 @@ namespace Convai.Scripts.Runtime.Multiplayer
ConvaiLogger.Error($"Failed to send end signal: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
private async Task SendStartOfRecordingSignalAndAwaitAck()
{
try
{
const int maxAttempts = 3;
const int ackTimeoutMs = 250;
for (int attempt = 1; attempt <= maxAttempts && !_startAckReceived; attempt++)
{
// Build START control packet (no audio, special flag)
byte[] packet = new byte[17];
int offset = 0;
BitConverter.GetBytes(AUDIO_MAGIC).CopyTo(packet, offset);
offset += 4;
// Use -1 as the special sequence for START control
BitConverter.GetBytes(-1).CopyTo(packet, offset);
offset += 4;
BitConverter.GetBytes(0).CopyTo(packet, offset);
offset += 4;
BitConverter.GetBytes(_lastMicrophonePosition).CopyTo(packet, offset);
offset += 4;
packet[offset] = FLAG_START;
await _udpClient.SendAsync(packet, packet.Length, _targetEndPoint);
// Wait for ACK
int waited = 0;
while (!_startAckReceived && waited < ackTimeoutMs)
{
await Task.Delay(10);
waited += 10;
}
if (_startAckReceived)
{
if (enableDebugLogging)
ConvaiLogger.DebugLog("Received START ACK from receiver", ConvaiLogger.LogCategory.Character);
break;
}
else if (enableDebugLogging)
{
ConvaiLogger.Warn($"No START ACK (attempt {attempt}/{maxAttempts}), retrying...", ConvaiLogger.LogCategory.Character);
}
}
}
catch (Exception ex)
{
ConvaiLogger.Warn($"Error during START ACK process: {ex.Message}", ConvaiLogger.LogCategory.Character);
}
}
private async Task ListenForAcks(CancellationToken token)
{
// Use the same UDP client; acks will be sent back to our ephemeral local port
while (!token.IsCancellationRequested)
{
try
{
var result = await _udpClient.ReceiveAsync();
var data = result.Buffer;
if (data == null || data.Length < 8)
continue;
uint magic = BitConverter.ToUInt32(data, 0);
if (magic != ACK_MAGIC)
continue;
int seq = BitConverter.ToInt32(data, 4);
if (seq == -1)
{
_startAckReceived = true;
}
}
catch (ObjectDisposedException)
{
break;
}
catch (Exception)
{
// Ignore and keep listening
}
}
}
// Public methods for external control
public void SetTargetEndpoint(string ip, int port)