From 7aeb173789c36acb3b43d1775e117591cc341c65 Mon Sep 17 00:00:00 2001 From: "tom.hempel" Date: Mon, 22 Sep 2025 17:19:14 +0200 Subject: [PATCH] improved audio networking --- .../Multiplayer/ConvaiSimpleUDPAudioSender.cs | 94 +++++++++++++++---- .../Assets/Scripts/NetworkConfig.asset | 4 +- .../XRSimulationRuntimeSettings.asset.meta | 2 +- Unity-Master/Assets/XR/Temp.meta | 8 ++ .../XR/Temp/XRSimulationPreferences.asset | 3 + .../Temp/XRSimulationPreferences.asset.meta | 8 ++ .../XR/Temp/XRSimulationRuntimeSettings.asset | 3 + .../XRSimulationRuntimeSettings.asset.meta | 8 ++ .../XRSimulationPreferences.asset.meta | 2 +- .../ProjectSettings/ProjectSettings.asset | 4 +- 10 files changed, 112 insertions(+), 24 deletions(-) create mode 100644 Unity-Master/Assets/XR/Temp.meta create mode 100644 Unity-Master/Assets/XR/Temp/XRSimulationPreferences.asset create mode 100644 Unity-Master/Assets/XR/Temp/XRSimulationPreferences.asset.meta create mode 100644 Unity-Master/Assets/XR/Temp/XRSimulationRuntimeSettings.asset create mode 100644 Unity-Master/Assets/XR/Temp/XRSimulationRuntimeSettings.asset.meta diff --git a/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs b/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs index a092f15..9d8ff17 100644 --- a/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs +++ b/Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs @@ -29,6 +29,7 @@ namespace Convai.Scripts.Runtime.Multiplayer [Header("UI")] [SerializeField] private KeyCode talkKey = KeyCode.T; [SerializeField] private bool useHoldToTalk = true; + [SerializeField] private KeyCode controllerTalkButton = KeyCode.JoystickButton0; // A button on most controllers [Header("Debug")] [SerializeField] private bool enableDebugLogging = true; @@ -93,12 +94,34 @@ namespace Convai.Scripts.Runtime.Multiplayer private void InitializeAudio() { - _selectedMicrophone = MicrophoneManager.Instance.SelectedMicrophoneName; - _audioBuffer = new float[recordingFrequency * recordingLength]; - + try + { + // Try to get selected microphone from Convai's UI system + _selectedMicrophone = MicrophoneManager.Instance?.SelectedMicrophoneName; + } + catch (Exception ex) + { + // If UISaveLoadSystem / MicrophoneManager isn't initialized yet, fall back to first available device + ConvaiLogger.Warn($"MicrophoneManager not available; falling back to default device. {ex.Message}", ConvaiLogger.LogCategory.Character); + _selectedMicrophone = null; + } + + // Fallback: pick the first available microphone if none selected or manager unavailable if (string.IsNullOrEmpty(_selectedMicrophone)) { - ConvaiLogger.Error("No microphone selected for UDP audio sender", ConvaiLogger.LogCategory.Character); + var devices = Microphone.devices; + if (devices != null && devices.Length > 0) + { + _selectedMicrophone = devices[0]; + ConvaiLogger.Info($"Using default microphone: {_selectedMicrophone}", ConvaiLogger.LogCategory.Character); + } + } + + _audioBuffer = new float[recordingFrequency * recordingLength]; + + if (string.IsNullOrEmpty(_selectedMicrophone)) + { + ConvaiLogger.Error("No microphone available or selected for UDP audio sender", ConvaiLogger.LogCategory.Character); } } @@ -107,18 +130,18 @@ namespace Convai.Scripts.Runtime.Multiplayer // Handle talk key if (useHoldToTalk) { - if (Input.GetKeyDown(talkKey) && !_isRecording) + if ((Input.GetKeyDown(talkKey) || Input.GetKeyDown(controllerTalkButton)) && !_isRecording) { StartRecording(); } - else if (Input.GetKeyUp(talkKey) && _isRecording) + else if ((Input.GetKeyUp(talkKey) || Input.GetKeyUp(controllerTalkButton)) && _isRecording) { StopRecording(); } } else { - if (Input.GetKeyDown(talkKey)) + if (Input.GetKeyDown(talkKey) || Input.GetKeyDown(controllerTalkButton)) { if (_isRecording) StopRecording(); @@ -141,7 +164,8 @@ namespace Convai.Scripts.Runtime.Multiplayer try { - _audioClip = Microphone.Start(_selectedMicrophone, false, recordingLength, recordingFrequency); + // Use looping clip so we can handle ring-buffer wrap-around reliably + _audioClip = Microphone.Start(_selectedMicrophone, true, recordingLength, recordingFrequency); _isRecording = true; _lastMicrophonePosition = 0; _packetSequence = 0; @@ -186,24 +210,58 @@ namespace Convai.Scripts.Runtime.Multiplayer { try { - await Task.Delay(100, cancellationToken); // Process every 100ms + await Task.Delay(30, cancellationToken); // Process ~33 times/sec for better capture granularity if (_audioClip == null || !Microphone.IsRecording(_selectedMicrophone)) break; int currentMicrophonePosition = Microphone.GetPosition(_selectedMicrophone); + int clipSamples = _audioClip.samples; + + if (clipSamples <= 0) + continue; + + // Compute how many new samples are available, accounting for wrap-around int audioDataLength = currentMicrophonePosition - _lastMicrophonePosition; - - if (audioDataLength > 0) + bool wrapped = false; + if (audioDataLength < 0) { - // Get audio data from the microphone clip - _audioClip.GetData(_audioBuffer, _lastMicrophonePosition); - - // Send data in smaller chunks to avoid array bounds issues - await SendAudioDataInChunks(_audioBuffer, audioDataLength); - - _lastMicrophonePosition = currentMicrophonePosition; + audioDataLength += clipSamples; + wrapped = true; } + + if (audioDataLength <= 0) + continue; + + if (!wrapped) + { + // Contiguous region, read exactly the new samples + int segmentLen = audioDataLength; + var segment = new float[segmentLen]; + _audioClip.GetData(segment, _lastMicrophonePosition); + await SendAudioDataInChunks(segment, segmentLen); + } + else + { + // Wrapped: send tail [lastPos .. end) then head [0 .. currentPos) + int firstLen = clipSamples - _lastMicrophonePosition; + if (firstLen > 0) + { + var firstSeg = new float[firstLen]; + _audioClip.GetData(firstSeg, _lastMicrophonePosition); + await SendAudioDataInChunks(firstSeg, firstLen); + } + + int secondLen = currentMicrophonePosition; + if (secondLen > 0) + { + var secondSeg = new float[secondLen]; + _audioClip.GetData(secondSeg, 0); + await SendAudioDataInChunks(secondSeg, secondLen); + } + } + + _lastMicrophonePosition = currentMicrophonePosition; } catch (Exception ex) when (!(ex is OperationCanceledException)) { diff --git a/Unity-Master/Assets/Scripts/NetworkConfig.asset b/Unity-Master/Assets/Scripts/NetworkConfig.asset index 638378a..baa203c 100644 --- a/Unity-Master/Assets/Scripts/NetworkConfig.asset +++ b/Unity-Master/Assets/Scripts/NetworkConfig.asset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5f914c2d99734607969803eb007610cf2ccac5b16b93a43d2596f0268b4748b5 -size 444 +oid sha256:79445491a897bce57a20454b758ae53e213754ff494e89faddc53b6db5584d28 +size 503 diff --git a/Unity-Master/Assets/XR/Resources/XRSimulationRuntimeSettings.asset.meta b/Unity-Master/Assets/XR/Resources/XRSimulationRuntimeSettings.asset.meta index a390f72..2b9e941 100644 --- a/Unity-Master/Assets/XR/Resources/XRSimulationRuntimeSettings.asset.meta +++ b/Unity-Master/Assets/XR/Resources/XRSimulationRuntimeSettings.asset.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: b3f0e4c8e44049f4383103acf7518cab +guid: 430fa8b3199c49f40a5b3c10dd88a5e5 NativeFormatImporter: externalObjects: {} mainObjectFileID: 11400000 diff --git a/Unity-Master/Assets/XR/Temp.meta b/Unity-Master/Assets/XR/Temp.meta new file mode 100644 index 0000000..1f4ec55 --- /dev/null +++ b/Unity-Master/Assets/XR/Temp.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: acb11a8edd06879488f49f2a67973a93 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Unity-Master/Assets/XR/Temp/XRSimulationPreferences.asset b/Unity-Master/Assets/XR/Temp/XRSimulationPreferences.asset new file mode 100644 index 0000000..489a5e5 --- /dev/null +++ b/Unity-Master/Assets/XR/Temp/XRSimulationPreferences.asset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58364ec3131f9197df55d5fd61bb69760fb265a4adfdbf6b689cca12cb18754c +size 581 diff --git a/Unity-Master/Assets/XR/Temp/XRSimulationPreferences.asset.meta b/Unity-Master/Assets/XR/Temp/XRSimulationPreferences.asset.meta new file mode 100644 index 0000000..dcb3d5d --- /dev/null +++ b/Unity-Master/Assets/XR/Temp/XRSimulationPreferences.asset.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 97cf5143ccdc1af44bf5bbfdf47d31b6 +NativeFormatImporter: + externalObjects: {} + mainObjectFileID: 11400000 + userData: + assetBundleName: + assetBundleVariant: diff --git a/Unity-Master/Assets/XR/Temp/XRSimulationRuntimeSettings.asset b/Unity-Master/Assets/XR/Temp/XRSimulationRuntimeSettings.asset new file mode 100644 index 0000000..61920d8 --- /dev/null +++ b/Unity-Master/Assets/XR/Temp/XRSimulationRuntimeSettings.asset @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5eccaa8b0522899e31e7c9523ab9c30916fc2c62b1bb8eb3f6eff72b0b26e8c +size 1779 diff --git a/Unity-Master/Assets/XR/Temp/XRSimulationRuntimeSettings.asset.meta b/Unity-Master/Assets/XR/Temp/XRSimulationRuntimeSettings.asset.meta new file mode 100644 index 0000000..a390f72 --- /dev/null +++ b/Unity-Master/Assets/XR/Temp/XRSimulationRuntimeSettings.asset.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: b3f0e4c8e44049f4383103acf7518cab +NativeFormatImporter: + externalObjects: {} + mainObjectFileID: 11400000 + userData: + assetBundleName: + assetBundleVariant: diff --git a/Unity-Master/Assets/XR/UserSimulationSettings/Resources/XRSimulationPreferences.asset.meta b/Unity-Master/Assets/XR/UserSimulationSettings/Resources/XRSimulationPreferences.asset.meta index dcb3d5d..c99aa6b 100644 --- a/Unity-Master/Assets/XR/UserSimulationSettings/Resources/XRSimulationPreferences.asset.meta +++ b/Unity-Master/Assets/XR/UserSimulationSettings/Resources/XRSimulationPreferences.asset.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: 97cf5143ccdc1af44bf5bbfdf47d31b6 +guid: b3ae67cc68a994449a213f10f65698fd NativeFormatImporter: externalObjects: {} mainObjectFileID: 11400000 diff --git a/Unity-Master/ProjectSettings/ProjectSettings.asset b/Unity-Master/ProjectSettings/ProjectSettings.asset index badc214..f7d21f1 100644 --- a/Unity-Master/ProjectSettings/ProjectSettings.asset +++ b/Unity-Master/ProjectSettings/ProjectSettings.asset @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:451275d3667f0867921812ddff1f6e59f7ad43d7c74767f015f5d7bd7ccae6b2 -size 24650 +oid sha256:5a2643c536bb34c3c4db6a008600a452a81ce34ada5145268e1832c14a4b239c +size 24640