improved audio networking

Unity-Master/Assets/Scripts/Multiplayer/ConvaiSimpleUDPAudioSender.cs

@@ -29,6 +29,7 @@ namespace Convai.Scripts.Runtime.Multiplayer
         [Header("UI")]
         [SerializeField] private KeyCode talkKey = KeyCode.T;
         [SerializeField] private bool useHoldToTalk = true;
+        [SerializeField] private KeyCode controllerTalkButton = KeyCode.JoystickButton0; // A button on most controllers
         
         [Header("Debug")]
         [SerializeField] private bool enableDebugLogging = true;
@@ -93,12 +94,34 @@ namespace Convai.Scripts.Runtime.Multiplayer
         
         private void InitializeAudio()
         {
-            _selectedMicrophone = MicrophoneManager.Instance.SelectedMicrophoneName;
-            _audioBuffer = new float[recordingFrequency * recordingLength];
-            
+            try
+            {
+                // Try to get selected microphone from Convai's UI system
+                _selectedMicrophone = MicrophoneManager.Instance?.SelectedMicrophoneName;
+            }
+            catch (Exception ex)
+            {
+                // If UISaveLoadSystem / MicrophoneManager isn't initialized yet, fall back to first available device
+                ConvaiLogger.Warn($"MicrophoneManager not available; falling back to default device. {ex.Message}", ConvaiLogger.LogCategory.Character);
+                _selectedMicrophone = null;
+            }
+
+            // Fallback: pick the first available microphone if none selected or manager unavailable
             if (string.IsNullOrEmpty(_selectedMicrophone))
             {
-                ConvaiLogger.Error("No microphone selected for UDP audio sender", ConvaiLogger.LogCategory.Character);
+                var devices = Microphone.devices;
+                if (devices != null && devices.Length > 0)
+                {
+                    _selectedMicrophone = devices[0];
+                    ConvaiLogger.Info($"Using default microphone: {_selectedMicrophone}", ConvaiLogger.LogCategory.Character);
+                }
+            }
+
+            _audioBuffer = new float[recordingFrequency * recordingLength];
+
+            if (string.IsNullOrEmpty(_selectedMicrophone))
+            {
+                ConvaiLogger.Error("No microphone available or selected for UDP audio sender", ConvaiLogger.LogCategory.Character);
             }
         }
         
@@ -107,18 +130,18 @@ namespace Convai.Scripts.Runtime.Multiplayer
             // Handle talk key
             if (useHoldToTalk)
             {
-                if (Input.GetKeyDown(talkKey) && !_isRecording)
+                if ((Input.GetKeyDown(talkKey) || Input.GetKeyDown(controllerTalkButton)) && !_isRecording)
                 {
                     StartRecording();
                 }
-                else if (Input.GetKeyUp(talkKey) && _isRecording)
+                else if ((Input.GetKeyUp(talkKey) || Input.GetKeyUp(controllerTalkButton)) && _isRecording)
                 {
                     StopRecording();
                 }
             }
             else
             {
-                if (Input.GetKeyDown(talkKey))
+                if (Input.GetKeyDown(talkKey) || Input.GetKeyDown(controllerTalkButton))
                 {
                     if (_isRecording)
                         StopRecording();
@@ -141,7 +164,8 @@ namespace Convai.Scripts.Runtime.Multiplayer
                 
             try
             {
-                _audioClip = Microphone.Start(_selectedMicrophone, false, recordingLength, recordingFrequency);
+                // Use looping clip so we can handle ring-buffer wrap-around reliably
+                _audioClip = Microphone.Start(_selectedMicrophone, true, recordingLength, recordingFrequency);
                 _isRecording = true;
                 _lastMicrophonePosition = 0;
                 _packetSequence = 0;
@@ -186,24 +210,58 @@ namespace Convai.Scripts.Runtime.Multiplayer
             {
                 try
                 {
-                    await Task.Delay(100, cancellationToken); // Process every 100ms
+                    await Task.Delay(30, cancellationToken); // Process ~33 times/sec for better capture granularity
                     
                     if (_audioClip == null || !Microphone.IsRecording(_selectedMicrophone))
                         break;
                         
                     int currentMicrophonePosition = Microphone.GetPosition(_selectedMicrophone);
+                    int clipSamples = _audioClip.samples;
+
+                    if (clipSamples <= 0)
+                        continue;
+
+                    // Compute how many new samples are available, accounting for wrap-around
                     int audioDataLength = currentMicrophonePosition - _lastMicrophonePosition;
-                    
-                    if (audioDataLength > 0)
+                    bool wrapped = false;
+                    if (audioDataLength < 0)
                     {
-                        // Get audio data from the microphone clip
-                        _audioClip.GetData(_audioBuffer, _lastMicrophonePosition);
-                        
-                        // Send data in smaller chunks to avoid array bounds issues
-                        await SendAudioDataInChunks(_audioBuffer, audioDataLength);
-                        
-                        _lastMicrophonePosition = currentMicrophonePosition;
+                        audioDataLength += clipSamples;
+                        wrapped = true;
                     }
+
+                    if (audioDataLength <= 0)
+                        continue;
+
+                    if (!wrapped)
+                    {
+                        // Contiguous region, read exactly the new samples
+                        int segmentLen = audioDataLength;
+                        var segment = new float[segmentLen];
+                        _audioClip.GetData(segment, _lastMicrophonePosition);
+                        await SendAudioDataInChunks(segment, segmentLen);
+                    }
+                    else
+                    {
+                        // Wrapped: send tail [lastPos .. end) then head [0 .. currentPos)
+                        int firstLen = clipSamples - _lastMicrophonePosition;
+                        if (firstLen > 0)
+                        {
+                            var firstSeg = new float[firstLen];
+                            _audioClip.GetData(firstSeg, _lastMicrophonePosition);
+                            await SendAudioDataInChunks(firstSeg, firstLen);
+                        }
+
+                        int secondLen = currentMicrophonePosition;
+                        if (secondLen > 0)
+                        {
+                            var secondSeg = new float[secondLen];
+                            _audioClip.GetData(secondSeg, 0);
+                            await SendAudioDataInChunks(secondSeg, secondLen);
+                        }
+                    }
+
+                    _lastMicrophonePosition = currentMicrophonePosition;
                 }
                 catch (Exception ex) when (!(ex is OperationCanceledException))
                 {

Unity-Master/Assets/XR/Resources/XRSimulationRuntimeSettings.asset.meta

@@ -1,5 +1,5 @@
 fileFormatVersion: 2
-guid: b3f0e4c8e44049f4383103acf7518cab
+guid: 430fa8b3199c49f40a5b3c10dd88a5e5
 NativeFormatImporter:
   externalObjects: {}
   mainObjectFileID: 11400000

Unity-Master/Assets/XR/Temp.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: acb11a8edd06879488f49f2a67973a93
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity-Master/Assets/XR/Temp/XRSimulationPreferences.asset.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 97cf5143ccdc1af44bf5bbfdf47d31b6
+NativeFormatImporter:
+  externalObjects: {}
+  mainObjectFileID: 11400000
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity-Master/Assets/XR/Temp/XRSimulationRuntimeSettings.asset.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: b3f0e4c8e44049f4383103acf7518cab
+NativeFormatImporter:
+  externalObjects: {}
+  mainObjectFileID: 11400000
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Unity-Master/Assets/XR/UserSimulationSettings/Resources/XRSimulationPreferences.asset.meta

@@ -1,5 +1,5 @@
 fileFormatVersion: 2
-guid: 97cf5143ccdc1af44bf5bbfdf47d31b6
+guid: b3ae67cc68a994449a213f10f65698fd
 NativeFormatImporter:
   externalObjects: {}
   mainObjectFileID: 11400000