enhanced UDP audio sender/receiver scripts with metrics for packet tracking and improved NPC assignment logic
This commit is contained in:
tom.hempel
541
Unity-Master/Assets/Scripts/Multiplayer/NetworkDebugUI.cs
Normal file
541
Unity-Master/Assets/Scripts/Multiplayer/NetworkDebugUI.cs
Normal file
@ -0,0 +1,541 @@
|
||||
using System;
|
||||
using System.Text;
|
||||
using Convai.Scripts.Runtime.Multiplayer;
|
||||
using UnityEngine;
|
||||
using UnityEngine.InputSystem;
|
||||
using UnityEngine.UI;
|
||||
|
||||
namespace Convai.Scripts.Runtime.Multiplayer
|
||||
{
|
||||
/// <summary>
|
||||
/// In-game debug UI for network diagnostics in VR builds
|
||||
/// Shows peer discovery, voice/speech status, and packet counters
|
||||
/// </summary>
|
||||
public class NetworkDebugUI : MonoBehaviour
|
||||
{
|
||||
[Header("UI Configuration")]
|
||||
[SerializeField] private bool showOnStart = true;
|
||||
[SerializeField] private KeyCode toggleKey = KeyCode.F1;
|
||||
[SerializeField] private bool useVRInput = true;
|
||||
|
||||
[Header("UI Positioning (VR)")]
|
||||
[SerializeField] private float distanceFromCamera = 2.5f;
|
||||
[SerializeField] private float verticalOffset = 0.5f;
|
||||
[SerializeField] private Vector3 rotationOffset = new Vector3(0, 0, 0);
|
||||
|
||||
[Header("Update Settings")]
|
||||
[SerializeField] private float updateInterval = 0.5f; // Update UI twice per second
|
||||
|
||||
// UI Components
|
||||
private Canvas _canvas;
|
||||
private Text _debugText;
|
||||
private GameObject _panel;
|
||||
private Camera _mainCamera;
|
||||
|
||||
// Component references
|
||||
private UDPPeerDiscovery _peerDiscovery;
|
||||
private ConvaiSimpleUDPAudioSender _audioSender;
|
||||
private ConvaiUDPSpeechSender _speechSender;
|
||||
private ConvaiSimpleUDPAudioReceiver _audioReceiver;
|
||||
private ConvaiUDPSpeechReceiver _speechReceiver;
|
||||
|
||||
// State
|
||||
private bool _isVisible = true;
|
||||
private float _lastUpdateTime;
|
||||
private InputAction _toggleAction;
|
||||
|
||||
private void Start()
|
||||
{
|
||||
_mainCamera = Camera.main;
|
||||
|
||||
// Find components
|
||||
FindNetworkComponents();
|
||||
|
||||
// Create UI
|
||||
CreateDebugUI();
|
||||
|
||||
_isVisible = showOnStart;
|
||||
_panel.SetActive(_isVisible);
|
||||
|
||||
// Setup VR input for toggle
|
||||
if (useVRInput)
|
||||
{
|
||||
SetupVRToggleInput();
|
||||
}
|
||||
|
||||
UpdateDebugInfo();
|
||||
}
|
||||
|
||||
private void FindNetworkComponents()
|
||||
{
|
||||
_peerDiscovery = UDPPeerDiscovery.Instance;
|
||||
|
||||
// Always re-scan for the best (most active) component
|
||||
// This is important for components on NPCs that get enabled/disabled
|
||||
_audioSender = FindBestComponent<ConvaiSimpleUDPAudioSender>();
|
||||
_speechSender = FindBestComponent<ConvaiUDPSpeechSender>();
|
||||
_audioReceiver = FindBestComponent<ConvaiSimpleUDPAudioReceiver>();
|
||||
_speechReceiver = FindBestComponent<ConvaiUDPSpeechReceiver>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds a component, prioritizing enabled ones in active hierarchies
|
||||
/// </summary>
|
||||
private T FindBestComponent<T>() where T : MonoBehaviour
|
||||
{
|
||||
var allComponents = FindObjectsOfType<T>(true); // Include inactive
|
||||
|
||||
if (allComponents.Length == 0)
|
||||
return null;
|
||||
|
||||
// Priority 1: Enabled component in active hierarchy
|
||||
foreach (var comp in allComponents)
|
||||
{
|
||||
if (comp.enabled && comp.gameObject.activeInHierarchy)
|
||||
return comp;
|
||||
}
|
||||
|
||||
// Priority 2: Component on active GameObject (even if component disabled)
|
||||
foreach (var comp in allComponents)
|
||||
{
|
||||
if (comp.gameObject.activeInHierarchy)
|
||||
return comp;
|
||||
}
|
||||
|
||||
// Priority 3: Any component (even if GameObject is inactive)
|
||||
return allComponents[0];
|
||||
}
|
||||
|
||||
private void CreateDebugUI()
|
||||
{
|
||||
// Create canvas
|
||||
GameObject canvasObj = new GameObject("NetworkDebugCanvas");
|
||||
canvasObj.transform.SetParent(transform);
|
||||
|
||||
_canvas = canvasObj.AddComponent<Canvas>();
|
||||
_canvas.renderMode = RenderMode.WorldSpace;
|
||||
|
||||
CanvasScaler scaler = canvasObj.AddComponent<CanvasScaler>();
|
||||
scaler.dynamicPixelsPerUnit = 10;
|
||||
|
||||
// Create panel background
|
||||
_panel = new GameObject("DebugPanel");
|
||||
_panel.transform.SetParent(canvasObj.transform, false);
|
||||
|
||||
Image panelImage = _panel.AddComponent<Image>();
|
||||
panelImage.color = new Color(0, 0, 0, 0.85f);
|
||||
|
||||
RectTransform panelRect = _panel.GetComponent<RectTransform>();
|
||||
panelRect.sizeDelta = new Vector2(800, 900);
|
||||
|
||||
// Create text
|
||||
GameObject textObj = new GameObject("DebugText");
|
||||
textObj.transform.SetParent(_panel.transform, false);
|
||||
|
||||
_debugText = textObj.AddComponent<Text>();
|
||||
_debugText.font = Resources.GetBuiltinResource<Font>("LegacyRuntime.ttf");
|
||||
_debugText.fontSize = 18;
|
||||
_debugText.color = Color.white;
|
||||
_debugText.alignment = TextAnchor.UpperLeft;
|
||||
_debugText.horizontalOverflow = HorizontalWrapMode.Overflow;
|
||||
_debugText.verticalOverflow = VerticalWrapMode.Overflow;
|
||||
|
||||
RectTransform textRect = textObj.GetComponent<RectTransform>();
|
||||
textRect.anchorMin = new Vector2(0, 0);
|
||||
textRect.anchorMax = new Vector2(1, 1);
|
||||
textRect.offsetMin = new Vector2(20, 20);
|
||||
textRect.offsetMax = new Vector2(-20, -20);
|
||||
|
||||
// Setup canvas transform for VR
|
||||
RectTransform canvasRect = _canvas.GetComponent<RectTransform>();
|
||||
canvasRect.sizeDelta = new Vector2(800, 900);
|
||||
canvasRect.localScale = Vector3.one * 0.001f; // Scale down for VR viewing
|
||||
}
|
||||
|
||||
private void SetupVRToggleInput()
|
||||
{
|
||||
try
|
||||
{
|
||||
// Create input action for Y/B button (left controller secondary button)
|
||||
_toggleAction = new InputAction("ToggleDebug", InputActionType.Button);
|
||||
|
||||
// Bind to left controller secondary button (Y on Quest)
|
||||
_toggleAction.AddBinding("<XRController>{LeftHand}/secondaryButton");
|
||||
_toggleAction.AddBinding("<OculusTouchController>{LeftHand}/secondaryButton");
|
||||
_toggleAction.AddBinding("<MetaTouchController>{LeftHand}/secondaryButton");
|
||||
_toggleAction.AddBinding("<QuestProTouchController>{LeftHand}/secondaryButton");
|
||||
|
||||
// Also bind keyboard for editor testing
|
||||
_toggleAction.AddBinding("<Keyboard>/f1");
|
||||
|
||||
_toggleAction.started += ctx => ToggleVisibility();
|
||||
_toggleAction.Enable();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Debug.LogWarning($"Failed to setup VR toggle input: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private void Update()
|
||||
{
|
||||
// Handle keyboard toggle
|
||||
if (Input.GetKeyDown(toggleKey))
|
||||
{
|
||||
ToggleVisibility();
|
||||
}
|
||||
|
||||
// Update UI text periodically
|
||||
if (Time.time - _lastUpdateTime >= updateInterval)
|
||||
{
|
||||
UpdateDebugInfo();
|
||||
_lastUpdateTime = Time.time;
|
||||
}
|
||||
|
||||
// Position UI in front of camera
|
||||
if (_isVisible && _mainCamera != null)
|
||||
{
|
||||
PositionUIInFrontOfCamera();
|
||||
}
|
||||
}
|
||||
|
||||
private void PositionUIInFrontOfCamera()
|
||||
{
|
||||
Vector3 cameraForward = _mainCamera.transform.forward;
|
||||
Vector3 cameraRight = _mainCamera.transform.right;
|
||||
Vector3 cameraUp = _mainCamera.transform.up;
|
||||
|
||||
// Position in front of camera
|
||||
Vector3 targetPosition = _mainCamera.transform.position +
|
||||
cameraForward * distanceFromCamera +
|
||||
cameraUp * verticalOffset;
|
||||
|
||||
_canvas.transform.position = targetPosition;
|
||||
|
||||
// Rotate to face camera
|
||||
_canvas.transform.rotation = Quaternion.LookRotation(cameraForward, cameraUp);
|
||||
_canvas.transform.Rotate(rotationOffset);
|
||||
}
|
||||
|
||||
private void ToggleVisibility()
|
||||
{
|
||||
_isVisible = !_isVisible;
|
||||
_panel.SetActive(_isVisible);
|
||||
}
|
||||
|
||||
private void UpdateDebugInfo()
|
||||
{
|
||||
if (_debugText == null) return;
|
||||
|
||||
// Re-check for components that might have been disabled at startup
|
||||
FindNetworkComponents();
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
// Header
|
||||
sb.AppendLine("═══ NETWORK DEBUG ═══");
|
||||
sb.AppendLine($"Time: {DateTime.Now:HH:mm:ss}");
|
||||
sb.AppendLine();
|
||||
|
||||
// Peer Discovery Status
|
||||
AppendPeerDiscoveryStatus(sb);
|
||||
sb.AppendLine();
|
||||
|
||||
// Audio Sender (Voice Input)
|
||||
AppendAudioSenderStatus(sb);
|
||||
sb.AppendLine();
|
||||
|
||||
// Speech Sender (NPC Response)
|
||||
AppendSpeechSenderStatus(sb);
|
||||
sb.AppendLine();
|
||||
|
||||
// Audio Receiver
|
||||
AppendAudioReceiverStatus(sb);
|
||||
sb.AppendLine();
|
||||
|
||||
// Speech Receiver
|
||||
AppendSpeechReceiverStatus(sb);
|
||||
sb.AppendLine();
|
||||
|
||||
// Event Log
|
||||
AppendEventLog(sb);
|
||||
|
||||
_debugText.text = sb.ToString();
|
||||
}
|
||||
|
||||
private void AppendPeerDiscoveryStatus(StringBuilder sb)
|
||||
{
|
||||
sb.AppendLine("🔍 PEER DISCOVERY");
|
||||
|
||||
if (_peerDiscovery != null)
|
||||
{
|
||||
string stateColor = GetConnectionStateColor(_peerDiscovery.CurrentState);
|
||||
string stateIcon = GetConnectionStateIcon(_peerDiscovery.CurrentState);
|
||||
|
||||
sb.AppendLine($"State: {_peerDiscovery.CurrentState} {stateIcon}");
|
||||
sb.AppendLine($"Local Player ID: {_peerDiscovery.LocalPlayerID}");
|
||||
sb.AppendLine($"Peer Player ID: {(_peerDiscovery.PeerPlayerID > 0 ? _peerDiscovery.PeerPlayerID.ToString() : "None")}");
|
||||
sb.AppendLine($"Peer IP: {(_peerDiscovery.IsConnected ? _peerDiscovery.PeerIP : "None")}");
|
||||
|
||||
if (_peerDiscovery.IsConnected)
|
||||
{
|
||||
float timeSince = _peerDiscovery.TimeSinceLastPeerPacket;
|
||||
sb.AppendLine($"Last Packet: {timeSince:F1}s ago");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine("❌ NOT FOUND - Add UDPPeerDiscovery component!");
|
||||
}
|
||||
}
|
||||
|
||||
private void AppendAudioSenderStatus(StringBuilder sb)
|
||||
{
|
||||
sb.AppendLine("🎤 VOICE INPUT SENDER");
|
||||
|
||||
if (_audioSender != null && _audioSender.gameObject != null)
|
||||
{
|
||||
// Check if component's GameObject is active AND enabled in hierarchy
|
||||
bool isActive = _audioSender.enabled && _audioSender.gameObject.activeInHierarchy;
|
||||
|
||||
if (!isActive)
|
||||
{
|
||||
sb.AppendLine("⏸️ FOUND BUT DISABLED");
|
||||
sb.AppendLine($"(GameObject: {_audioSender.gameObject.name})");
|
||||
sb.AppendLine($"(Active: {_audioSender.gameObject.activeSelf}, InHierarchy: {_audioSender.gameObject.activeInHierarchy})");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($"GameObject: {_audioSender.gameObject.name} ✅");
|
||||
sb.AppendLine($"Target: {_audioSender.CurrentTargetIP}:{_audioSender.CurrentTargetPort}");
|
||||
sb.AppendLine($"Recording: {(_audioSender.IsRecording ? "YES ✅" : "NO")}");
|
||||
sb.AppendLine($"Packets Sent: {_audioSender.TotalPacketsSent}");
|
||||
|
||||
float timeSince = _audioSender.TimeSinceLastSend;
|
||||
if (timeSince >= 0)
|
||||
{
|
||||
sb.AppendLine($"Last Send: {timeSince:F1}s ago");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($"Last Send: Never");
|
||||
}
|
||||
|
||||
sb.AppendLine($"Using Discovery: {(_audioSender.UsingDiscovery ? "YES" : "NO")}");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine("❌ NOT FOUND IN SCENE");
|
||||
}
|
||||
}
|
||||
|
||||
private void AppendSpeechSenderStatus(StringBuilder sb)
|
||||
{
|
||||
sb.AppendLine("🔊 SPEECH SENDER");
|
||||
|
||||
if (_speechSender != null && _speechSender.gameObject != null)
|
||||
{
|
||||
// Check if component's GameObject is active AND enabled in hierarchy
|
||||
bool isActive = _speechSender.enabled && _speechSender.gameObject.activeInHierarchy;
|
||||
|
||||
if (!isActive)
|
||||
{
|
||||
sb.AppendLine("⏸️ FOUND BUT DISABLED");
|
||||
sb.AppendLine($"(GameObject: {_speechSender.gameObject.name})");
|
||||
sb.AppendLine($"(Active: {_speechSender.gameObject.activeSelf}, InHierarchy: {_speechSender.gameObject.activeInHierarchy})");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($"GameObject: {_speechSender.gameObject.name} ✅");
|
||||
sb.AppendLine($"Target: {_speechSender.CurrentTargetIP}:{_speechSender.CurrentTargetPort}");
|
||||
|
||||
// Show source NPC
|
||||
var sourceNPC = _speechSender.SourceNPC;
|
||||
if (sourceNPC != null)
|
||||
{
|
||||
sb.AppendLine($"Source NPC: {sourceNPC.characterName} ✅");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($"Source NPC: None ⚠️");
|
||||
}
|
||||
|
||||
sb.AppendLine($"Transmitting: {(_speechSender.IsSendingSpeech ? "YES ✅" : "NO")}");
|
||||
sb.AppendLine($"Clips Sent: {_speechSender.TotalClipsSent}");
|
||||
|
||||
float timeSince = _speechSender.TimeSinceLastSend;
|
||||
if (timeSince >= 0)
|
||||
{
|
||||
sb.AppendLine($"Last Send: {timeSince:F1}s ago");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($"Last Send: Never");
|
||||
}
|
||||
|
||||
sb.AppendLine($"Using Discovery: {(_speechSender.UsingDiscovery ? "YES" : "NO")}");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine("❌ NOT FOUND IN SCENE");
|
||||
}
|
||||
}
|
||||
|
||||
private void AppendAudioReceiverStatus(StringBuilder sb)
|
||||
{
|
||||
sb.AppendLine("🎧 VOICE INPUT RECEIVER");
|
||||
|
||||
if (_audioReceiver != null && _audioReceiver.gameObject != null)
|
||||
{
|
||||
// Check if component's GameObject is active AND enabled in hierarchy
|
||||
bool isActive = _audioReceiver.enabled && _audioReceiver.gameObject.activeInHierarchy;
|
||||
|
||||
if (!isActive)
|
||||
{
|
||||
sb.AppendLine("⏸️ FOUND BUT DISABLED");
|
||||
sb.AppendLine($"(GameObject: {_audioReceiver.gameObject.name})");
|
||||
sb.AppendLine($"(Active: {_audioReceiver.gameObject.activeSelf}, InHierarchy: {_audioReceiver.gameObject.activeInHierarchy})");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($"GameObject: {_audioReceiver.gameObject.name} ✅");
|
||||
sb.AppendLine($"Listen Port: {_audioReceiver.ListenPort}");
|
||||
|
||||
// Show active NPC
|
||||
var targetNPC = _audioReceiver.TargetNPC;
|
||||
if (targetNPC != null)
|
||||
{
|
||||
sb.AppendLine($"Active NPC: {targetNPC.characterName} ✅");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($"Active NPC: None ⚠️");
|
||||
}
|
||||
|
||||
sb.AppendLine($"Receiving: {(_audioReceiver.IsReceivingAudio ? "YES ✅" : "NO")}");
|
||||
sb.AppendLine($"Packets Received: {_audioReceiver.TotalPacketsReceived}");
|
||||
|
||||
float timeSince = _audioReceiver.TimeSinceLastReceive;
|
||||
if (timeSince >= 0)
|
||||
{
|
||||
sb.AppendLine($"Last Receive: {timeSince:F1}s ago");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($"Last Receive: Never");
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine("❌ NOT FOUND IN SCENE");
|
||||
}
|
||||
}
|
||||
|
||||
private void AppendSpeechReceiverStatus(StringBuilder sb)
|
||||
{
|
||||
sb.AppendLine("🔉 SPEECH RECEIVER");
|
||||
|
||||
if (_speechReceiver != null && _speechReceiver.gameObject != null)
|
||||
{
|
||||
// Check if component's GameObject is active AND enabled in hierarchy
|
||||
bool isActive = _speechReceiver.enabled && _speechReceiver.gameObject.activeInHierarchy;
|
||||
|
||||
if (!isActive)
|
||||
{
|
||||
sb.AppendLine("⏸️ FOUND BUT DISABLED");
|
||||
sb.AppendLine($"(GameObject: {_speechReceiver.gameObject.name})");
|
||||
sb.AppendLine($"(Active: {_speechReceiver.gameObject.activeSelf}, InHierarchy: {_speechReceiver.gameObject.activeInHierarchy})");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($"GameObject: {_speechReceiver.gameObject.name} ✅");
|
||||
sb.AppendLine($"Listen Port: {_speechReceiver.ListenPort}");
|
||||
sb.AppendLine($"Playing: {(_speechReceiver.IsPlayingSequence ? "YES ✅" : "NO")}");
|
||||
sb.AppendLine($"Clips Received: {_speechReceiver.TotalClipsReceived}");
|
||||
sb.AppendLine($"Queued Clips: {_speechReceiver.QueuedClipCount}");
|
||||
|
||||
float timeSince = _speechReceiver.TimeSinceLastReceive;
|
||||
if (timeSince >= 0)
|
||||
{
|
||||
sb.AppendLine($"Last Receive: {timeSince:F1}s ago");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($"Last Receive: Never");
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine("❌ NOT FOUND IN SCENE");
|
||||
}
|
||||
}
|
||||
|
||||
private void AppendEventLog(StringBuilder sb)
|
||||
{
|
||||
sb.AppendLine("📋 EVENT LOG");
|
||||
|
||||
if (_peerDiscovery != null && _peerDiscovery.EventLog.Count > 0)
|
||||
{
|
||||
int startIndex = Math.Max(0, _peerDiscovery.EventLog.Count - 8);
|
||||
for (int i = startIndex; i < _peerDiscovery.EventLog.Count; i++)
|
||||
{
|
||||
sb.AppendLine(_peerDiscovery.EventLog[i]);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine("No events yet");
|
||||
}
|
||||
}
|
||||
|
||||
private string GetConnectionStateColor(UDPPeerDiscovery.ConnectionState state)
|
||||
{
|
||||
return state switch
|
||||
{
|
||||
UDPPeerDiscovery.ConnectionState.Connected => "green",
|
||||
UDPPeerDiscovery.ConnectionState.Discovering => "yellow",
|
||||
UDPPeerDiscovery.ConnectionState.Lost => "red",
|
||||
_ => "white"
|
||||
};
|
||||
}
|
||||
|
||||
private string GetConnectionStateIcon(UDPPeerDiscovery.ConnectionState state)
|
||||
{
|
||||
return state switch
|
||||
{
|
||||
UDPPeerDiscovery.ConnectionState.Connected => "✅",
|
||||
UDPPeerDiscovery.ConnectionState.Discovering => "⏳",
|
||||
UDPPeerDiscovery.ConnectionState.Lost => "❌",
|
||||
UDPPeerDiscovery.ConnectionState.Disconnected => "⭕",
|
||||
_ => "❓"
|
||||
};
|
||||
}
|
||||
|
||||
private void OnDestroy()
|
||||
{
|
||||
if (_toggleAction != null)
|
||||
{
|
||||
_toggleAction.Disable();
|
||||
_toggleAction.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
// Public methods for external control
|
||||
public void Show()
|
||||
{
|
||||
_isVisible = true;
|
||||
_panel.SetActive(true);
|
||||
}
|
||||
|
||||
public void Hide()
|
||||
{
|
||||
_isVisible = false;
|
||||
_panel.SetActive(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user