using System; using System.Text; using Convai.Scripts.Runtime.Multiplayer; using UnityEngine; using UnityEngine.InputSystem; using UnityEngine.UI; namespace Convai.Scripts.Runtime.Multiplayer { /// /// In-game debug UI for network diagnostics in VR builds /// Shows peer discovery, voice/speech status, and packet counters /// public class NetworkDebugUI : MonoBehaviour { [Header("UI Configuration")] [SerializeField] private bool showOnStart = true; [SerializeField] private KeyCode toggleKey = KeyCode.F1; [SerializeField] private bool useVRInput = true; [Header("UI Positioning (VR)")] [SerializeField] private float distanceFromCamera = 2.5f; [SerializeField] private float verticalOffset = 0.5f; [SerializeField] private Vector3 rotationOffset = new Vector3(0, 0, 0); [Header("Update Settings")] [SerializeField] private float updateInterval = 0.5f; // Update UI twice per second // UI Components private Canvas _canvas; private Text _debugText; private GameObject _panel; private Camera _mainCamera; // Component references private UDPPeerDiscovery _peerDiscovery; private ConvaiSimpleUDPAudioSender _audioSender; private ConvaiUDPSpeechSender _speechSender; private ConvaiSimpleUDPAudioReceiver _audioReceiver; private ConvaiUDPSpeechReceiver _speechReceiver; // State private bool _isVisible = true; private float _lastUpdateTime; private InputAction _toggleAction; // Packet tracking for enhanced debugging private System.Collections.Generic.Queue _packetLog = new System.Collections.Generic.Queue(); private const int MAX_PACKET_LOG = 15; private int _lastAudioSentCount = 0; private int _lastAudioReceivedCount = 0; private int _lastSpeechSentCount = 0; private int _lastSpeechReceivedCount = 0; private float _lastRateCheckTime = 0f; private void Start() { _mainCamera = Camera.main; // Find components FindNetworkComponents(); // Create UI CreateDebugUI(); _isVisible = showOnStart; _panel.SetActive(_isVisible); // Setup VR input for toggle if (useVRInput) { SetupVRToggleInput(); } UpdateDebugInfo(); } private void FindNetworkComponents() { _peerDiscovery = UDPPeerDiscovery.Instance; // Always re-scan for the best (most active) component // This is important for components on NPCs that get enabled/disabled _audioSender = FindBestComponent(); _speechSender = FindBestComponent(); _audioReceiver = FindBestComponent(); _speechReceiver = FindBestComponent(); } /// /// Finds a component, prioritizing enabled ones in active hierarchies /// private T FindBestComponent() where T : MonoBehaviour { var allComponents = FindObjectsOfType(true); // Include inactive if (allComponents.Length == 0) return null; // Priority 1: Enabled component in active hierarchy foreach (var comp in allComponents) { if (comp.enabled && comp.gameObject.activeInHierarchy) return comp; } // Priority 2: Component on active GameObject (even if component disabled) foreach (var comp in allComponents) { if (comp.gameObject.activeInHierarchy) return comp; } // Priority 3: Any component (even if GameObject is inactive) return allComponents[0]; } private void CreateDebugUI() { // Create canvas GameObject canvasObj = new GameObject("NetworkDebugCanvas"); canvasObj.transform.SetParent(transform); _canvas = canvasObj.AddComponent(); _canvas.renderMode = RenderMode.WorldSpace; CanvasScaler scaler = canvasObj.AddComponent(); scaler.dynamicPixelsPerUnit = 10; // Create panel background _panel = new GameObject("DebugPanel"); _panel.transform.SetParent(canvasObj.transform, false); Image panelImage = _panel.AddComponent(); panelImage.color = new Color(0, 0, 0, 0.85f); RectTransform panelRect = _panel.GetComponent(); panelRect.sizeDelta = new Vector2(900, 1200); // Increased size for more info // Create text GameObject textObj = new GameObject("DebugText"); textObj.transform.SetParent(_panel.transform, false); _debugText = textObj.AddComponent(); _debugText.font = Resources.GetBuiltinResource("LegacyRuntime.ttf"); _debugText.fontSize = 18; _debugText.color = Color.white; _debugText.alignment = TextAnchor.UpperLeft; _debugText.horizontalOverflow = HorizontalWrapMode.Overflow; _debugText.verticalOverflow = VerticalWrapMode.Overflow; RectTransform textRect = textObj.GetComponent(); textRect.anchorMin = new Vector2(0, 0); textRect.anchorMax = new Vector2(1, 1); textRect.offsetMin = new Vector2(20, 20); textRect.offsetMax = new Vector2(-20, -20); // Setup canvas transform for VR RectTransform canvasRect = _canvas.GetComponent(); canvasRect.sizeDelta = new Vector2(900, 1200); canvasRect.localScale = Vector3.one * 0.001f; // Scale down for VR viewing } private void SetupVRToggleInput() { try { // Create input action for Y/B button (left controller secondary button) _toggleAction = new InputAction("ToggleDebug", InputActionType.Button); // Bind to left controller secondary button (Y on Quest) _toggleAction.AddBinding("{LeftHand}/secondaryButton"); _toggleAction.AddBinding("{LeftHand}/secondaryButton"); _toggleAction.AddBinding("{LeftHand}/secondaryButton"); _toggleAction.AddBinding("{LeftHand}/secondaryButton"); // Also bind keyboard for editor testing _toggleAction.AddBinding("/f1"); _toggleAction.started += ctx => ToggleVisibility(); _toggleAction.Enable(); } catch (Exception ex) { Debug.LogWarning($"Failed to setup VR toggle input: {ex.Message}"); } } private void Update() { // Handle keyboard toggle if (Input.GetKeyDown(toggleKey)) { ToggleVisibility(); } // Update UI text periodically if (Time.time - _lastUpdateTime >= updateInterval) { UpdateDebugInfo(); _lastUpdateTime = Time.time; } // Position UI in front of camera if (_isVisible && _mainCamera != null) { PositionUIInFrontOfCamera(); } } private void PositionUIInFrontOfCamera() { Vector3 cameraForward = _mainCamera.transform.forward; Vector3 cameraRight = _mainCamera.transform.right; Vector3 cameraUp = _mainCamera.transform.up; // Position in front of camera Vector3 targetPosition = _mainCamera.transform.position + cameraForward * distanceFromCamera + cameraUp * verticalOffset; _canvas.transform.position = targetPosition; // Rotate to face camera _canvas.transform.rotation = Quaternion.LookRotation(cameraForward, cameraUp); _canvas.transform.Rotate(rotationOffset); } private void ToggleVisibility() { _isVisible = !_isVisible; _panel.SetActive(_isVisible); } private void UpdateDebugInfo() { if (_debugText == null) return; // Re-check for components that might have been disabled at startup FindNetworkComponents(); StringBuilder sb = new StringBuilder(); // Header sb.AppendLine("═══ NETWORK DEBUG ═══"); sb.AppendLine($"Time: {DateTime.Now:HH:mm:ss}"); sb.AppendLine(); // Port Binding Status AppendPortBindingStatus(sb); sb.AppendLine(); // Peer Discovery Status AppendPeerDiscoveryStatus(sb); sb.AppendLine(); // Audio Sender (Voice Input) AppendAudioSenderStatus(sb); sb.AppendLine(); // Speech Sender (NPC Response) AppendSpeechSenderStatus(sb); sb.AppendLine(); // Audio Receiver AppendAudioReceiverStatus(sb); sb.AppendLine(); // Speech Receiver AppendSpeechReceiverStatus(sb); sb.AppendLine(); // Packet Rates AppendPacketRates(sb); sb.AppendLine(); // Event Log AppendEventLog(sb); sb.AppendLine(); // Packet Log AppendPacketLog(sb); _debugText.text = sb.ToString(); } private void AppendPortBindingStatus(StringBuilder sb) { sb.AppendLine("🔌 PORT BINDING STATUS"); // Check if shared listener is active bool sharedListenerActive = SharedUDPListener.Instance != null && SharedUDPListener.Instance.IsListening; if (sharedListenerActive) { sb.AppendLine($"✅ Shared UDP Listener ACTIVE"); sb.AppendLine($" Port: {SharedUDPListener.Instance.ListenPort}"); sb.AppendLine($" Total Packets: {SharedUDPListener.Instance.TotalPacketsReceived}"); } else { sb.AppendLine($"❌ Shared UDP Listener NOT FOUND!"); sb.AppendLine($" Add SharedUDPListener to scene!"); } // Check which components are subscribed bool discoveryActive = _peerDiscovery != null; bool audioReceiverActive = _audioReceiver != null; bool speechReceiverActive = _speechReceiver != null; sb.AppendLine($"Discovery: {(discoveryActive ? "✅ SUBSCRIBED" : "❌ NOT FOUND")} (0x44495343)"); sb.AppendLine($"Audio RX: {(audioReceiverActive ? "✅ SUBSCRIBED" : "❌ NOT FOUND")} (0xC0A1)"); sb.AppendLine($"Speech RX: {(speechReceiverActive ? "✅ SUBSCRIBED" : "❌ NOT FOUND")} (0xC0A3)"); if (!sharedListenerActive) { sb.AppendLine("⚠️ CRITICAL: No shared listener!"); } } private void AppendPeerDiscoveryStatus(StringBuilder sb) { sb.AppendLine("🔍 PEER DISCOVERY"); if (_peerDiscovery != null) { string stateColor = GetConnectionStateColor(_peerDiscovery.CurrentState); string stateIcon = GetConnectionStateIcon(_peerDiscovery.CurrentState); sb.AppendLine($"State: {_peerDiscovery.CurrentState} {stateIcon}"); sb.AppendLine($"Local Player ID: {_peerDiscovery.LocalPlayerID}"); sb.AppendLine($"Peer Player ID: {(_peerDiscovery.PeerPlayerID > 0 ? _peerDiscovery.PeerPlayerID.ToString() : "None")}"); sb.AppendLine($"Peer IP: {(_peerDiscovery.IsConnected ? _peerDiscovery.PeerIP : "None")}"); if (_peerDiscovery.IsConnected) { float timeSince = _peerDiscovery.TimeSinceLastPeerPacket; sb.AppendLine($"Last Packet: {timeSince:F1}s ago"); } } else { sb.AppendLine("❌ NOT FOUND - Add UDPPeerDiscovery component!"); } } private void AppendAudioSenderStatus(StringBuilder sb) { sb.AppendLine("🎤 VOICE INPUT SENDER"); if (_audioSender != null && _audioSender.gameObject != null) { // Check if component's GameObject is active AND enabled in hierarchy bool isActive = _audioSender.enabled && _audioSender.gameObject.activeInHierarchy; if (!isActive) { sb.AppendLine("⏸️ FOUND BUT DISABLED"); sb.AppendLine($"(GameObject: {_audioSender.gameObject.name})"); sb.AppendLine($"(Active: {_audioSender.gameObject.activeSelf}, InHierarchy: {_audioSender.gameObject.activeInHierarchy})"); } else { sb.AppendLine($"GameObject: {_audioSender.gameObject.name} ✅"); sb.AppendLine($"Target: {_audioSender.CurrentTargetIP}:{_audioSender.CurrentTargetPort}"); sb.AppendLine($"Recording: {(_audioSender.IsRecording ? "YES ✅" : "NO")}"); sb.AppendLine($"Packets Sent: {_audioSender.TotalPacketsSent}"); float timeSince = _audioSender.TimeSinceLastSend; if (timeSince >= 0) { sb.AppendLine($"Last Send: {timeSince:F1}s ago"); } else { sb.AppendLine($"Last Send: Never"); } sb.AppendLine($"Using Discovery: {(_audioSender.UsingDiscovery ? "YES" : "NO")}"); // WARNING if not sending packets while recording if (_audioSender.IsRecording && _audioSender.TotalPacketsSent == 0) { sb.AppendLine("⚠️ RECORDING BUT NO PACKETS SENT!"); } } } else { sb.AppendLine("❌ NOT FOUND IN SCENE"); } } private void AppendSpeechSenderStatus(StringBuilder sb) { sb.AppendLine("🔊 SPEECH SENDER"); if (_speechSender != null && _speechSender.gameObject != null) { // Check if component's GameObject is active AND enabled in hierarchy bool isActive = _speechSender.enabled && _speechSender.gameObject.activeInHierarchy; if (!isActive) { sb.AppendLine("⏸️ FOUND BUT DISABLED"); sb.AppendLine($"(GameObject: {_speechSender.gameObject.name})"); sb.AppendLine($"(Active: {_speechSender.gameObject.activeSelf}, InHierarchy: {_speechSender.gameObject.activeInHierarchy})"); } else { sb.AppendLine($"GameObject: {_speechSender.gameObject.name} ✅"); sb.AppendLine($"Target: {_speechSender.CurrentTargetIP}:{_speechSender.CurrentTargetPort}"); // Show source NPC var sourceNPC = _speechSender.SourceNPC; if (sourceNPC != null) { sb.AppendLine($"Source NPC: {sourceNPC.characterName} ✅"); // Show if NPC is currently talking if (sourceNPC.IsCharacterTalking) { sb.AppendLine($"NPC Talking: YES 🗣️"); } } else { sb.AppendLine($"Source NPC: None ⚠️"); } sb.AppendLine($"Transmitting: {(_speechSender.IsSendingSpeech ? "YES ✅" : "NO")}"); sb.AppendLine($"Clips Sent: {_speechSender.TotalClipsSent}"); float timeSince = _speechSender.TimeSinceLastSend; if (timeSince >= 0) { sb.AppendLine($"Last Send: {timeSince:F1}s ago"); } else { sb.AppendLine($"Last Send: Never"); } sb.AppendLine($"Using Discovery: {(_speechSender.UsingDiscovery ? "YES" : "NO")}"); // WARNING if transmitting but no clips sent if (_speechSender.IsSendingSpeech && _speechSender.TotalClipsSent == 0) { sb.AppendLine("⚠️ TRANSMITTING BUT NO CLIPS SENT!"); } } } else { sb.AppendLine("❌ NOT FOUND IN SCENE"); } } private void AppendAudioReceiverStatus(StringBuilder sb) { sb.AppendLine("🎧 VOICE INPUT RECEIVER"); if (_audioReceiver != null && _audioReceiver.gameObject != null) { // Check if component's GameObject is active AND enabled in hierarchy bool isActive = _audioReceiver.enabled && _audioReceiver.gameObject.activeInHierarchy; if (!isActive) { sb.AppendLine("⏸️ FOUND BUT DISABLED"); sb.AppendLine($"(GameObject: {_audioReceiver.gameObject.name})"); sb.AppendLine($"(Active: {_audioReceiver.gameObject.activeSelf}, InHierarchy: {_audioReceiver.gameObject.activeInHierarchy})"); } else { sb.AppendLine($"GameObject: {_audioReceiver.gameObject.name} ✅"); sb.AppendLine($"Listen Port: {_audioReceiver.ListenPort} (shared)"); sb.AppendLine($"Listening: {(_audioReceiver.IsListening ? "YES ✅" : "NO ❌")}"); // Show active NPC var targetNPC = _audioReceiver.TargetNPC; if (targetNPC != null) { sb.AppendLine($"Active NPC: {targetNPC.characterName} ✅"); } else { sb.AppendLine($"Active NPC: None ⚠️"); } sb.AppendLine($"Receiving: {(_audioReceiver.IsReceivingAudio ? "YES ✅" : "NO")}"); sb.AppendLine($"Packets Received: {_audioReceiver.TotalPacketsReceived}"); float timeSince = _audioReceiver.TimeSinceLastReceive; if (timeSince >= 0) { sb.AppendLine($"Last Receive: {timeSince:F1}s ago"); } else { sb.AppendLine($"Last Receive: Never"); } // WARNING if not listening if (!_audioReceiver.IsListening) { sb.AppendLine("⚠️ NOT LISTENING - PORT BIND FAILED?"); } } } else { sb.AppendLine("❌ NOT FOUND IN SCENE"); } } private void AppendSpeechReceiverStatus(StringBuilder sb) { sb.AppendLine("🔉 SPEECH RECEIVER"); if (_speechReceiver != null && _speechReceiver.gameObject != null) { // Check if component's GameObject is active AND enabled in hierarchy bool isActive = _speechReceiver.enabled && _speechReceiver.gameObject.activeInHierarchy; if (!isActive) { sb.AppendLine("⏸️ FOUND BUT DISABLED"); sb.AppendLine($"(GameObject: {_speechReceiver.gameObject.name})"); sb.AppendLine($"(Active: {_speechReceiver.gameObject.activeSelf}, InHierarchy: {_speechReceiver.gameObject.activeInHierarchy})"); } else { sb.AppendLine($"GameObject: {_speechReceiver.gameObject.name} ✅"); sb.AppendLine($"Listen Port: {_speechReceiver.ListenPort} (shared)"); sb.AppendLine($"Listening: {(_speechReceiver.IsListening ? "YES ✅" : "NO ❌")}"); sb.AppendLine($"Playing: {(_speechReceiver.IsPlayingSequence ? "YES ✅" : "NO")}"); sb.AppendLine($"Clips Received: {_speechReceiver.TotalClipsReceived}"); sb.AppendLine($"Queued Clips: {_speechReceiver.QueuedClipCount}"); float timeSince = _speechReceiver.TimeSinceLastReceive; if (timeSince >= 0) { sb.AppendLine($"Last Receive: {timeSince:F1}s ago"); } else { sb.AppendLine($"Last Receive: Never"); } // WARNING if not listening if (!_speechReceiver.IsListening) { sb.AppendLine("⚠️ NOT LISTENING - PORT BIND FAILED?"); } } } else { sb.AppendLine("❌ NOT FOUND IN SCENE"); } } private void AppendPacketRates(StringBuilder sb) { sb.AppendLine("📊 PACKET RATES (last second)"); float currentTime = Time.time; float deltaTime = currentTime - _lastRateCheckTime; if (deltaTime >= 1f) { // Calculate rates int audioSentDelta = 0; int audioReceivedDelta = 0; int speechSentDelta = 0; int speechReceivedDelta = 0; if (_audioSender != null) { audioSentDelta = _audioSender.TotalPacketsSent - _lastAudioSentCount; _lastAudioSentCount = _audioSender.TotalPacketsSent; } if (_audioReceiver != null) { audioReceivedDelta = _audioReceiver.TotalPacketsReceived - _lastAudioReceivedCount; _lastAudioReceivedCount = _audioReceiver.TotalPacketsReceived; } if (_speechSender != null) { speechSentDelta = _speechSender.TotalClipsSent - _lastSpeechSentCount; _lastSpeechSentCount = _speechSender.TotalClipsSent; } if (_speechReceiver != null) { speechReceivedDelta = _speechReceiver.TotalClipsReceived - _lastSpeechReceivedCount; _lastSpeechReceivedCount = _speechReceiver.TotalClipsReceived; } _lastRateCheckTime = currentTime; // Log significant activity if (audioSentDelta > 0) LogPacketActivity($"Sent {audioSentDelta} voice packets (0xC0A1)"); if (audioReceivedDelta > 0) LogPacketActivity($"Received {audioReceivedDelta} voice packets (0xC0A1)"); if (speechSentDelta > 0) LogPacketActivity($"Sent {speechSentDelta} speech clips (0xC0A3)"); if (speechReceivedDelta > 0) LogPacketActivity($"Received {speechReceivedDelta} speech clips (0xC0A3)"); } sb.AppendLine($"Voice Sent: {(_audioSender != null ? _audioSender.TotalPacketsSent : 0)} total"); sb.AppendLine($"Voice Received: {(_audioReceiver != null ? _audioReceiver.TotalPacketsReceived : 0)} total"); sb.AppendLine($"Speech Sent: {(_speechSender != null ? _speechSender.TotalClipsSent : 0)} clips"); sb.AppendLine($"Speech Received: {(_speechReceiver != null ? _speechReceiver.TotalClipsReceived : 0)} clips"); } private void LogPacketActivity(string message) { string timestamp = DateTime.Now.ToString("HH:mm:ss"); _packetLog.Enqueue($"[{timestamp}] {message}"); while (_packetLog.Count > MAX_PACKET_LOG) { _packetLog.Dequeue(); } } private void AppendPacketLog(StringBuilder sb) { sb.AppendLine("📦 PACKET ACTIVITY LOG"); if (_packetLog.Count > 0) { foreach (var entry in _packetLog) { sb.AppendLine(entry); } } else { sb.AppendLine("No packet activity yet"); } } private void AppendEventLog(StringBuilder sb) { sb.AppendLine("📋 CONNECTION EVENTS"); if (_peerDiscovery != null && _peerDiscovery.EventLog.Count > 0) { int startIndex = Math.Max(0, _peerDiscovery.EventLog.Count - 6); for (int i = startIndex; i < _peerDiscovery.EventLog.Count; i++) { sb.AppendLine(_peerDiscovery.EventLog[i]); } } else { sb.AppendLine("No events yet"); } } private string GetConnectionStateColor(UDPPeerDiscovery.ConnectionState state) { return state switch { UDPPeerDiscovery.ConnectionState.Connected => "green", UDPPeerDiscovery.ConnectionState.Discovering => "yellow", UDPPeerDiscovery.ConnectionState.Lost => "red", _ => "white" }; } private string GetConnectionStateIcon(UDPPeerDiscovery.ConnectionState state) { return state switch { UDPPeerDiscovery.ConnectionState.Connected => "✅", UDPPeerDiscovery.ConnectionState.Discovering => "⏳", UDPPeerDiscovery.ConnectionState.Lost => "❌", UDPPeerDiscovery.ConnectionState.Disconnected => "⭕", _ => "❓" }; } private void OnDestroy() { if (_toggleAction != null) { _toggleAction.Disable(); _toggleAction.Dispose(); } } // Public methods for external control public void Show() { _isVisible = true; _panel.SetActive(true); } public void Hide() { _isVisible = false; _panel.SetActive(false); } } }