From 1bafa43d110d43ebd6355d09f9da9d6017fdf395 Mon Sep 17 00:00:00 2001 From: "tom.hempel" Date: Tue, 7 Oct 2025 15:36:07 +0200 Subject: [PATCH] created new UDP scripts for Agents --- .stfolder/syncthing-folder-9cdae2.txt | 5 + Unity-Master/Assets/Scenes/VR_Player1.unity | 4 +- Unity-Master/Assets/Scenes/VR_Player2.unity | 4 +- .../Scripts/README_AvatarSync_Agent.md.meta | 7 + .../Scripts/UDPAvatarBroadcasterAgent.cs | 487 ++++++++++++ .../Scripts/UDPAvatarBroadcasterAgent.cs.meta | 11 + .../Assets/Scripts/UDPAvatarReceiverAgent.cs | 726 ++++++++++++++++++ .../Scripts/UDPAvatarReceiverAgent.cs.meta | 11 + 8 files changed, 1251 insertions(+), 4 deletions(-) create mode 100644 .stfolder/syncthing-folder-9cdae2.txt create mode 100644 Unity-Master/Assets/Scripts/README_AvatarSync_Agent.md.meta create mode 100644 Unity-Master/Assets/Scripts/UDPAvatarBroadcasterAgent.cs create mode 100644 Unity-Master/Assets/Scripts/UDPAvatarBroadcasterAgent.cs.meta create mode 100644 Unity-Master/Assets/Scripts/UDPAvatarReceiverAgent.cs create mode 100644 Unity-Master/Assets/Scripts/UDPAvatarReceiverAgent.cs.meta diff --git a/.stfolder/syncthing-folder-9cdae2.txt b/.stfolder/syncthing-folder-9cdae2.txt new file mode 100644 index 0000000..d741204 --- /dev/null +++ b/.stfolder/syncthing-folder-9cdae2.txt @@ -0,0 +1,5 @@ +# This directory is a Syncthing folder marker. +# Do not delete. + +folderID: wvyum-ftfs2 +created: 2025-09-26T17:23:38+02:00 diff --git a/Unity-Master/Assets/Scenes/VR_Player1.unity b/Unity-Master/Assets/Scenes/VR_Player1.unity index 79df12b..941731e 100644 --- a/Unity-Master/Assets/Scenes/VR_Player1.unity +++ b/Unity-Master/Assets/Scenes/VR_Player1.unity @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c911d645c4a0bba6fd8b8c7300f5e58a1c276b26429924f982550bffcb05e2ef -size 4411853 +oid sha256:6c40431e096fa409f80eda6e08d7cb7ddbdb1366dea3b822c3ffa5871604ae7f +size 4413537 diff --git a/Unity-Master/Assets/Scenes/VR_Player2.unity b/Unity-Master/Assets/Scenes/VR_Player2.unity index ff75ebf..99141a9 100644 --- a/Unity-Master/Assets/Scenes/VR_Player2.unity +++ b/Unity-Master/Assets/Scenes/VR_Player2.unity @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5145690491a5812a2ebdef849e1aa792df5c21ccd8ea1db742c79423e467a322 -size 4491068 +oid sha256:cc3d371e91f4e972530263aa0b85f604acb79f640cdbdc90c40a922983ef1f1b +size 4492434 diff --git a/Unity-Master/Assets/Scripts/README_AvatarSync_Agent.md.meta b/Unity-Master/Assets/Scripts/README_AvatarSync_Agent.md.meta new file mode 100644 index 0000000..acfa965 --- /dev/null +++ b/Unity-Master/Assets/Scripts/README_AvatarSync_Agent.md.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: a39805992c022064eaa29d4fbc63b48e +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Unity-Master/Assets/Scripts/UDPAvatarBroadcasterAgent.cs b/Unity-Master/Assets/Scripts/UDPAvatarBroadcasterAgent.cs new file mode 100644 index 0000000..f1d86bc --- /dev/null +++ b/Unity-Master/Assets/Scripts/UDPAvatarBroadcasterAgent.cs @@ -0,0 +1,487 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Sockets; +using UnityEngine; + +/// +/// UDPAvatarBroadcasterAgent - Adapted for CC_Base_ bone naming convention (Ready Player Me / Reallusion Character Creator) +/// +public class UDPAvatarBroadcasterAgent : MonoBehaviour +{ + [Header("Network Configuration")] + [SerializeField] private int broadcastPort = 8080; + [SerializeField] private string broadcastAddress = "10.138.6.255"; // Local network broadcast + [SerializeField] private bool useGlobalNetworkConfig = true; + [SerializeField] private NetworkConfig networkConfigAsset; + [SerializeField] private bool enableBroadcast = true; + + [Header("Avatar Configuration")] + [SerializeField] private Transform avatarRoot; + [SerializeField] private byte playerID = 5; // Unique ID for this player (1-255) + [SerializeField] private float updateRate = 30f; // Updates per second + + [Header("Data Selection Mode")] + [SerializeField] private bool fullDataMode = false; // Send ALL bones and blend shapes + [SerializeField] private bool showFullModeWarning = true; + + [Header("Optimized Mode - Core Bones (CC_Base_ naming)")] + [SerializeField] private string[] priorityBones = { + "CC_Base_Hip", "CC_Base_Pelvis", "CC_Base_Spine01", "CC_Base_Spine02", "CC_Base_Neck", "CC_Base_Head", + "CC_Base_L_Clavicle", "CC_Base_L_Upperarm", "CC_Base_L_Forearm", "CC_Base_L_Hand", + "CC_Base_R_Clavicle", "CC_Base_R_Upperarm", "CC_Base_R_Forearm", "CC_Base_R_Hand", + "CC_Base_L_Thigh", "CC_Base_L_Calf", "CC_Base_L_Foot", + "CC_Base_R_Thigh", "CC_Base_R_Calf", "CC_Base_R_Foot" + }; + + [Header("Facial Animation")] + [SerializeField] private bool includeFacialData = true; + [SerializeField] private int maxBlendShapes = 10; // Most important blend shapes only (optimized mode) + + [Header("Debug")] + [SerializeField] private bool showDebugInfo = false; + [SerializeField] private bool logPacketSize = false; + + private UdpClient udpClient; + private IPEndPoint broadcastEndPoint; + private Dictionary boneCache; + private List allBones; // For full data mode + private List allMeshes; // For full blend shapes + private SkinnedMeshRenderer headMesh; + private float lastUpdateTime; + private uint sequenceNumber = 0; + private int currentBoneCount; + private int currentBlendShapeCount; + + // Avatar data structure - supports both optimized and full data modes + private struct CompactAvatarData + { + public byte playerID; // 1 byte + public uint sequenceNumber; // 4 bytes + public uint timestamp; // 4 bytes (Unity time as uint milliseconds) + public bool isFullDataMode; // 1 byte - indicates if this is full or optimized data + + // Root transform (40 bytes) + public Vector3 rootPosition; // 12 bytes + public Quaternion rootRotation; // 16 bytes + public Vector3 rootScale; // 12 bytes + + // Variable-length bone data + // Optimized: 20 bones * 28 bytes = 560 bytes + // Full mode: All bones * 28 bytes (can be 80+ bones = 2240+ bytes) + public BoneTransform[] bones; + + // Variable-length blend shape data + // Optimized: 10 floats = 40 bytes + // Full mode: All blend shapes = up to 200+ floats = 800+ bytes + public float[] blendShapes; + + // Total optimized: ~650 bytes, Full mode: potentially 3000+ bytes + } + + private struct BoneTransform + { + public Vector3 position; // 12 bytes + public Quaternion rotation; // 16 bytes + } + + void Start() + { + // Apply global config if enabled + if (useGlobalNetworkConfig) + { + var cfg = networkConfigAsset != null ? networkConfigAsset : NetworkConfig.Instance; + if (cfg != null) + { + broadcastAddress = cfg.ipAddress; + broadcastPort = cfg.port; + } + } + InitializeNetworking(); + CacheAvatarComponents(); + + if (showDebugInfo) + { + Debug.Log($"UDP Avatar Broadcaster Agent initialized - Player {playerID}"); + Debug.Log($"Broadcasting to {broadcastAddress}:{broadcastPort}"); + Debug.Log($"Priority bones: {priorityBones.Length}"); + } + } + + void InitializeNetworking() + { + try + { + udpClient = new UdpClient(); + udpClient.EnableBroadcast = true; + broadcastEndPoint = new IPEndPoint(IPAddress.Parse(broadcastAddress), broadcastPort); + + if (showDebugInfo) + Debug.Log("UDP client initialized successfully"); + } + catch (Exception e) + { + Debug.LogError($"Failed to initialize UDP client: {e.Message}"); + enableBroadcast = false; + } + } + + void CacheAvatarComponents() + { + if (avatarRoot == null) + avatarRoot = transform; + + boneCache = new Dictionary(); + allBones = new List(); + allMeshes = new List(); + + Transform[] allChildren = avatarRoot.GetComponentsInChildren(); + + if (fullDataMode) + { + // Full data mode: Cache ALL bones with SkinnedMeshRenderer references + SkinnedMeshRenderer[] meshRenderers = avatarRoot.GetComponentsInChildren(); + HashSet uniqueBones = new HashSet(); + + foreach (SkinnedMeshRenderer smr in meshRenderers) + { + if (smr.bones != null) + { + foreach (Transform bone in smr.bones) + { + if (bone != null) + { + uniqueBones.Add(bone); + } + } + } + } + + allBones.AddRange(uniqueBones); + currentBoneCount = allBones.Count; + + if (showDebugInfo) + Debug.Log($"Full data mode: Cached {currentBoneCount} bones"); + + // Show warning about packet size + if (showFullModeWarning) + { + int estimatedSize = CalculateFullModePacketSize(); + if (estimatedSize > 1400) // Close to UDP limit + { + Debug.LogWarning($"Full data mode packet size: ~{estimatedSize} bytes. Consider optimized mode for better network performance."); + } + } + } + else + { + // Optimized mode: Cache priority bones only + foreach (Transform child in allChildren) + { + foreach (string boneName in priorityBones) + { + if (child.name.Equals(boneName, StringComparison.OrdinalIgnoreCase)) + { + if (!boneCache.ContainsKey(boneName)) + { + boneCache[boneName] = child; + if (showDebugInfo) + Debug.Log($"Cached priority bone: {boneName}"); + } + break; + } + } + } + currentBoneCount = priorityBones.Length; + } + + // Cache facial animation data + if (includeFacialData) + { + SkinnedMeshRenderer[] meshes = avatarRoot.GetComponentsInChildren(); + + if (fullDataMode) + { + // Full mode: collect all meshes with blend shapes + foreach (var mesh in meshes) + { + if (mesh.sharedMesh != null && mesh.sharedMesh.blendShapeCount > 0) + { + allMeshes.Add(mesh); + } + } + + // Calculate total blend shape count + currentBlendShapeCount = 0; + foreach (var mesh in allMeshes) + { + currentBlendShapeCount += mesh.sharedMesh.blendShapeCount; + } + + if (showDebugInfo) + Debug.Log($"Full data mode: Found {allMeshes.Count} meshes with {currentBlendShapeCount} total blend shapes"); + } + else + { + // Optimized mode: find head mesh only + foreach (var mesh in meshes) + { + if (mesh.name.ToLower().Contains("head") || mesh.name.ToLower().Contains("face") || mesh.name.ToLower().Contains("wolf3d_head")) + { + headMesh = mesh; + if (showDebugInfo) + Debug.Log($"Found head mesh: {mesh.name} with {mesh.sharedMesh.blendShapeCount} blend shapes"); + break; + } + } + currentBlendShapeCount = maxBlendShapes; + } + } + + string modeStr = fullDataMode ? "FULL" : "OPTIMIZED"; + Debug.Log($"{modeStr} mode (Agent): {currentBoneCount} bones, {currentBlendShapeCount} blend shapes"); + } + + int CalculateFullModePacketSize() + { + int headerSize = 1 + 4 + 4 + 1; // playerID + sequence + timestamp + isFullDataMode + int rootTransformSize = (3 + 4 + 3) * 4; // position + rotation + scale + int bonesSize = 4 + (currentBoneCount * 7 * 4); // bone count + bones * (pos + rot) * float size + int blendShapesSize = 4 + (currentBlendShapeCount * 4); // blend shape count + weights * float size + return headerSize + rootTransformSize + bonesSize + blendShapesSize; + } + + void Update() + { + if (!enableBroadcast || udpClient == null) + return; + + // Rate-limited updates + if (Time.time - lastUpdateTime >= 1f / updateRate) + { + BroadcastAvatarData(); + lastUpdateTime = Time.time; + } + } + + void BroadcastAvatarData() + { + try + { + CompactAvatarData data = new CompactAvatarData + { + playerID = playerID, + sequenceNumber = sequenceNumber++, + timestamp = (uint)(Time.time * 1000), // Convert to milliseconds + isFullDataMode = fullDataMode, + + rootPosition = avatarRoot.position, + rootRotation = avatarRoot.rotation, + rootScale = avatarRoot.localScale, + + bones = new BoneTransform[currentBoneCount], + blendShapes = new float[currentBlendShapeCount] + }; + + // Pack bone data based on mode + if (fullDataMode) + { + // Full mode: pack all cached bones + for (int i = 0; i < allBones.Count; i++) + { + Transform bone = allBones[i]; + data.bones[i] = new BoneTransform + { + position = bone.localPosition, + rotation = bone.localRotation + }; + } + } + else + { + // Optimized mode: pack priority bones + for (int i = 0; i < priorityBones.Length; i++) + { + if (boneCache.TryGetValue(priorityBones[i], out Transform bone)) + { + data.bones[i] = new BoneTransform + { + position = bone.localPosition, + rotation = bone.localRotation + }; + } + else + { + // Default values for missing bones + data.bones[i] = new BoneTransform + { + position = Vector3.zero, + rotation = Quaternion.identity + }; + } + } + } + + // Pack facial blend shape data based on mode + if (includeFacialData) + { + if (fullDataMode) + { + // Full mode: collect all blend shapes from all meshes + int blendShapeIndex = 0; + foreach (var mesh in allMeshes) + { + if (mesh != null && mesh.sharedMesh != null) + { + for (int i = 0; i < mesh.sharedMesh.blendShapeCount; i++) + { + if (blendShapeIndex < data.blendShapes.Length) + { + data.blendShapes[blendShapeIndex] = mesh.GetBlendShapeWeight(i) / 100f; // Normalize to 0-1 + blendShapeIndex++; + } + } + } + } + } + else + { + // Optimized mode: use head mesh only + if (headMesh != null && headMesh.sharedMesh != null) + { + int blendShapeCount = Mathf.Min(maxBlendShapes, headMesh.sharedMesh.blendShapeCount); + for (int i = 0; i < blendShapeCount; i++) + { + data.blendShapes[i] = headMesh.GetBlendShapeWeight(i) / 100f; // Normalize to 0-1 + } + } + } + } + + // Serialize to bytes + byte[] packetData = SerializeCompactData(data); + + // Send UDP packet + udpClient.Send(packetData, packetData.Length, broadcastEndPoint); + + if (logPacketSize && sequenceNumber % 30 == 0) // Log every 30 packets (about once per second) + { + string modeStr = fullDataMode ? "FULL" : "OPT"; + Debug.Log($"Sent {modeStr} packet #{sequenceNumber}, size: {packetData.Length} bytes, bones: {currentBoneCount}, blend shapes: {currentBlendShapeCount}"); + } + } + catch (Exception e) + { + Debug.LogError($"Failed to broadcast avatar data: {e.Message}"); + } + } + + byte[] SerializeCompactData(CompactAvatarData data) + { + using (MemoryStream stream = new MemoryStream()) + using (BinaryWriter writer = new BinaryWriter(stream)) + { + // Header + writer.Write(data.playerID); + writer.Write(data.sequenceNumber); + writer.Write(data.timestamp); + writer.Write(data.isFullDataMode); + + // Root transform + WriteVector3(writer, data.rootPosition); + WriteQuaternion(writer, data.rootRotation); + WriteVector3(writer, data.rootScale); + + // Variable-length bones + writer.Write(data.bones.Length); // Write bone count first + for (int i = 0; i < data.bones.Length; i++) + { + WriteVector3(writer, data.bones[i].position); + WriteQuaternion(writer, data.bones[i].rotation); + } + + // Variable-length blend shapes + writer.Write(data.blendShapes.Length); // Write blend shape count first + for (int i = 0; i < data.blendShapes.Length; i++) + { + writer.Write(data.blendShapes[i]); + } + + return stream.ToArray(); + } + } + + void WriteVector3(BinaryWriter writer, Vector3 vector) + { + writer.Write(vector.x); + writer.Write(vector.y); + writer.Write(vector.z); + } + + void WriteQuaternion(BinaryWriter writer, Quaternion quaternion) + { + writer.Write(quaternion.x); + writer.Write(quaternion.y); + writer.Write(quaternion.z); + writer.Write(quaternion.w); + } + + public void SetPlayerID(byte newPlayerID) + { + playerID = newPlayerID; + } + + public void SetUpdateRate(float newRate) + { + updateRate = Mathf.Clamp(newRate, 1f, 120f); + } + + void OnDestroy() + { + if (udpClient != null) + { + udpClient.Close(); + udpClient.Dispose(); + } + } + + void OnApplicationPause(bool pauseStatus) + { + enableBroadcast = !pauseStatus; + } + + void OnGUI() + { + if (!showDebugInfo) return; + + GUILayout.BeginArea(new Rect(10, 10, 300, 240)); + GUILayout.Label($"UDP Broadcaster Agent - Player {playerID}"); + GUILayout.Label($"Mode: {(fullDataMode ? "FULL DATA" : "OPTIMIZED")}"); + GUILayout.Label($"Status: {(enableBroadcast ? "Broadcasting" : "Stopped")}"); + GUILayout.Label($"Sequence: {sequenceNumber}"); + GUILayout.Label($"Update Rate: {updateRate:F1} Hz"); + GUILayout.Label($"Bones: {currentBoneCount}"); + GUILayout.Label($"Blend Shapes: {currentBlendShapeCount}"); + + if (fullDataMode) + { + int estimatedSize = CalculateFullModePacketSize(); + GUILayout.Label($"Est. Packet Size: {estimatedSize} bytes"); + if (estimatedSize > 1400) + { + GUI.color = Color.yellow; + GUILayout.Label("⚠️ Large packet size!"); + GUI.color = Color.white; + } + } + + if (GUILayout.Button(enableBroadcast ? "Stop Broadcasting" : "Start Broadcasting")) + { + enableBroadcast = !enableBroadcast; + } + + GUILayout.EndArea(); + } +} + diff --git a/Unity-Master/Assets/Scripts/UDPAvatarBroadcasterAgent.cs.meta b/Unity-Master/Assets/Scripts/UDPAvatarBroadcasterAgent.cs.meta new file mode 100644 index 0000000..eb22e4a --- /dev/null +++ b/Unity-Master/Assets/Scripts/UDPAvatarBroadcasterAgent.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 68f69be5ef1c06c4ea9fcc9d9f59f1d6 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Unity-Master/Assets/Scripts/UDPAvatarReceiverAgent.cs b/Unity-Master/Assets/Scripts/UDPAvatarReceiverAgent.cs new file mode 100644 index 0000000..0bf8d6c --- /dev/null +++ b/Unity-Master/Assets/Scripts/UDPAvatarReceiverAgent.cs @@ -0,0 +1,726 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Threading; +using UnityEngine; + +/// +/// UDPAvatarReceiverAgent - Adapted for CC_Base_ bone naming convention (Ready Player Me / Reallusion Character Creator) +/// +public class UDPAvatarReceiverAgent : MonoBehaviour +{ + [Header("Network Configuration")] + [SerializeField] private int listenPort = 8080; + [SerializeField] private bool useGlobalNetworkConfig = true; + [SerializeField] private NetworkConfig networkConfigAsset; + [SerializeField] private bool enableReceiver = true; + [SerializeField] private byte targetPlayerID = 5; // Which player to receive data from (0 = any) + [SerializeField] private bool allowPortSharing = true; // For local testing with multiple components + + [Header("Avatar Configuration")] + [SerializeField] private Transform targetAvatarRoot; + [SerializeField] private bool smoothTransitions = true; + [SerializeField] private float transitionSpeed = 10f; + + [Header("Data Selection - Core Bones (CC_Base_ naming)")] + [SerializeField] private string[] priorityBones = { + "CC_Base_Hip", "CC_Base_Pelvis", "CC_Base_Spine01", "CC_Base_Spine02", "CC_Base_Neck", "CC_Base_Head", + "CC_Base_L_Clavicle", "CC_Base_L_Upperarm", "CC_Base_L_Forearm", "CC_Base_L_Hand", + "CC_Base_R_Clavicle", "CC_Base_R_Upperarm", "CC_Base_R_Forearm", "CC_Base_R_Hand", + "CC_Base_L_Thigh", "CC_Base_L_Calf", "CC_Base_L_Foot", + "CC_Base_R_Thigh", "CC_Base_R_Calf", "CC_Base_R_Foot" + }; + + [Header("Synchronization Options")] + [SerializeField] private bool syncRootPosition = true; + [SerializeField] private bool syncRootRotation = true; + [SerializeField] private bool syncRootScale = false; + [SerializeField] private bool syncFacialData = true; + + [Header("Debug")] + [SerializeField] private bool showDebugInfo = false; + [SerializeField] private bool logReceivedPackets = false; + + private UdpClient udpClient; + private Thread udpListenerThread; + private bool threadRunning = false; + private Dictionary boneCache; + private List allBones; // For full data mode + private List allMeshes; // For full blend shapes + private SkinnedMeshRenderer headMesh; + + // Latest received data + private CompactAvatarData lastReceivedData; + private bool hasNewData = false; + private object dataLock = new object(); + + // Stats + private uint lastSequenceNumber = 0; + private int packetsReceived = 0; + private int packetsDropped = 0; + private float lastPacketTime = 0f; + + // Avatar data structure - supports both optimized and full data modes (matches broadcaster) + private struct CompactAvatarData + { + public byte playerID; + public uint sequenceNumber; + public uint timestamp; + public bool isFullDataMode; // indicates if this is full or optimized data + + public Vector3 rootPosition; + public Quaternion rootRotation; + public Vector3 rootScale; + + public BoneTransform[] bones; // Variable-length + public float[] blendShapes; // Variable-length + } + + private struct BoneTransform + { + public Vector3 position; + public Quaternion rotation; + } + + void Start() + { + if (targetAvatarRoot == null) + targetAvatarRoot = transform; + + // Apply global config if enabled + if (useGlobalNetworkConfig) + { + var cfg = networkConfigAsset != null ? networkConfigAsset : NetworkConfig.Instance; + if (cfg != null) + { + listenPort = cfg.port; + } + } + + CacheAvatarComponents(); + + if (enableReceiver) + { + StartUDPListener(); + } + + if (showDebugInfo) + { + Debug.Log($"UDP Avatar Receiver Agent initialized"); + Debug.Log($"Listening on port {listenPort}"); + Debug.Log($"Target Player ID: {(targetPlayerID == 0 ? "Any" : targetPlayerID.ToString())}"); + } + } + + void CacheAvatarComponents() + { + boneCache = new Dictionary(); + allBones = new List(); + allMeshes = new List(); + + Transform[] allChildren = targetAvatarRoot.GetComponentsInChildren(); + + // Cache priority bones for optimized mode + foreach (Transform child in allChildren) + { + foreach (string boneName in priorityBones) + { + if (child.name.Equals(boneName, StringComparison.OrdinalIgnoreCase)) + { + if (!boneCache.ContainsKey(boneName)) + { + boneCache[boneName] = child; + if (showDebugInfo) + Debug.Log($"Cached priority bone: {boneName}"); + } + break; + } + } + } + + // Cache ALL bones for full data mode + SkinnedMeshRenderer[] meshRenderers = targetAvatarRoot.GetComponentsInChildren(); + HashSet uniqueBones = new HashSet(); + + foreach (SkinnedMeshRenderer smr in meshRenderers) + { + if (smr.bones != null) + { + foreach (Transform bone in smr.bones) + { + if (bone != null) + { + uniqueBones.Add(bone); + } + } + } + } + + allBones.AddRange(uniqueBones); + if (showDebugInfo) + Debug.Log($"Cached {allBones.Count} total bones for full data mode"); + + // Cache facial animation data + if (syncFacialData) + { + SkinnedMeshRenderer[] meshes = targetAvatarRoot.GetComponentsInChildren(); + + // Find head mesh for optimized mode + foreach (var mesh in meshes) + { + if (mesh.name.ToLower().Contains("head") || mesh.name.ToLower().Contains("face") || mesh.name.ToLower().Contains("wolf3d_head")) + { + headMesh = mesh; + if (showDebugInfo) + Debug.Log($"Found head mesh: {mesh.name} with {mesh.sharedMesh.blendShapeCount} blend shapes"); + break; + } + } + + // Cache all meshes with blend shapes for full data mode + foreach (var mesh in meshes) + { + if (mesh.sharedMesh != null && mesh.sharedMesh.blendShapeCount > 0) + { + allMeshes.Add(mesh); + } + } + + if (showDebugInfo) + { + int totalBlendShapes = 0; + foreach (var mesh in allMeshes) + totalBlendShapes += mesh.sharedMesh.blendShapeCount; + Debug.Log($"Found {allMeshes.Count} meshes with {totalBlendShapes} total blend shapes for full data mode"); + } + } + + Debug.Log($"Receiver Agent ready: {boneCache.Count}/{priorityBones.Length} priority bones, {allBones.Count} total bones"); + } + + void StartUDPListener() + { + try + { + if (allowPortSharing) + { + // Create UDP client with port reuse for local testing + udpClient = new UdpClient(); + udpClient.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); + udpClient.Client.Bind(new IPEndPoint(IPAddress.Any, listenPort)); + } + else + { + // Standard UDP client binding + udpClient = new UdpClient(listenPort); + } + + threadRunning = true; + udpListenerThread = new Thread(new ThreadStart(UDPListenerThread)); + udpListenerThread.IsBackground = true; + udpListenerThread.Start(); + + if (showDebugInfo) + Debug.Log($"UDP Avatar Receiver Agent started on port {listenPort} (filtering binary avatar data only, Port sharing: {allowPortSharing})"); + } + catch (Exception e) + { + if (allowPortSharing) + { + Debug.LogWarning($"Failed to start UDP listener with port sharing: {e.Message}"); + Debug.LogWarning("Trying with different port..."); + TryAlternativePort(); + } + else + { + Debug.LogError($"Failed to start UDP listener: {e.Message}"); + enableReceiver = false; + } + } + } + + void TryAlternativePort() + { + // Try a few alternative ports for local testing + int[] alternativePorts = { 8081, 8082, 8083, 8084, 8085 }; + + foreach (int port in alternativePorts) + { + try + { + udpClient = new UdpClient(port); + threadRunning = true; + udpListenerThread = new Thread(new ThreadStart(UDPListenerThread)); + udpListenerThread.IsBackground = true; + udpListenerThread.Start(); + + Debug.Log($"UDP listener started on alternative port {port}"); + return; + } + catch (Exception) + { + // Try next port + continue; + } + } + + Debug.LogError("Failed to start UDP listener on any available port"); + enableReceiver = false; + } + + void UDPListenerThread() + { + IPEndPoint remoteEndPoint = new IPEndPoint(IPAddress.Any, 0); + + while (threadRunning) + { + try + { + byte[] data = udpClient.Receive(ref remoteEndPoint); + + if (data.Length > 0 && IsAvatarData(data)) + { + CompactAvatarData avatarData = DeserializeCompactData(data); + + // Check if this is from the target player (0 means accept from any player) + if (targetPlayerID == 0 || avatarData.playerID == targetPlayerID) + { + // Check for packet loss + if (avatarData.sequenceNumber > lastSequenceNumber + 1) + { + packetsDropped += (int)(avatarData.sequenceNumber - lastSequenceNumber - 1); + } + + lastSequenceNumber = avatarData.sequenceNumber; + packetsReceived++; + + // Store the new data (thread-safe) + lock (dataLock) + { + lastReceivedData = avatarData; + hasNewData = true; + } + + if (logReceivedPackets && packetsReceived % 30 == 0) + { + string modeStr = avatarData.isFullDataMode ? "FULL" : "OPT"; + Debug.Log($"Received {modeStr} packet #{avatarData.sequenceNumber} from player {avatarData.playerID}, size: {data.Length} bytes, bones: {avatarData.bones?.Length ?? 0}, blend shapes: {avatarData.blendShapes?.Length ?? 0}"); + } + } + } + } + catch (Exception e) + { + if (threadRunning) // Only log errors if we're supposed to be running + { + Debug.LogError($"UDP receive error: {e.Message}"); + } + } + } + } + + bool IsAvatarData(byte[] data) + { + try + { + // Avatar data should be binary and typically larger than JSON experiment messages + if (data.Length < 20) // Avatar data should have at least header info + return false; + + // Check if it looks like text/JSON (experiment control message) + // JSON messages will be valid UTF-8 text starting with '{' + try + { + string text = Encoding.UTF8.GetString(data); + if (text.TrimStart().StartsWith("{") && text.Contains("\"command\"")) + { + // This is likely an experiment control message, not avatar data + return false; + } + } + catch + { + // If it fails to decode as UTF-8, it's likely binary avatar data + } + + // Additional check: avatar data should start with a reasonable playerID (0-255) + // and have a structure that makes sense + using (MemoryStream stream = new MemoryStream(data)) + using (BinaryReader reader = new BinaryReader(stream)) + { + // Basic structure check - should be able to read at least the header + if (data.Length >= 17) // byte + uint32 + uint32 + bool + 12 bytes for Vector3 + { + byte playerID = reader.ReadByte(); + uint sequenceNumber = reader.ReadUInt32(); + uint timestamp = reader.ReadUInt32(); + bool isFullDataMode = reader.ReadBoolean(); + + // Basic sanity checks + if (playerID <= 10 && sequenceNumber < uint.MaxValue / 2) // Reasonable values + { + return true; + } + } + } + + return false; + } + catch + { + // If any parsing fails, assume it's not valid avatar data + return false; + } + } + + CompactAvatarData DeserializeCompactData(byte[] data) + { + using (MemoryStream stream = new MemoryStream(data)) + using (BinaryReader reader = new BinaryReader(stream)) + { + CompactAvatarData result = new CompactAvatarData + { + // Header + playerID = reader.ReadByte(), + sequenceNumber = reader.ReadUInt32(), + timestamp = reader.ReadUInt32(), + isFullDataMode = reader.ReadBoolean(), + + // Root transform + rootPosition = ReadVector3(reader), + rootRotation = ReadQuaternion(reader), + rootScale = ReadVector3(reader) + }; + + // Variable-length bones + int boneCount = reader.ReadInt32(); + result.bones = new BoneTransform[boneCount]; + for (int i = 0; i < boneCount; i++) + { + result.bones[i] = new BoneTransform + { + position = ReadVector3(reader), + rotation = ReadQuaternion(reader) + }; + } + + // Variable-length blend shapes + int blendShapeCount = reader.ReadInt32(); + result.blendShapes = new float[blendShapeCount]; + for (int i = 0; i < blendShapeCount; i++) + { + result.blendShapes[i] = reader.ReadSingle(); + } + + return result; + } + } + + Vector3 ReadVector3(BinaryReader reader) + { + return new Vector3( + reader.ReadSingle(), + reader.ReadSingle(), + reader.ReadSingle() + ); + } + + Quaternion ReadQuaternion(BinaryReader reader) + { + return new Quaternion( + reader.ReadSingle(), + reader.ReadSingle(), + reader.ReadSingle(), + reader.ReadSingle() + ); + } + + void Update() + { + if (!enableReceiver) + return; + + // Apply new data if available + lock (dataLock) + { + if (hasNewData) + { + ApplyAvatarData(lastReceivedData); + hasNewData = false; + lastPacketTime = Time.time; + } + } + } + + void ApplyAvatarData(CompactAvatarData data) + { + // Apply root transform + if (syncRootPosition) + { + if (smoothTransitions) + { + targetAvatarRoot.position = Vector3.Lerp( + targetAvatarRoot.position, + data.rootPosition, + Time.deltaTime * transitionSpeed + ); + } + else + { + targetAvatarRoot.position = data.rootPosition; + } + } + + if (syncRootRotation) + { + if (smoothTransitions) + { + targetAvatarRoot.rotation = Quaternion.Lerp( + targetAvatarRoot.rotation, + data.rootRotation, + Time.deltaTime * transitionSpeed + ); + } + else + { + targetAvatarRoot.rotation = data.rootRotation; + } + } + + if (syncRootScale) + { + if (smoothTransitions) + { + targetAvatarRoot.localScale = Vector3.Lerp( + targetAvatarRoot.localScale, + data.rootScale, + Time.deltaTime * transitionSpeed + ); + } + else + { + targetAvatarRoot.localScale = data.rootScale; + } + } + + // Apply bone transforms based on data mode + if (data.isFullDataMode) + { + // Full data mode: apply to all bones based on order + for (int i = 0; i < Mathf.Min(data.bones.Length, allBones.Count); i++) + { + Transform bone = allBones[i]; + if (bone != null) + { + if (smoothTransitions) + { + bone.localPosition = Vector3.Lerp( + bone.localPosition, + data.bones[i].position, + Time.deltaTime * transitionSpeed + ); + + bone.localRotation = Quaternion.Lerp( + bone.localRotation, + data.bones[i].rotation, + Time.deltaTime * transitionSpeed + ); + } + else + { + bone.localPosition = data.bones[i].position; + bone.localRotation = data.bones[i].rotation; + } + } + } + } + else + { + // Optimized mode: apply to priority bones + for (int i = 0; i < Mathf.Min(data.bones.Length, priorityBones.Length); i++) + { + if (boneCache.TryGetValue(priorityBones[i], out Transform bone)) + { + if (smoothTransitions) + { + bone.localPosition = Vector3.Lerp( + bone.localPosition, + data.bones[i].position, + Time.deltaTime * transitionSpeed + ); + + bone.localRotation = Quaternion.Lerp( + bone.localRotation, + data.bones[i].rotation, + Time.deltaTime * transitionSpeed + ); + } + else + { + bone.localPosition = data.bones[i].position; + bone.localRotation = data.bones[i].rotation; + } + } + } + } + + // Apply facial blend shapes based on data mode + if (syncFacialData) + { + if (data.isFullDataMode) + { + // Full data mode: apply all blend shapes to all meshes + int blendShapeIndex = 0; + foreach (var mesh in allMeshes) + { + if (mesh != null && mesh.sharedMesh != null) + { + for (int i = 0; i < mesh.sharedMesh.blendShapeCount; i++) + { + if (blendShapeIndex < data.blendShapes.Length) + { + float targetWeight = data.blendShapes[blendShapeIndex] * 100f; // Convert back to 0-100 range + + if (smoothTransitions) + { + float currentWeight = mesh.GetBlendShapeWeight(i); + float newWeight = Mathf.Lerp(currentWeight, targetWeight, Time.deltaTime * transitionSpeed); + mesh.SetBlendShapeWeight(i, newWeight); + } + else + { + mesh.SetBlendShapeWeight(i, targetWeight); + } + blendShapeIndex++; + } + } + } + } + } + else + { + // Optimized mode: apply to head mesh only + if (headMesh != null && headMesh.sharedMesh != null) + { + int blendShapeCount = Mathf.Min(data.blendShapes.Length, headMesh.sharedMesh.blendShapeCount); + for (int i = 0; i < blendShapeCount; i++) + { + float targetWeight = data.blendShapes[i] * 100f; // Convert back to 0-100 range + + if (smoothTransitions) + { + float currentWeight = headMesh.GetBlendShapeWeight(i); + float newWeight = Mathf.Lerp(currentWeight, targetWeight, Time.deltaTime * transitionSpeed); + headMesh.SetBlendShapeWeight(i, newWeight); + } + else + { + headMesh.SetBlendShapeWeight(i, targetWeight); + } + } + } + } + } + } + + public void SetTargetPlayerID(byte playerID) + { + targetPlayerID = playerID; + } + + public bool IsReceivingData() + { + return Time.time - lastPacketTime < 1f; // Consider "receiving" if we got data in the last second + } + + public float GetPacketLossRate() + { + int totalExpected = packetsReceived + packetsDropped; + return totalExpected > 0 ? (float)packetsDropped / totalExpected : 0f; + } + + void StopUDPListener() + { + threadRunning = false; + + if (udpClient != null) + { + udpClient.Close(); + udpClient.Dispose(); + udpClient = null; + } + + if (udpListenerThread != null) + { + udpListenerThread.Join(1000); // Wait up to 1 second for thread to finish + udpListenerThread = null; + } + } + + void OnDestroy() + { + StopUDPListener(); + } + + void OnApplicationPause(bool pauseStatus) + { + if (pauseStatus) + { + StopUDPListener(); + } + else if (enableReceiver) + { + StartUDPListener(); + } + } + + void OnGUI() + { + if (!showDebugInfo) return; + + GUILayout.BeginArea(new Rect(320, 10, 300, 260)); + GUILayout.Label($"UDP Avatar Receiver Agent (Binary Only)"); + GUILayout.Label($"Status: {(IsReceivingData() ? "Receiving" : "No Data")}"); + + // Show last received data mode + if (hasNewData) + { + string modeStr = lastReceivedData.isFullDataMode ? "FULL DATA" : "OPTIMIZED"; + GUILayout.Label($"Last Data Mode: {modeStr}"); + GUILayout.Label($"Bones Received: {lastReceivedData.bones?.Length ?? 0}"); + GUILayout.Label($"Blend Shapes: {lastReceivedData.blendShapes?.Length ?? 0}"); + } + + GUILayout.Label($"Listen Port: {GetActualListenPort()} (shared)"); + GUILayout.Label($"Target Player: {(targetPlayerID == 0 ? "Any" : targetPlayerID.ToString())}"); + GUILayout.Label($"Packets: {packetsReceived} received, {packetsDropped} dropped"); + GUILayout.Label($"Packet Loss: {GetPacketLossRate():P1}"); + GUILayout.Label($"Last Sequence: {lastSequenceNumber}"); + GUILayout.Label($"Cached: {boneCache.Count} priority, {allBones.Count} total bones"); + GUILayout.Label($"Port Sharing: {allowPortSharing}"); + + if (GUILayout.Button(enableReceiver ? "Stop Receiver" : "Start Receiver")) + { + enableReceiver = !enableReceiver; + if (enableReceiver) + StartUDPListener(); + else + StopUDPListener(); + } + + GUILayout.EndArea(); + } + + int GetActualListenPort() + { + if (udpClient?.Client?.LocalEndPoint != null) + { + return ((IPEndPoint)udpClient.Client.LocalEndPoint).Port; + } + return listenPort; + } +} + diff --git a/Unity-Master/Assets/Scripts/UDPAvatarReceiverAgent.cs.meta b/Unity-Master/Assets/Scripts/UDPAvatarReceiverAgent.cs.meta new file mode 100644 index 0000000..50f3699 --- /dev/null +++ b/Unity-Master/Assets/Scripts/UDPAvatarReceiverAgent.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 5f33b7a715d62bb489bac0262f29f846 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: