626 lines
21 KiB
C#
626 lines
21 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Net;
|
|
using UnityEngine;
|
|
|
|
public class UDPAvatarReceiver : MonoBehaviour
|
|
{
|
|
[Header("Network Configuration")]
|
|
[SerializeField] private bool enableReceiver = true;
|
|
[SerializeField] private byte targetPlayerID = 2; // Which player to receive data from (0 = any)
|
|
[SerializeField] private bool allowPortSharing = true; // For local testing with multiple components
|
|
|
|
[Header("Avatar Configuration")]
|
|
[SerializeField] private Transform targetAvatarRoot;
|
|
[SerializeField] private bool smoothTransitions = true;
|
|
[SerializeField] private float transitionSpeed = 10f;
|
|
|
|
[Header("Data Selection - Core Bones (Optimized Mode)")]
|
|
[SerializeField] private string[] priorityBones = {
|
|
"Hips", "Spine", "Spine1", "Spine2", "Neck", "Head",
|
|
"LeftShoulder", "LeftArm", "LeftForeArm", "LeftHand",
|
|
"RightShoulder", "RightArm", "RightForeArm", "RightHand",
|
|
"LeftUpLeg", "LeftLeg", "LeftFoot",
|
|
"RightUpLeg", "RightLeg", "RightFoot"
|
|
};
|
|
|
|
[Header("Synchronization Options")]
|
|
[SerializeField] private bool syncRootPosition = true;
|
|
[SerializeField] private bool syncRootRotation = true;
|
|
[SerializeField] private bool syncRootScale = false;
|
|
[SerializeField] private bool syncFacialData = true;
|
|
|
|
[Header("Debug")]
|
|
[SerializeField] private bool showDebugInfo = false;
|
|
[SerializeField] private bool logReceivedPackets = false;
|
|
|
|
private bool threadRunning = false;
|
|
private int listenPort;
|
|
private Dictionary<string, Transform> boneCache;
|
|
private List<Transform> allBones; // For full data mode
|
|
private List<SkinnedMeshRenderer> allMeshes; // For full blend shapes
|
|
private SkinnedMeshRenderer headMesh;
|
|
|
|
// Latest received data
|
|
private CompactAvatarData lastReceivedData;
|
|
private bool hasNewData = false;
|
|
private object dataLock = new object();
|
|
|
|
// Stats
|
|
private uint lastSequenceNumber = 0;
|
|
private int packetsReceived = 0;
|
|
private int packetsDropped = 0;
|
|
private float lastPacketTime = 0f;
|
|
|
|
// Magic number for packet identification
|
|
private const uint AVATAR_MAGIC = 0xC0A0;
|
|
|
|
// Avatar data structure - supports both optimized and full data modes (matches broadcaster)
|
|
private struct CompactAvatarData
|
|
{
|
|
public byte playerID;
|
|
public uint sequenceNumber;
|
|
public uint timestamp;
|
|
public bool isFullDataMode; // indicates if this is full or optimized data
|
|
|
|
public Vector3 rootPosition;
|
|
public Quaternion rootRotation;
|
|
public Vector3 rootScale;
|
|
|
|
public BoneTransform[] bones; // Variable-length
|
|
public float[] blendShapes; // Variable-length
|
|
}
|
|
|
|
private struct BoneTransform
|
|
{
|
|
public Vector3 position;
|
|
public Quaternion rotation;
|
|
}
|
|
|
|
void Start()
|
|
{
|
|
if (targetAvatarRoot == null)
|
|
targetAvatarRoot = transform;
|
|
|
|
// Get network config from global instance
|
|
var cfg = NetworkConfig.Instance;
|
|
if (cfg != null)
|
|
{
|
|
listenPort = cfg.port;
|
|
}
|
|
else
|
|
{
|
|
Debug.LogError("NetworkConfig not found! Please ensure NetworkConfig.asset exists in Resources folder.");
|
|
listenPort = 1221;
|
|
}
|
|
|
|
CacheAvatarComponents();
|
|
|
|
if (enableReceiver)
|
|
{
|
|
StartUDPListener();
|
|
}
|
|
|
|
if (showDebugInfo)
|
|
{
|
|
Debug.Log($"UDP Avatar Receiver initialized");
|
|
Debug.Log($"Listening on port {listenPort}");
|
|
Debug.Log($"Target Player ID: {(targetPlayerID == 0 ? "Any" : targetPlayerID.ToString())}");
|
|
}
|
|
}
|
|
|
|
void CacheAvatarComponents()
|
|
{
|
|
boneCache = new Dictionary<string, Transform>();
|
|
allBones = new List<Transform>();
|
|
allMeshes = new List<SkinnedMeshRenderer>();
|
|
|
|
Transform[] allChildren = targetAvatarRoot.GetComponentsInChildren<Transform>();
|
|
|
|
// Cache priority bones for optimized mode
|
|
foreach (Transform child in allChildren)
|
|
{
|
|
foreach (string boneName in priorityBones)
|
|
{
|
|
if (child.name.Equals(boneName, StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
if (!boneCache.ContainsKey(boneName))
|
|
{
|
|
boneCache[boneName] = child;
|
|
if (showDebugInfo)
|
|
Debug.Log($"Cached priority bone: {boneName}");
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Cache ALL bones for full data mode
|
|
SkinnedMeshRenderer[] meshRenderers = targetAvatarRoot.GetComponentsInChildren<SkinnedMeshRenderer>();
|
|
HashSet<Transform> uniqueBones = new HashSet<Transform>();
|
|
|
|
foreach (SkinnedMeshRenderer smr in meshRenderers)
|
|
{
|
|
if (smr.bones != null)
|
|
{
|
|
foreach (Transform bone in smr.bones)
|
|
{
|
|
if (bone != null)
|
|
{
|
|
uniqueBones.Add(bone);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
allBones.AddRange(uniqueBones);
|
|
if (showDebugInfo)
|
|
Debug.Log($"Cached {allBones.Count} total bones for full data mode");
|
|
|
|
// Cache facial animation data
|
|
if (syncFacialData)
|
|
{
|
|
SkinnedMeshRenderer[] meshes = targetAvatarRoot.GetComponentsInChildren<SkinnedMeshRenderer>();
|
|
|
|
// Find head mesh for optimized mode
|
|
foreach (var mesh in meshes)
|
|
{
|
|
if (mesh.name.ToLower().Contains("head") || mesh.name.ToLower().Contains("face"))
|
|
{
|
|
headMesh = mesh;
|
|
if (showDebugInfo)
|
|
Debug.Log($"Found head mesh: {mesh.name} with {mesh.sharedMesh.blendShapeCount} blend shapes");
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Cache all meshes with blend shapes for full data mode
|
|
foreach (var mesh in meshes)
|
|
{
|
|
if (mesh.sharedMesh != null && mesh.sharedMesh.blendShapeCount > 0)
|
|
{
|
|
allMeshes.Add(mesh);
|
|
}
|
|
}
|
|
|
|
if (showDebugInfo)
|
|
{
|
|
int totalBlendShapes = 0;
|
|
foreach (var mesh in allMeshes)
|
|
totalBlendShapes += mesh.sharedMesh.blendShapeCount;
|
|
Debug.Log($"Found {allMeshes.Count} meshes with {totalBlendShapes} total blend shapes for full data mode");
|
|
}
|
|
}
|
|
|
|
Debug.Log($"Receiver ready: {boneCache.Count}/{priorityBones.Length} priority bones, {allBones.Count} total bones");
|
|
}
|
|
|
|
void StartUDPListener()
|
|
{
|
|
try
|
|
{
|
|
// Wait for shared listener to be ready
|
|
if (Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance == null)
|
|
{
|
|
Debug.LogError("SharedUDPListener not found! Make sure it's in the scene.");
|
|
enableReceiver = false;
|
|
return;
|
|
}
|
|
|
|
// Subscribe to shared listener
|
|
Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance.OnPacketReceived += HandlePacketReceived;
|
|
threadRunning = true;
|
|
|
|
if (showDebugInfo)
|
|
Debug.Log($"UDP Avatar Receiver subscribed to shared listener on port {listenPort} (filtering avatar magic 0x{AVATAR_MAGIC:X})");
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
Debug.LogError($"Failed to subscribe to shared UDP listener: {e.Message}");
|
|
enableReceiver = false;
|
|
}
|
|
}
|
|
|
|
void HandlePacketReceived(byte[] data, IPEndPoint senderEndPoint)
|
|
{
|
|
// Check if this is avatar data (by magic number)
|
|
if (!IsAvatarData(data)) return;
|
|
|
|
// Process avatar packet
|
|
CompactAvatarData avatarData = DeserializeCompactData(data);
|
|
|
|
// Check if this is from the target player (0 means accept from any player)
|
|
if (targetPlayerID == 0 || avatarData.playerID == targetPlayerID)
|
|
{
|
|
// Check for packet loss
|
|
if (avatarData.sequenceNumber > lastSequenceNumber + 1)
|
|
{
|
|
packetsDropped += (int)(avatarData.sequenceNumber - lastSequenceNumber - 1);
|
|
}
|
|
|
|
lastSequenceNumber = avatarData.sequenceNumber;
|
|
packetsReceived++;
|
|
|
|
// Store the new data (thread-safe)
|
|
lock (dataLock)
|
|
{
|
|
lastReceivedData = avatarData;
|
|
hasNewData = true;
|
|
}
|
|
|
|
if (logReceivedPackets && packetsReceived % 30 == 0)
|
|
{
|
|
string modeStr = avatarData.isFullDataMode ? "FULL" : "OPT";
|
|
Debug.Log($"Received {modeStr} packet #{avatarData.sequenceNumber} from player {avatarData.playerID}, size: {data.Length} bytes, bones: {avatarData.bones?.Length ?? 0}, blend shapes: {avatarData.blendShapes?.Length ?? 0}");
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
bool IsAvatarData(byte[] data)
|
|
{
|
|
try
|
|
{
|
|
// Check minimum size for magic number
|
|
if (data.Length < 4)
|
|
return false;
|
|
|
|
// Check for avatar magic number (0xC0A0)
|
|
uint magic = BitConverter.ToUInt32(data, 0);
|
|
return magic == AVATAR_MAGIC;
|
|
}
|
|
catch
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
CompactAvatarData DeserializeCompactData(byte[] data)
|
|
{
|
|
using (MemoryStream stream = new MemoryStream(data))
|
|
using (BinaryReader reader = new BinaryReader(stream))
|
|
{
|
|
// Read and validate magic number
|
|
uint magic = reader.ReadUInt32();
|
|
if (magic != AVATAR_MAGIC)
|
|
{
|
|
Debug.LogWarning($"Invalid avatar packet magic number: 0x{magic:X} (expected 0x{AVATAR_MAGIC:X})");
|
|
return default;
|
|
}
|
|
|
|
CompactAvatarData result = new CompactAvatarData
|
|
{
|
|
// Header
|
|
playerID = reader.ReadByte(),
|
|
sequenceNumber = reader.ReadUInt32(),
|
|
timestamp = reader.ReadUInt32(),
|
|
isFullDataMode = reader.ReadBoolean(),
|
|
|
|
// Root transform
|
|
rootPosition = ReadVector3(reader),
|
|
rootRotation = ReadQuaternion(reader),
|
|
rootScale = ReadVector3(reader)
|
|
};
|
|
|
|
// Variable-length bones
|
|
int boneCount = reader.ReadInt32();
|
|
result.bones = new BoneTransform[boneCount];
|
|
for (int i = 0; i < boneCount; i++)
|
|
{
|
|
result.bones[i] = new BoneTransform
|
|
{
|
|
position = ReadVector3(reader),
|
|
rotation = ReadQuaternion(reader)
|
|
};
|
|
}
|
|
|
|
// Variable-length blend shapes
|
|
int blendShapeCount = reader.ReadInt32();
|
|
result.blendShapes = new float[blendShapeCount];
|
|
for (int i = 0; i < blendShapeCount; i++)
|
|
{
|
|
result.blendShapes[i] = reader.ReadSingle();
|
|
}
|
|
|
|
return result;
|
|
}
|
|
}
|
|
|
|
Vector3 ReadVector3(BinaryReader reader)
|
|
{
|
|
return new Vector3(
|
|
reader.ReadSingle(),
|
|
reader.ReadSingle(),
|
|
reader.ReadSingle()
|
|
);
|
|
}
|
|
|
|
Quaternion ReadQuaternion(BinaryReader reader)
|
|
{
|
|
return new Quaternion(
|
|
reader.ReadSingle(),
|
|
reader.ReadSingle(),
|
|
reader.ReadSingle(),
|
|
reader.ReadSingle()
|
|
);
|
|
}
|
|
|
|
void Update()
|
|
{
|
|
if (!enableReceiver)
|
|
return;
|
|
|
|
// Apply new data if available
|
|
lock (dataLock)
|
|
{
|
|
if (hasNewData)
|
|
{
|
|
ApplyAvatarData(lastReceivedData);
|
|
hasNewData = false;
|
|
lastPacketTime = Time.time;
|
|
}
|
|
}
|
|
}
|
|
|
|
void ApplyAvatarData(CompactAvatarData data)
|
|
{
|
|
// Apply root transform
|
|
if (syncRootPosition)
|
|
{
|
|
if (smoothTransitions)
|
|
{
|
|
targetAvatarRoot.position = Vector3.Lerp(
|
|
targetAvatarRoot.position,
|
|
data.rootPosition,
|
|
Time.deltaTime * transitionSpeed
|
|
);
|
|
}
|
|
else
|
|
{
|
|
targetAvatarRoot.position = data.rootPosition;
|
|
}
|
|
}
|
|
|
|
if (syncRootRotation)
|
|
{
|
|
if (smoothTransitions)
|
|
{
|
|
targetAvatarRoot.rotation = Quaternion.Lerp(
|
|
targetAvatarRoot.rotation,
|
|
data.rootRotation,
|
|
Time.deltaTime * transitionSpeed
|
|
);
|
|
}
|
|
else
|
|
{
|
|
targetAvatarRoot.rotation = data.rootRotation;
|
|
}
|
|
}
|
|
|
|
if (syncRootScale)
|
|
{
|
|
if (smoothTransitions)
|
|
{
|
|
targetAvatarRoot.localScale = Vector3.Lerp(
|
|
targetAvatarRoot.localScale,
|
|
data.rootScale,
|
|
Time.deltaTime * transitionSpeed
|
|
);
|
|
}
|
|
else
|
|
{
|
|
targetAvatarRoot.localScale = data.rootScale;
|
|
}
|
|
}
|
|
|
|
// Apply bone transforms based on data mode
|
|
if (data.isFullDataMode)
|
|
{
|
|
// Full data mode: apply to all bones based on order
|
|
for (int i = 0; i < Mathf.Min(data.bones.Length, allBones.Count); i++)
|
|
{
|
|
Transform bone = allBones[i];
|
|
if (bone != null)
|
|
{
|
|
if (smoothTransitions)
|
|
{
|
|
bone.localPosition = Vector3.Lerp(
|
|
bone.localPosition,
|
|
data.bones[i].position,
|
|
Time.deltaTime * transitionSpeed
|
|
);
|
|
|
|
bone.localRotation = Quaternion.Lerp(
|
|
bone.localRotation,
|
|
data.bones[i].rotation,
|
|
Time.deltaTime * transitionSpeed
|
|
);
|
|
}
|
|
else
|
|
{
|
|
bone.localPosition = data.bones[i].position;
|
|
bone.localRotation = data.bones[i].rotation;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// Optimized mode: apply to priority bones
|
|
for (int i = 0; i < Mathf.Min(data.bones.Length, priorityBones.Length); i++)
|
|
{
|
|
if (boneCache.TryGetValue(priorityBones[i], out Transform bone))
|
|
{
|
|
if (smoothTransitions)
|
|
{
|
|
bone.localPosition = Vector3.Lerp(
|
|
bone.localPosition,
|
|
data.bones[i].position,
|
|
Time.deltaTime * transitionSpeed
|
|
);
|
|
|
|
bone.localRotation = Quaternion.Lerp(
|
|
bone.localRotation,
|
|
data.bones[i].rotation,
|
|
Time.deltaTime * transitionSpeed
|
|
);
|
|
}
|
|
else
|
|
{
|
|
bone.localPosition = data.bones[i].position;
|
|
bone.localRotation = data.bones[i].rotation;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Apply facial blend shapes based on data mode
|
|
if (syncFacialData)
|
|
{
|
|
if (data.isFullDataMode)
|
|
{
|
|
// Full data mode: apply all blend shapes to all meshes
|
|
int blendShapeIndex = 0;
|
|
foreach (var mesh in allMeshes)
|
|
{
|
|
if (mesh != null && mesh.sharedMesh != null)
|
|
{
|
|
for (int i = 0; i < mesh.sharedMesh.blendShapeCount; i++)
|
|
{
|
|
if (blendShapeIndex < data.blendShapes.Length)
|
|
{
|
|
float targetWeight = data.blendShapes[blendShapeIndex] * 100f; // Convert back to 0-100 range
|
|
|
|
if (smoothTransitions)
|
|
{
|
|
float currentWeight = mesh.GetBlendShapeWeight(i);
|
|
float newWeight = Mathf.Lerp(currentWeight, targetWeight, Time.deltaTime * transitionSpeed);
|
|
mesh.SetBlendShapeWeight(i, newWeight);
|
|
}
|
|
else
|
|
{
|
|
mesh.SetBlendShapeWeight(i, targetWeight);
|
|
}
|
|
blendShapeIndex++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// Optimized mode: apply to head mesh only
|
|
if (headMesh != null && headMesh.sharedMesh != null)
|
|
{
|
|
int blendShapeCount = Mathf.Min(data.blendShapes.Length, headMesh.sharedMesh.blendShapeCount);
|
|
for (int i = 0; i < blendShapeCount; i++)
|
|
{
|
|
float targetWeight = data.blendShapes[i] * 100f; // Convert back to 0-100 range
|
|
|
|
if (smoothTransitions)
|
|
{
|
|
float currentWeight = headMesh.GetBlendShapeWeight(i);
|
|
float newWeight = Mathf.Lerp(currentWeight, targetWeight, Time.deltaTime * transitionSpeed);
|
|
headMesh.SetBlendShapeWeight(i, newWeight);
|
|
}
|
|
else
|
|
{
|
|
headMesh.SetBlendShapeWeight(i, targetWeight);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
public void SetTargetPlayerID(byte playerID)
|
|
{
|
|
targetPlayerID = playerID;
|
|
}
|
|
|
|
public bool IsReceivingData()
|
|
{
|
|
return Time.time - lastPacketTime < 1f; // Consider "receiving" if we got data in the last second
|
|
}
|
|
|
|
public float GetPacketLossRate()
|
|
{
|
|
int totalExpected = packetsReceived + packetsDropped;
|
|
return totalExpected > 0 ? (float)packetsDropped / totalExpected : 0f;
|
|
}
|
|
|
|
void StopUDPListener()
|
|
{
|
|
threadRunning = false;
|
|
|
|
// Unsubscribe from shared listener
|
|
if (Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance != null)
|
|
{
|
|
Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance.OnPacketReceived -= HandlePacketReceived;
|
|
}
|
|
}
|
|
|
|
void OnDestroy()
|
|
{
|
|
StopUDPListener();
|
|
}
|
|
|
|
void OnApplicationPause(bool pauseStatus)
|
|
{
|
|
if (pauseStatus)
|
|
{
|
|
StopUDPListener();
|
|
}
|
|
else if (enableReceiver)
|
|
{
|
|
StartUDPListener();
|
|
}
|
|
}
|
|
|
|
void OnGUI()
|
|
{
|
|
if (!showDebugInfo) return;
|
|
|
|
GUILayout.BeginArea(new Rect(320, 10, 300, 260));
|
|
GUILayout.Label($"UDP Avatar Receiver (Binary Only)");
|
|
GUILayout.Label($"Status: {(IsReceivingData() ? "Receiving" : "No Data")}");
|
|
|
|
// Show last received data mode
|
|
if (hasNewData)
|
|
{
|
|
string modeStr = lastReceivedData.isFullDataMode ? "FULL DATA" : "OPTIMIZED";
|
|
GUILayout.Label($"Last Data Mode: {modeStr}");
|
|
GUILayout.Label($"Bones Received: {lastReceivedData.bones?.Length ?? 0}");
|
|
GUILayout.Label($"Blend Shapes: {lastReceivedData.blendShapes?.Length ?? 0}");
|
|
}
|
|
|
|
GUILayout.Label($"Listen Port: {GetActualListenPort()} (shared)");
|
|
GUILayout.Label($"Target Player: {(targetPlayerID == 0 ? "Any" : targetPlayerID.ToString())}");
|
|
GUILayout.Label($"Packets: {packetsReceived} received, {packetsDropped} dropped");
|
|
GUILayout.Label($"Packet Loss: {GetPacketLossRate():P1}");
|
|
GUILayout.Label($"Last Sequence: {lastSequenceNumber}");
|
|
GUILayout.Label($"Cached: {boneCache.Count} priority, {allBones.Count} total bones");
|
|
GUILayout.Label($"Port Sharing: {allowPortSharing}");
|
|
|
|
if (GUILayout.Button(enableReceiver ? "Stop Receiver" : "Start Receiver"))
|
|
{
|
|
enableReceiver = !enableReceiver;
|
|
if (enableReceiver)
|
|
StartUDPListener();
|
|
else
|
|
StopUDPListener();
|
|
}
|
|
|
|
GUILayout.EndArea();
|
|
}
|
|
|
|
int GetActualListenPort()
|
|
{
|
|
if (Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance != null)
|
|
{
|
|
return Convai.Scripts.Runtime.Multiplayer.SharedUDPListener.Instance.ListenPort;
|
|
}
|
|
return listenPort;
|
|
}
|
|
} |