// Copyright (c) Meta Platforms, Inc. and affiliates. using System; using System.Collections.Concurrent; using System.Linq; using System.Net; using System.Net.Sockets; using System.Text; using System.Threading; using Meta.XR.Movement.FaceTracking; using TMPro; using UnityEngine; using static OVRFaceExpressions; [DefaultExecutionOrder(140)] public class CustomWeightsProvider : Meta.XR.Movement.FaceTracking.Samples.WeightsProvider { [SerializeField] private CustomSkeletonDataProvider skeletonDataProvider; [SerializeField] private TrackWeights trackWeights; [SerializeField] private Transform leftEye; [SerializeField] private Transform rightEye; [SerializeField] private Transform[] bones; [SerializeField] private CharadeWord charadeWord; private UdpClient receiver; private Thread receiveThread; private ConcurrentQueue positionQueue = new ConcurrentQueue(); [SerializeField] private GameObject[] FaceParts; [SerializeField] private GameObject[] HandParts; [SerializeField] private TMP_Text dbgText; [SerializeField] private GameObject dbgObject; private bool showHead = true; private bool showFacialExpression = true; private bool showEyeRotation = true; private bool showHands = true; void Start() { receiver = new UdpClient(); receiver.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); receiver.Client.Bind(new IPEndPoint(IPAddress.Any, 5000)); receiveThread = new Thread(ReceiveData); receiveThread.IsBackground = true; receiveThread.Start(); } void ReceiveData() { IPEndPoint from = new IPEndPoint(IPAddress.Any, 0); while (true) { try { //Debug.Log("Recieving..."); byte[] data = receiver.Receive(ref from); string msg = Encoding.UTF8.GetString(data); if (msg.StartsWith("IP:")) { string serverIP = msg[3..].Trim(); trackWeights.serverEndpoint = new IPEndPoint(IPAddress.Parse(serverIP), 5000); } else if (msg.StartsWith("CHARADE:")) { string parts_str = msg[8..].Trim(); string[] parts = parts_str.Split(';'); charadeWord.playSound = int.Parse(parts[0]); charadeWord.remainingTime = float.Parse(parts[1]); charadeWord.currentWord = parts[2] == "" ? null : parts[2]; } else if (msg.StartsWith("MODE:")) { string partsStr = msg[5..].Trim(); string[] parts = partsStr.Split(';'); showHead = int.Parse(parts[0]) != 0; showFacialExpression = int.Parse(parts[1]) != 0; showEyeRotation = int.Parse(parts[2]) != 0; showHands = int.Parse(parts[3]) != 0; } else { positionQueue.Enqueue(msg); } } catch (Exception e) { Debug.Log("Receive error: " + e.Message); } } } /// /// The face expressions provider to source from. /// // [SerializeField] // [Tooltip(OVRWeightsProviderTooltips.OvrFaceExpressions)] // protected OVRFaceExpressions _ovrFaceExpressions; // /// // public OVRFaceExpressions OVRFaceExpressionComp // { // get => _ovrFaceExpressions; // set => _ovrFaceExpressions = value; // } float[] _allWeights = null; string[] _weightNames = null; private void Awake() { // Assert.IsNotNull(_ovrFaceExpressions); } private void EnsureInitialize() { if (_allWeights == null) { _allWeights = new float[(int)FaceExpression.Max]; } if (_weightNames == null) { _weightNames = new string[(int)FaceExpression.Max]; for (int i = (int)FaceExpression.BrowLowererL; i < (int)FaceExpression.Max; i++) { _weightNames[i] = ((FaceExpression)i).ToString(); } } } /// public override bool IsValid => true; /// public override float[] GetWeights() { EnsureInitialize(); return _allWeights; } /// public override string[] GetWeightNames() { EnsureInitialize(); return _weightNames; } private void Update() { EnsureInitialize(); if (!IsValid) { return; } GameObject headObject = null; { // find the headObject int childIndex = 0; dbgText.text = "Children of " + dbgObject.name + ":\n\n"; foreach (Transform child in dbgObject.transform) { dbgText.text += "[" + childIndex + "]: " + child.name + '\n'; if (child.gameObject.name.StartsWith("latina_avatar")) { int child2Index = 0; foreach (Transform child2 in child.gameObject.transform) { dbgText.text += child2.name + ","; if (child2.name == "LinaA2ECharacterMirrored (2)_NormalRecalc") { headObject = child2.gameObject; break; } } if (child2Index > 0) { dbgText.text += "\n"; } } childIndex += 1; } } { foreach (var part in FaceParts) { part.SetActive(showHead); } if (headObject != null) { headObject.SetActive(showHead); } foreach (var part in HandParts) { part.SetActive(showHands); } } string latestMsg = null; while (positionQueue.TryDequeue(out string msg)) { latestMsg = msg; } if (latestMsg != null) { // for (int i = (int)FaceExpression.BrowLowererL; i < (int)FaceExpression.Max; i++) // { // _allWeights[i] = value; // } string[] parts = latestMsg.Split(';'); // const int expectedWeights = (int)FaceExpression.Max - (int)FaceExpression.BrowLowererL; // float[] weights = new float[(int)FaceExpression.Max]; int num = 0; while (num < _allWeights.Length) { float faceWeight = 0.0f; if (showFacialExpression) { faceWeight = float.Parse(parts[num]); } _allWeights[num] = faceWeight; num += 1; } OVRPlugin.Posef rootPose; rootPose.Orientation.w = float.Parse(parts[num]); num += 1; rootPose.Orientation.x = float.Parse(parts[num]); num += 1; rootPose.Orientation.y = float.Parse(parts[num]); num += 1; rootPose.Orientation.z = float.Parse(parts[num]); num += 1; rootPose.Position.x = float.Parse(parts[num]); num += 1; rootPose.Position.y = float.Parse(parts[num]); num += 1; rootPose.Position.z = float.Parse(parts[num]); num += 1; float rootScale = float.Parse(parts[num]); num += 1; int n = int.Parse(parts[num]); num += 1; OVRPlugin.Quatf[] boneRotations = new OVRPlugin.Quatf[n]; for (int i = 0; i < n; ++i) { boneRotations[i].w = float.Parse(parts[num]); num += 1; boneRotations[i].x = float.Parse(parts[num]); num += 1; boneRotations[i].y = float.Parse(parts[num]); num += 1; boneRotations[i].z = float.Parse(parts[num]); num += 1; } bool isDataValid = int.Parse(parts[num]) == 1; num += 1; bool isDataHighConfidence = int.Parse(parts[num]) == 1; num += 1; n = int.Parse(parts[num]); num += 1; OVRPlugin.Vector3f[] boneTranslations = new OVRPlugin.Vector3f[n]; for (int i = 0; i < n; ++i) { boneTranslations[i].x = float.Parse(parts[num]); num += 1; boneTranslations[i].y = float.Parse(parts[num]); num += 1; boneTranslations[i].z = float.Parse(parts[num]); num += 1; } int skeletonChangedCount = int.Parse(parts[num]); num += 1; OVRSkeleton.SkeletonPoseData poseData = new OVRSkeleton.SkeletonPoseData { RootPose = rootPose, RootScale = rootScale, BoneRotations = boneRotations, IsDataValid = isDataValid, IsDataHighConfidence = isDataHighConfidence, BoneTranslations = boneTranslations, SkeletonChangedCount = skeletonChangedCount, }; skeletonDataProvider.latestPoseData = poseData; Quaternion leftEyeRotation; leftEyeRotation.w = float.Parse(parts[num]); num += 1; leftEyeRotation.x = float.Parse(parts[num]); num += 1; leftEyeRotation.y = float.Parse(parts[num]); num += 1; leftEyeRotation.z = float.Parse(parts[num]); num += 1; leftEye.localRotation = showEyeRotation ? leftEyeRotation : Quaternion.Euler(new Vector3(-175.16f, 0.3800049f, 84.529f)); Quaternion rightEyeRotation; rightEyeRotation.w = float.Parse(parts[num]); num += 1; rightEyeRotation.x = float.Parse(parts[num]); num += 1; rightEyeRotation.y = float.Parse(parts[num]); num += 1; rightEyeRotation.z = float.Parse(parts[num]); num += 1; rightEye.localRotation = showEyeRotation ? rightEyeRotation : Quaternion.Euler(new Vector3(3.97f, 179.62f, 84.529f)); } // for (int i = (int)FaceExpression.BrowLowererL; i < (int)FaceExpression.Max; i++) // { // float value = OVRInput.Get(OVRInput.Axis1D.PrimaryHandTrigger, OVRInput.Controller.Touch); // _allWeights[i] = value; // } } void OnApplicationQuit() { receiveThread?.Abort(); receiver?.Close(); } }