361 lines
11 KiB
C#
361 lines
11 KiB
C#
// Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
|
|
using System;
|
|
using System.Collections.Concurrent;
|
|
using System.Linq;
|
|
using System.Net;
|
|
using System.Net.Sockets;
|
|
using System.Text;
|
|
using System.Threading;
|
|
using Meta.XR.Movement.FaceTracking;
|
|
using TMPro;
|
|
using UnityEngine;
|
|
using static OVRFaceExpressions;
|
|
|
|
[DefaultExecutionOrder(140)]
|
|
public class CustomWeightsProvider : Meta.XR.Movement.FaceTracking.Samples.WeightsProvider
|
|
{
|
|
[SerializeField]
|
|
private CustomSkeletonDataProvider skeletonDataProvider;
|
|
|
|
[SerializeField]
|
|
private TrackWeights trackWeights;
|
|
|
|
[SerializeField]
|
|
private Transform leftEye;
|
|
[SerializeField]
|
|
private Transform rightEye;
|
|
|
|
[SerializeField]
|
|
private Transform[] bones;
|
|
|
|
[SerializeField]
|
|
private CharadeWord charadeWord;
|
|
|
|
private UdpClient receiver;
|
|
|
|
private Thread receiveThread;
|
|
private ConcurrentQueue<string> positionQueue = new ConcurrentQueue<string>();
|
|
|
|
[SerializeField]
|
|
private GameObject[] FaceParts;
|
|
[SerializeField]
|
|
private GameObject[] HandParts;
|
|
|
|
[SerializeField]
|
|
private TMP_Text dbgText;
|
|
|
|
[SerializeField]
|
|
private GameObject dbgObject;
|
|
|
|
private bool showHead = true;
|
|
private bool showFacialExpression = true;
|
|
private bool showEyeRotation = true;
|
|
private bool showHands = true;
|
|
|
|
void Start()
|
|
{
|
|
receiver = new UdpClient();
|
|
receiver.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
|
|
receiver.Client.Bind(new IPEndPoint(IPAddress.Any, 5000));
|
|
|
|
receiveThread = new Thread(ReceiveData);
|
|
receiveThread.IsBackground = true;
|
|
receiveThread.Start();
|
|
}
|
|
|
|
void ReceiveData()
|
|
{
|
|
IPEndPoint from = new IPEndPoint(IPAddress.Any, 0);
|
|
while (true)
|
|
{
|
|
try
|
|
{
|
|
//Debug.Log("Recieving...");
|
|
byte[] data = receiver.Receive(ref from);
|
|
string msg = Encoding.UTF8.GetString(data);
|
|
if (msg.StartsWith("IP:"))
|
|
{
|
|
string serverIP = msg[3..].Trim();
|
|
trackWeights.serverEndpoint = new IPEndPoint(IPAddress.Parse(serverIP), 5000);
|
|
}
|
|
else if (msg.StartsWith("CHARADE:"))
|
|
{
|
|
string parts_str = msg[8..].Trim();
|
|
string[] parts = parts_str.Split(';');
|
|
charadeWord.playSound = int.Parse(parts[0]);
|
|
charadeWord.remainingTime = float.Parse(parts[1]);
|
|
charadeWord.currentWord = parts[2] == "" ? null : parts[2];
|
|
}
|
|
else if (msg.StartsWith("MODE:"))
|
|
{
|
|
string partsStr = msg[5..].Trim();
|
|
string[] parts = partsStr.Split(';');
|
|
|
|
showHead = int.Parse(parts[0]) != 0;
|
|
showFacialExpression = int.Parse(parts[1]) != 0;
|
|
showEyeRotation = int.Parse(parts[2]) != 0;
|
|
showHands = int.Parse(parts[3]) != 0;
|
|
}
|
|
else
|
|
{
|
|
positionQueue.Enqueue(msg);
|
|
}
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
Debug.Log("Receive error: " + e.Message);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// The face expressions provider to source from.
|
|
/// </summary>
|
|
// [SerializeField]
|
|
// [Tooltip(OVRWeightsProviderTooltips.OvrFaceExpressions)]
|
|
// protected OVRFaceExpressions _ovrFaceExpressions;
|
|
// /// <inheritdoc cref="_ovrFaceExpressions"/>
|
|
// public OVRFaceExpressions OVRFaceExpressionComp
|
|
// {
|
|
// get => _ovrFaceExpressions;
|
|
// set => _ovrFaceExpressions = value;
|
|
// }
|
|
|
|
float[] _allWeights = null;
|
|
string[] _weightNames = null;
|
|
|
|
private void Awake()
|
|
{
|
|
// Assert.IsNotNull(_ovrFaceExpressions);
|
|
}
|
|
|
|
private void EnsureInitialize()
|
|
{
|
|
if (_allWeights == null)
|
|
{
|
|
_allWeights = new float[(int)FaceExpression.Max];
|
|
}
|
|
if (_weightNames == null)
|
|
{
|
|
_weightNames = new string[(int)FaceExpression.Max];
|
|
for (int i = (int)FaceExpression.BrowLowererL; i < (int)FaceExpression.Max; i++)
|
|
{
|
|
_weightNames[i] = ((FaceExpression)i).ToString();
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public override bool IsValid => true;
|
|
|
|
/// <inheritdoc />
|
|
public override float[] GetWeights()
|
|
{
|
|
EnsureInitialize();
|
|
return _allWeights;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public override string[] GetWeightNames()
|
|
{
|
|
EnsureInitialize();
|
|
return _weightNames;
|
|
}
|
|
|
|
private void Update()
|
|
{
|
|
EnsureInitialize();
|
|
if (!IsValid)
|
|
{
|
|
return;
|
|
}
|
|
|
|
GameObject headObject = null;
|
|
{
|
|
// find the headObject
|
|
int childIndex = 0;
|
|
dbgText.text = "Children of " + dbgObject.name + ":\n\n";
|
|
foreach (Transform child in dbgObject.transform)
|
|
{
|
|
dbgText.text += "[" + childIndex + "]: " + child.name + '\n';
|
|
if (child.gameObject.name.StartsWith("latina_avatar"))
|
|
{
|
|
int child2Index = 0;
|
|
foreach (Transform child2 in child.gameObject.transform)
|
|
{
|
|
dbgText.text += child2.name + ",";
|
|
if (child2.name == "LinaA2ECharacterMirrored (2)_NormalRecalc")
|
|
{
|
|
headObject = child2.gameObject;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (child2Index > 0)
|
|
{
|
|
dbgText.text += "\n";
|
|
}
|
|
}
|
|
|
|
childIndex += 1;
|
|
}
|
|
}
|
|
|
|
|
|
{
|
|
foreach (var part in FaceParts)
|
|
{
|
|
part.SetActive(showHead);
|
|
}
|
|
if (headObject != null)
|
|
{
|
|
headObject.SetActive(showHead);
|
|
}
|
|
|
|
foreach (var part in HandParts)
|
|
{
|
|
part.SetActive(showHands);
|
|
}
|
|
}
|
|
|
|
string latestMsg = null;
|
|
while (positionQueue.TryDequeue(out string msg))
|
|
{
|
|
latestMsg = msg;
|
|
}
|
|
|
|
if (latestMsg != null)
|
|
{
|
|
// for (int i = (int)FaceExpression.BrowLowererL; i < (int)FaceExpression.Max; i++)
|
|
// {
|
|
// _allWeights[i] = value;
|
|
// }
|
|
|
|
string[] parts = latestMsg.Split(';');
|
|
// const int expectedWeights = (int)FaceExpression.Max - (int)FaceExpression.BrowLowererL;
|
|
// float[] weights = new float[(int)FaceExpression.Max];
|
|
|
|
int num = 0;
|
|
|
|
while (num < _allWeights.Length)
|
|
{
|
|
float faceWeight = 0.0f;
|
|
if (showFacialExpression)
|
|
{
|
|
faceWeight = float.Parse(parts[num]);
|
|
}
|
|
|
|
_allWeights[num] = faceWeight;
|
|
num += 1;
|
|
}
|
|
|
|
|
|
OVRPlugin.Posef rootPose;
|
|
rootPose.Orientation.w = float.Parse(parts[num]);
|
|
num += 1;
|
|
rootPose.Orientation.x = float.Parse(parts[num]);
|
|
num += 1;
|
|
rootPose.Orientation.y = float.Parse(parts[num]);
|
|
num += 1;
|
|
rootPose.Orientation.z = float.Parse(parts[num]);
|
|
num += 1;
|
|
|
|
rootPose.Position.x = float.Parse(parts[num]);
|
|
num += 1;
|
|
rootPose.Position.y = float.Parse(parts[num]);
|
|
num += 1;
|
|
rootPose.Position.z = float.Parse(parts[num]);
|
|
num += 1;
|
|
|
|
float rootScale = float.Parse(parts[num]);
|
|
num += 1;
|
|
|
|
|
|
int n = int.Parse(parts[num]);
|
|
num += 1;
|
|
OVRPlugin.Quatf[] boneRotations = new OVRPlugin.Quatf[n];
|
|
for (int i = 0; i < n; ++i)
|
|
{
|
|
boneRotations[i].w = float.Parse(parts[num]);
|
|
num += 1;
|
|
boneRotations[i].x = float.Parse(parts[num]);
|
|
num += 1;
|
|
boneRotations[i].y = float.Parse(parts[num]);
|
|
num += 1;
|
|
boneRotations[i].z = float.Parse(parts[num]);
|
|
num += 1;
|
|
}
|
|
|
|
bool isDataValid = int.Parse(parts[num]) == 1;
|
|
num += 1;
|
|
|
|
bool isDataHighConfidence = int.Parse(parts[num]) == 1;
|
|
num += 1;
|
|
|
|
n = int.Parse(parts[num]);
|
|
num += 1;
|
|
OVRPlugin.Vector3f[] boneTranslations = new OVRPlugin.Vector3f[n];
|
|
for (int i = 0; i < n; ++i)
|
|
{
|
|
boneTranslations[i].x = float.Parse(parts[num]);
|
|
num += 1;
|
|
boneTranslations[i].y = float.Parse(parts[num]);
|
|
num += 1;
|
|
boneTranslations[i].z = float.Parse(parts[num]);
|
|
num += 1;
|
|
}
|
|
|
|
int skeletonChangedCount = int.Parse(parts[num]);
|
|
num += 1;
|
|
|
|
OVRSkeleton.SkeletonPoseData poseData = new OVRSkeleton.SkeletonPoseData
|
|
{
|
|
RootPose = rootPose,
|
|
RootScale = rootScale,
|
|
BoneRotations = boneRotations,
|
|
IsDataValid = isDataValid,
|
|
IsDataHighConfidence = isDataHighConfidence,
|
|
BoneTranslations = boneTranslations,
|
|
SkeletonChangedCount = skeletonChangedCount,
|
|
};
|
|
|
|
skeletonDataProvider.latestPoseData = poseData;
|
|
|
|
Quaternion leftEyeRotation;
|
|
leftEyeRotation.w = float.Parse(parts[num]);
|
|
num += 1;
|
|
leftEyeRotation.x = float.Parse(parts[num]);
|
|
num += 1;
|
|
leftEyeRotation.y = float.Parse(parts[num]);
|
|
num += 1;
|
|
leftEyeRotation.z = float.Parse(parts[num]);
|
|
num += 1;
|
|
leftEye.localRotation = showEyeRotation ? leftEyeRotation : Quaternion.Euler(new Vector3(-175.16f, 0.3800049f, 84.529f));
|
|
|
|
Quaternion rightEyeRotation;
|
|
rightEyeRotation.w = float.Parse(parts[num]);
|
|
num += 1;
|
|
rightEyeRotation.x = float.Parse(parts[num]);
|
|
num += 1;
|
|
rightEyeRotation.y = float.Parse(parts[num]);
|
|
num += 1;
|
|
rightEyeRotation.z = float.Parse(parts[num]);
|
|
num += 1;
|
|
rightEye.localRotation = showEyeRotation ? rightEyeRotation : Quaternion.Euler(new Vector3(3.97f, 179.62f, 84.529f));
|
|
}
|
|
|
|
// for (int i = (int)FaceExpression.BrowLowererL; i < (int)FaceExpression.Max; i++)
|
|
// {
|
|
// float value = OVRInput.Get(OVRInput.Axis1D.PrimaryHandTrigger, OVRInput.Controller.Touch);
|
|
// _allWeights[i] = value;
|
|
// }
|
|
}
|
|
|
|
void OnApplicationQuit()
|
|
{
|
|
receiveThread?.Abort();
|
|
receiver?.Close();
|
|
}
|
|
}
|