initial upload
This commit is contained in:
tom.hempel
@ -0,0 +1,360 @@
|
||||
// Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using Meta.XR.Movement.FaceTracking;
|
||||
using TMPro;
|
||||
using UnityEngine;
|
||||
using static OVRFaceExpressions;
|
||||
|
||||
[DefaultExecutionOrder(140)]
|
||||
public class CustomWeightsProvider : Meta.XR.Movement.FaceTracking.Samples.WeightsProvider
|
||||
{
|
||||
[SerializeField]
|
||||
private CustomSkeletonDataProvider skeletonDataProvider;
|
||||
|
||||
[SerializeField]
|
||||
private TrackWeights trackWeights;
|
||||
|
||||
[SerializeField]
|
||||
private Transform leftEye;
|
||||
[SerializeField]
|
||||
private Transform rightEye;
|
||||
|
||||
[SerializeField]
|
||||
private Transform[] bones;
|
||||
|
||||
[SerializeField]
|
||||
private CharadeWord charadeWord;
|
||||
|
||||
private UdpClient receiver;
|
||||
|
||||
private Thread receiveThread;
|
||||
private ConcurrentQueue<string> positionQueue = new ConcurrentQueue<string>();
|
||||
|
||||
[SerializeField]
|
||||
private GameObject[] FaceParts;
|
||||
[SerializeField]
|
||||
private GameObject[] HandParts;
|
||||
|
||||
[SerializeField]
|
||||
private TMP_Text dbgText;
|
||||
|
||||
[SerializeField]
|
||||
private GameObject dbgObject;
|
||||
|
||||
private bool showHead = true;
|
||||
private bool showFacialExpression = true;
|
||||
private bool showEyeRotation = true;
|
||||
private bool showHands = true;
|
||||
|
||||
void Start()
|
||||
{
|
||||
receiver = new UdpClient();
|
||||
receiver.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
|
||||
receiver.Client.Bind(new IPEndPoint(IPAddress.Any, 5000));
|
||||
|
||||
receiveThread = new Thread(ReceiveData);
|
||||
receiveThread.IsBackground = true;
|
||||
receiveThread.Start();
|
||||
}
|
||||
|
||||
void ReceiveData()
|
||||
{
|
||||
IPEndPoint from = new IPEndPoint(IPAddress.Any, 0);
|
||||
while (true)
|
||||
{
|
||||
try
|
||||
{
|
||||
//Debug.Log("Recieving...");
|
||||
byte[] data = receiver.Receive(ref from);
|
||||
string msg = Encoding.UTF8.GetString(data);
|
||||
if (msg.StartsWith("IP:"))
|
||||
{
|
||||
string serverIP = msg[3..].Trim();
|
||||
trackWeights.serverEndpoint = new IPEndPoint(IPAddress.Parse(serverIP), 5000);
|
||||
}
|
||||
else if (msg.StartsWith("CHARADE:"))
|
||||
{
|
||||
string parts_str = msg[8..].Trim();
|
||||
string[] parts = parts_str.Split(';');
|
||||
charadeWord.playSound = int.Parse(parts[0]);
|
||||
charadeWord.remainingTime = float.Parse(parts[1]);
|
||||
charadeWord.currentWord = parts[2] == "" ? null : parts[2];
|
||||
}
|
||||
else if (msg.StartsWith("MODE:"))
|
||||
{
|
||||
string partsStr = msg[5..].Trim();
|
||||
string[] parts = partsStr.Split(';');
|
||||
|
||||
showHead = int.Parse(parts[0]) != 0;
|
||||
showFacialExpression = int.Parse(parts[1]) != 0;
|
||||
showEyeRotation = int.Parse(parts[2]) != 0;
|
||||
showHands = int.Parse(parts[3]) != 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
positionQueue.Enqueue(msg);
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Debug.Log("Receive error: " + e.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The face expressions provider to source from.
|
||||
/// </summary>
|
||||
// [SerializeField]
|
||||
// [Tooltip(OVRWeightsProviderTooltips.OvrFaceExpressions)]
|
||||
// protected OVRFaceExpressions _ovrFaceExpressions;
|
||||
// /// <inheritdoc cref="_ovrFaceExpressions"/>
|
||||
// public OVRFaceExpressions OVRFaceExpressionComp
|
||||
// {
|
||||
// get => _ovrFaceExpressions;
|
||||
// set => _ovrFaceExpressions = value;
|
||||
// }
|
||||
|
||||
float[] _allWeights = null;
|
||||
string[] _weightNames = null;
|
||||
|
||||
private void Awake()
|
||||
{
|
||||
// Assert.IsNotNull(_ovrFaceExpressions);
|
||||
}
|
||||
|
||||
private void EnsureInitialize()
|
||||
{
|
||||
if (_allWeights == null)
|
||||
{
|
||||
_allWeights = new float[(int)FaceExpression.Max];
|
||||
}
|
||||
if (_weightNames == null)
|
||||
{
|
||||
_weightNames = new string[(int)FaceExpression.Max];
|
||||
for (int i = (int)FaceExpression.BrowLowererL; i < (int)FaceExpression.Max; i++)
|
||||
{
|
||||
_weightNames[i] = ((FaceExpression)i).ToString();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override bool IsValid => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override float[] GetWeights()
|
||||
{
|
||||
EnsureInitialize();
|
||||
return _allWeights;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string[] GetWeightNames()
|
||||
{
|
||||
EnsureInitialize();
|
||||
return _weightNames;
|
||||
}
|
||||
|
||||
private void Update()
|
||||
{
|
||||
EnsureInitialize();
|
||||
if (!IsValid)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
GameObject headObject = null;
|
||||
{
|
||||
// find the headObject
|
||||
int childIndex = 0;
|
||||
dbgText.text = "Children of " + dbgObject.name + ":\n\n";
|
||||
foreach (Transform child in dbgObject.transform)
|
||||
{
|
||||
dbgText.text += "[" + childIndex + "]: " + child.name + '\n';
|
||||
if (child.gameObject.name.StartsWith("latina_avatar"))
|
||||
{
|
||||
int child2Index = 0;
|
||||
foreach (Transform child2 in child.gameObject.transform)
|
||||
{
|
||||
dbgText.text += child2.name + ",";
|
||||
if (child2.name == "LinaA2ECharacterMirrored (2)_NormalRecalc")
|
||||
{
|
||||
headObject = child2.gameObject;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (child2Index > 0)
|
||||
{
|
||||
dbgText.text += "\n";
|
||||
}
|
||||
}
|
||||
|
||||
childIndex += 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
{
|
||||
foreach (var part in FaceParts)
|
||||
{
|
||||
part.SetActive(showHead);
|
||||
}
|
||||
if (headObject != null)
|
||||
{
|
||||
headObject.SetActive(showHead);
|
||||
}
|
||||
|
||||
foreach (var part in HandParts)
|
||||
{
|
||||
part.SetActive(showHands);
|
||||
}
|
||||
}
|
||||
|
||||
string latestMsg = null;
|
||||
while (positionQueue.TryDequeue(out string msg))
|
||||
{
|
||||
latestMsg = msg;
|
||||
}
|
||||
|
||||
if (latestMsg != null)
|
||||
{
|
||||
// for (int i = (int)FaceExpression.BrowLowererL; i < (int)FaceExpression.Max; i++)
|
||||
// {
|
||||
// _allWeights[i] = value;
|
||||
// }
|
||||
|
||||
string[] parts = latestMsg.Split(';');
|
||||
// const int expectedWeights = (int)FaceExpression.Max - (int)FaceExpression.BrowLowererL;
|
||||
// float[] weights = new float[(int)FaceExpression.Max];
|
||||
|
||||
int num = 0;
|
||||
|
||||
while (num < _allWeights.Length)
|
||||
{
|
||||
float faceWeight = 0.0f;
|
||||
if (showFacialExpression)
|
||||
{
|
||||
faceWeight = float.Parse(parts[num]);
|
||||
}
|
||||
|
||||
_allWeights[num] = faceWeight;
|
||||
num += 1;
|
||||
}
|
||||
|
||||
|
||||
OVRPlugin.Posef rootPose;
|
||||
rootPose.Orientation.w = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
rootPose.Orientation.x = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
rootPose.Orientation.y = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
rootPose.Orientation.z = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
|
||||
rootPose.Position.x = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
rootPose.Position.y = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
rootPose.Position.z = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
|
||||
float rootScale = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
|
||||
|
||||
int n = int.Parse(parts[num]);
|
||||
num += 1;
|
||||
OVRPlugin.Quatf[] boneRotations = new OVRPlugin.Quatf[n];
|
||||
for (int i = 0; i < n; ++i)
|
||||
{
|
||||
boneRotations[i].w = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
boneRotations[i].x = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
boneRotations[i].y = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
boneRotations[i].z = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
}
|
||||
|
||||
bool isDataValid = int.Parse(parts[num]) == 1;
|
||||
num += 1;
|
||||
|
||||
bool isDataHighConfidence = int.Parse(parts[num]) == 1;
|
||||
num += 1;
|
||||
|
||||
n = int.Parse(parts[num]);
|
||||
num += 1;
|
||||
OVRPlugin.Vector3f[] boneTranslations = new OVRPlugin.Vector3f[n];
|
||||
for (int i = 0; i < n; ++i)
|
||||
{
|
||||
boneTranslations[i].x = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
boneTranslations[i].y = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
boneTranslations[i].z = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
}
|
||||
|
||||
int skeletonChangedCount = int.Parse(parts[num]);
|
||||
num += 1;
|
||||
|
||||
OVRSkeleton.SkeletonPoseData poseData = new OVRSkeleton.SkeletonPoseData
|
||||
{
|
||||
RootPose = rootPose,
|
||||
RootScale = rootScale,
|
||||
BoneRotations = boneRotations,
|
||||
IsDataValid = isDataValid,
|
||||
IsDataHighConfidence = isDataHighConfidence,
|
||||
BoneTranslations = boneTranslations,
|
||||
SkeletonChangedCount = skeletonChangedCount,
|
||||
};
|
||||
|
||||
skeletonDataProvider.latestPoseData = poseData;
|
||||
|
||||
Quaternion leftEyeRotation;
|
||||
leftEyeRotation.w = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
leftEyeRotation.x = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
leftEyeRotation.y = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
leftEyeRotation.z = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
leftEye.localRotation = showEyeRotation ? leftEyeRotation : Quaternion.Euler(new Vector3(-175.16f, 0.3800049f, 84.529f));
|
||||
|
||||
Quaternion rightEyeRotation;
|
||||
rightEyeRotation.w = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
rightEyeRotation.x = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
rightEyeRotation.y = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
rightEyeRotation.z = float.Parse(parts[num]);
|
||||
num += 1;
|
||||
rightEye.localRotation = showEyeRotation ? rightEyeRotation : Quaternion.Euler(new Vector3(3.97f, 179.62f, 84.529f));
|
||||
}
|
||||
|
||||
// for (int i = (int)FaceExpression.BrowLowererL; i < (int)FaceExpression.Max; i++)
|
||||
// {
|
||||
// float value = OVRInput.Get(OVRInput.Axis1D.PrimaryHandTrigger, OVRInput.Controller.Touch);
|
||||
// _allWeights[i] = value;
|
||||
// }
|
||||
}
|
||||
|
||||
void OnApplicationQuit()
|
||||
{
|
||||
receiveThread?.Abort();
|
||||
receiver?.Close();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user