tomh104/charade_experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial upload

Browse Source
This commit is contained in:
tom.hempel
2025-09-17 13:19:48 +02:00
commit 68ddcae08b
1436 changed files with 339383 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
vrcharades-unity/vrcharades-master/Assets/Scripts/CharadeWord.cs Normal file
View File

@ -0,0 +1,63 @@
using UnityEngine;
using TMPro;
using UnityEngine.Assertions;
using static OVRFaceExpressions;
public class CharadeWord : MonoBehaviour
{
[SerializeField]
private AudioSource audioSource;
[SerializeField]
private AudioClip audioClipCorrect;
[SerializeField]
private AudioClip audioClipWrong;
[SerializeField]
private TMP_Text text;
public string currentWord = null;
public int playSound = -1;
public float remainingTime = 0.0f;
// Start is called once before the first execution of Update after the MonoBehaviour is created
private void Start()
{
text.text = "";
}
// Update is called once per frame
private void Update()
{
if (currentWord != null && remainingTime < Time.deltaTime)
{
currentWord = null;
}
else
{
remainingTime -= Time.deltaTime;
}
switch (playSound)
{
case 0:
audioSource.PlayOneShot(audioClipWrong);
break;
case 1:
audioSource.PlayOneShot(audioClipCorrect);
break;
default:
break;
}
playSound = -1;
if (currentWord == null)
{
text.gameObject.SetActive(false);
}
else
{
text.gameObject.SetActive(true);
text.text = $"{currentWord}\n{Mathf.Round(remainingTime)}";
}
}
}

2
vrcharades-unity/vrcharades-master/Assets/Scripts/CharadeWord.cs.meta Normal file
View File

@ -0,0 +1,2 @@
fileFormatVersion: 2
guid: 67614b7e8fbdb1045bdec1958f645ed0

22
vrcharades-unity/vrcharades-master/Assets/Scripts/CustomSkeletonDataProvider.cs Normal file
View File

@ -0,0 +1,22 @@
using UnityEngine;
public class CustomSkeletonDataProvider : MonoBehaviour, OVRSkeleton.IOVRSkeletonDataProvider // , OVRSkeletonRenderer.IOVRSkeletonRendererDataProvider
{
public OVRSkeleton.SkeletonPoseData latestPoseData = default;
OVRSkeleton.SkeletonType OVRSkeleton.IOVRSkeletonDataProvider.GetSkeletonType()
{
return OVRSkeleton.SkeletonType.Body;
}
OVRSkeleton.SkeletonPoseData OVRSkeleton.IOVRSkeletonDataProvider.GetSkeletonPoseData()
{
return latestPoseData;
}
// OVRSkeletonRenderer.SkeletonRendererData OVRSkeletonRenderer.IOVRSkeletonRendererDataProvider.GetSkeletonRendererData()
// {
// OVRSkeletonRenderer.IOVRSkeletonRendererDataProvider dataProvider = inner as OVRSkeletonRenderer.IOVRSkeletonRendererDataProvider;
// return dataProvider.GetSkeletonRendererData();
// }
}

2
vrcharades-unity/vrcharades-master/Assets/Scripts/CustomSkeletonDataProvider.cs.meta Normal file
View File

@ -0,0 +1,2 @@
fileFormatVersion: 2
guid: d84c36b1e15644846a2ed30f6283a366

360
vrcharades-unity/vrcharades-master/Assets/Scripts/CustomWeightsProvider.cs Normal file
View File

@ -0,0 +1,360 @@
// Copyright (c) Meta Platforms, Inc. and affiliates.
using System;
using System.Collections.Concurrent;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using Meta.XR.Movement.FaceTracking;
using TMPro;
using UnityEngine;
using static OVRFaceExpressions;
[DefaultExecutionOrder(140)]
public class CustomWeightsProvider : Meta.XR.Movement.FaceTracking.Samples.WeightsProvider
{
[SerializeField]
private CustomSkeletonDataProvider skeletonDataProvider;
[SerializeField]
private TrackWeights trackWeights;
[SerializeField]
private Transform leftEye;
[SerializeField]
private Transform rightEye;
[SerializeField]
private Transform[] bones;
[SerializeField]
private CharadeWord charadeWord;
private UdpClient receiver;
private Thread receiveThread;
private ConcurrentQueue<string> positionQueue = new ConcurrentQueue<string>();
[SerializeField]
private GameObject[] FaceParts;
[SerializeField]
private GameObject[] HandParts;
[SerializeField]
private TMP_Text dbgText;
[SerializeField]
private GameObject dbgObject;
private bool showHead = true;
private bool showFacialExpression = true;
private bool showEyeRotation = true;
private bool showHands = true;
void Start()
{
receiver = new UdpClient();
receiver.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
receiver.Client.Bind(new IPEndPoint(IPAddress.Any, 5000));
receiveThread = new Thread(ReceiveData);
receiveThread.IsBackground = true;
receiveThread.Start();
}
void ReceiveData()
{
IPEndPoint from = new IPEndPoint(IPAddress.Any, 0);
while (true)
{
try
{
//Debug.Log("Recieving...");
byte[] data = receiver.Receive(ref from);
string msg = Encoding.UTF8.GetString(data);
if (msg.StartsWith("IP:"))
{
string serverIP = msg[3..].Trim();
trackWeights.serverEndpoint = new IPEndPoint(IPAddress.Parse(serverIP), 5000);
}
else if (msg.StartsWith("CHARADE:"))
{
string parts_str = msg[8..].Trim();
string[] parts = parts_str.Split(';');
charadeWord.playSound = int.Parse(parts[0]);
charadeWord.remainingTime = float.Parse(parts[1]);
charadeWord.currentWord = parts[2] == "" ? null : parts[2];
}
else if (msg.StartsWith("MODE:"))
{
string partsStr = msg[5..].Trim();
string[] parts = partsStr.Split(';');
showHead = int.Parse(parts[0]) != 0;
showFacialExpression = int.Parse(parts[1]) != 0;
showEyeRotation = int.Parse(parts[2]) != 0;
showHands = int.Parse(parts[3]) != 0;
}
else
{
positionQueue.Enqueue(msg);
}
}
catch (Exception e)
{
Debug.Log("Receive error: " + e.Message);
}
}
}
/// <summary>
/// The face expressions provider to source from.
/// </summary>
// [SerializeField]
// [Tooltip(OVRWeightsProviderTooltips.OvrFaceExpressions)]
// protected OVRFaceExpressions _ovrFaceExpressions;
// /// <inheritdoc cref="_ovrFaceExpressions"/>
// public OVRFaceExpressions OVRFaceExpressionComp
// {
// get => _ovrFaceExpressions;
// set => _ovrFaceExpressions = value;
// }
float[] _allWeights = null;
string[] _weightNames = null;
private void Awake()
{
// Assert.IsNotNull(_ovrFaceExpressions);
}
private void EnsureInitialize()
{
if (_allWeights == null)
{
_allWeights = new float[(int)FaceExpression.Max];
}
if (_weightNames == null)
{
_weightNames = new string[(int)FaceExpression.Max];
for (int i = (int)FaceExpression.BrowLowererL; i < (int)FaceExpression.Max; i++)
{
_weightNames[i] = ((FaceExpression)i).ToString();
}
}
}
/// <inheritdoc />
public override bool IsValid => true;
/// <inheritdoc />
public override float[] GetWeights()
{
EnsureInitialize();
return _allWeights;
}
/// <inheritdoc />
public override string[] GetWeightNames()
{
EnsureInitialize();
return _weightNames;
}
private void Update()
{
EnsureInitialize();
if (!IsValid)
{
return;
}
GameObject headObject = null;
{
// find the headObject
int childIndex = 0;
dbgText.text = "Children of " + dbgObject.name + ":\n\n";
foreach (Transform child in dbgObject.transform)
{
dbgText.text += "[" + childIndex + "]: " + child.name + '\n';
if (child.gameObject.name.StartsWith("latina_avatar"))
{
int child2Index = 0;
foreach (Transform child2 in child.gameObject.transform)
{
dbgText.text += child2.name + ",";
if (child2.name == "LinaA2ECharacterMirrored (2)_NormalRecalc")
{
headObject = child2.gameObject;
break;
}
}
if (child2Index > 0)
{
dbgText.text += "\n";
}
}
childIndex += 1;
}
}
{
foreach (var part in FaceParts)
{
part.SetActive(showHead);
}
if (headObject != null)
{
headObject.SetActive(showHead);
}
foreach (var part in HandParts)
{
part.SetActive(showHands);
}
}
string latestMsg = null;
while (positionQueue.TryDequeue(out string msg))
{
latestMsg = msg;
}
if (latestMsg != null)
{
// for (int i = (int)FaceExpression.BrowLowererL; i < (int)FaceExpression.Max; i++)
// {
// _allWeights[i] = value;
// }
string[] parts = latestMsg.Split(';');
// const int expectedWeights = (int)FaceExpression.Max - (int)FaceExpression.BrowLowererL;
// float[] weights = new float[(int)FaceExpression.Max];
int num = 0;
while (num < _allWeights.Length)
{
float faceWeight = 0.0f;
if (showFacialExpression)
{
faceWeight = float.Parse(parts[num]);
}
_allWeights[num] = faceWeight;
num += 1;
}
OVRPlugin.Posef rootPose;
rootPose.Orientation.w = float.Parse(parts[num]);
num += 1;
rootPose.Orientation.x = float.Parse(parts[num]);
num += 1;
rootPose.Orientation.y = float.Parse(parts[num]);
num += 1;
rootPose.Orientation.z = float.Parse(parts[num]);
num += 1;
rootPose.Position.x = float.Parse(parts[num]);
num += 1;
rootPose.Position.y = float.Parse(parts[num]);
num += 1;
rootPose.Position.z = float.Parse(parts[num]);
num += 1;
float rootScale = float.Parse(parts[num]);
num += 1;
int n = int.Parse(parts[num]);
num += 1;
OVRPlugin.Quatf[] boneRotations = new OVRPlugin.Quatf[n];
for (int i = 0; i < n; ++i)
{
boneRotations[i].w = float.Parse(parts[num]);
num += 1;
boneRotations[i].x = float.Parse(parts[num]);
num += 1;
boneRotations[i].y = float.Parse(parts[num]);
num += 1;
boneRotations[i].z = float.Parse(parts[num]);
num += 1;
}
bool isDataValid = int.Parse(parts[num]) == 1;
num += 1;
bool isDataHighConfidence = int.Parse(parts[num]) == 1;
num += 1;
n = int.Parse(parts[num]);
num += 1;
OVRPlugin.Vector3f[] boneTranslations = new OVRPlugin.Vector3f[n];
for (int i = 0; i < n; ++i)
{
boneTranslations[i].x = float.Parse(parts[num]);
num += 1;
boneTranslations[i].y = float.Parse(parts[num]);
num += 1;
boneTranslations[i].z = float.Parse(parts[num]);
num += 1;
}
int skeletonChangedCount = int.Parse(parts[num]);
num += 1;
OVRSkeleton.SkeletonPoseData poseData = new OVRSkeleton.SkeletonPoseData
{
RootPose = rootPose,
RootScale = rootScale,
BoneRotations = boneRotations,
IsDataValid = isDataValid,
IsDataHighConfidence = isDataHighConfidence,
BoneTranslations = boneTranslations,
SkeletonChangedCount = skeletonChangedCount,
};
skeletonDataProvider.latestPoseData = poseData;
Quaternion leftEyeRotation;
leftEyeRotation.w = float.Parse(parts[num]);
num += 1;
leftEyeRotation.x = float.Parse(parts[num]);
num += 1;
leftEyeRotation.y = float.Parse(parts[num]);
num += 1;
leftEyeRotation.z = float.Parse(parts[num]);
num += 1;
leftEye.localRotation = showEyeRotation ? leftEyeRotation : Quaternion.Euler(new Vector3(-175.16f, 0.3800049f, 84.529f));
Quaternion rightEyeRotation;
rightEyeRotation.w = float.Parse(parts[num]);
num += 1;
rightEyeRotation.x = float.Parse(parts[num]);
num += 1;
rightEyeRotation.y = float.Parse(parts[num]);
num += 1;
rightEyeRotation.z = float.Parse(parts[num]);
num += 1;
rightEye.localRotation = showEyeRotation ? rightEyeRotation : Quaternion.Euler(new Vector3(3.97f, 179.62f, 84.529f));
}
// for (int i = (int)FaceExpression.BrowLowererL; i < (int)FaceExpression.Max; i++)
// {
// float value = OVRInput.Get(OVRInput.Axis1D.PrimaryHandTrigger, OVRInput.Controller.Touch);
// _allWeights[i] = value;
// }
}
void OnApplicationQuit()
{
receiveThread?.Abort();
receiver?.Close();
}
}

2
vrcharades-unity/vrcharades-master/Assets/Scripts/CustomWeightsProvider.cs.meta Normal file
View File

@ -0,0 +1,2 @@
fileFormatVersion: 2
guid: 5f4d296a32fa8924b96b5130bca85878

277
vrcharades-unity/vrcharades-master/Assets/Scripts/TrackWeights.cs Normal file
View File

@ -0,0 +1,277 @@
using UnityEngine;
using TMPro;
using UnityEngine.Assertions;
using static OVRFaceExpressions;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System;
using System.Globalization;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Threading;
[RequireComponent(typeof(TMP_Text))]
[DefaultExecutionOrder(140)]
public class TrackWeights : MonoBehaviour
{
private UdpClient sender;
public IPEndPoint serverEndpoint;
private float timeUntilNextUpdate = 0.0f;
[SerializeField]
private OVRBody ovrBody;
[SerializeField]
private Transform leftEye;
[SerializeField]
private Transform rightEye;
[SerializeField]
private Transform[] bones;
private ConcurrentQueue<string> sendQueue = new ConcurrentQueue<string>();
// void FixedUpdate()
// {
// EnsureInitialize();
// if (!IsValid)
// {
// return;
// }
// StringBuilder sb = new(1024);
// for (int i = (int)FaceExpression.BrowLowererL; i < (int)FaceExpression.Max; i++)
// {
// // float v = _ovrFaceExpressions[(FaceExpression)i];
// sb.Append(_allWeights[i].ToString("F4"));
// sb.Append(';');
// }
// byte[] data = Encoding.UTF8.GetBytes(sb.ToString());
// sender.Send(data, data.Length, serverEndpoint);
// }
private TMP_Text text;
float[] _allWeights = null;
string[] _weightNames = null;
private Thread senderThread;
/// <summary>
/// The face expressions provider to source from.
/// </summary>
[SerializeField]
protected OVRFaceExpressions _ovrFaceExpressions;
/// <inheritdoc cref="_ovrFaceExpressions"/>
public OVRFaceExpressions OVRFaceExpressionComp
{
get => _ovrFaceExpressions;
set => _ovrFaceExpressions = value;
}
private void Awake()
{
Assert.IsNotNull(_ovrFaceExpressions);
}
private void EnsureInitialize()
{
if (_allWeights == null)
{
_allWeights = new float[(int)FaceExpression.Max];
}
if (_weightNames == null)
{
_weightNames = new string[(int)FaceExpression.Max];
for (int i = (int)FaceExpression.BrowLowererL; i < (int)FaceExpression.Max; i++)
{
_weightNames[i] = ((FaceExpression)i).ToString();
}
}
}
/// <inheritdoc />
public bool IsValid =>
_ovrFaceExpressions.enabled &&
_ovrFaceExpressions.FaceTrackingEnabled &&
_ovrFaceExpressions.ValidExpressions;
// Start is called once before the first execution of Update after the MonoBehaviour is created
private void Start()
{
text = gameObject.GetComponent<TMP_Text>();
sender = new UdpClient();
sender.Client.Bind(new IPEndPoint(IPAddress.Any, 5001));
serverEndpoint = new IPEndPoint(IPAddress.Parse("0.0.0.0"), 5000);
// senderThread = new Thread(SendData);
// senderThread.IsBackground = true;
// senderThread.Start();
}
void SendData()
{
byte[] data = Encoding.UTF8.GetBytes(sb.ToString());
sender.Send(data, data.Length, serverEndpoint);
}
private StringBuilder sb = new(1024 * 10);
// Update is called once per frame
private void Update()
{
EnsureInitialize();
if (!IsValid)
{
return;
}
timeUntilNextUpdate -= Time.deltaTime;
if (timeUntilNextUpdate <= 0.0f)
{
timeUntilNextUpdate = 1.0f / 60.0f;
sb.Clear();
for (int i = (int)FaceExpression.BrowLowererL; i < (int)FaceExpression.Max; i++)
{
_allWeights[i] = _ovrFaceExpressions[(FaceExpression)i];
if (i > 0)
{
sb.Append(';');
}
sb.Append(_allWeights[i].ToString("F4"));
}
OVRSkeleton.IOVRSkeletonDataProvider dataProvider = ovrBody as OVRSkeleton.IOVRSkeletonDataProvider;
OVRSkeleton.SkeletonPoseData poseData = dataProvider.GetSkeletonPoseData();
OVRPlugin.Quatf rootOrientation = poseData.RootPose.Orientation;
OVRPlugin.Vector3f rootPosition = poseData.RootPose.Position;
sb.Append(';');
sb.Append(rootOrientation.w.ToString("F4"));
sb.Append(';');
sb.Append(rootOrientation.x.ToString("F4"));
sb.Append(';');
sb.Append(rootOrientation.y.ToString("F4"));
sb.Append(';');
sb.Append(rootOrientation.z.ToString("F4"));
sb.Append(';');
sb.Append(rootPosition.x.ToString("F4"));
sb.Append(';');
sb.Append(rootPosition.y.ToString("F4"));
sb.Append(';');
sb.Append(rootPosition.z.ToString("F4"));
sb.Append(';');
sb.Append(poseData.RootScale.ToString("F4"));
sb.Append(';');
sb.Append(poseData.BoneRotations.Length.ToString());
foreach (OVRPlugin.Quatf rot in poseData.BoneRotations)
{
sb.Append(';');
sb.Append(rot.w.ToString("F4"));
sb.Append(';');
sb.Append(rot.x.ToString("F4"));
sb.Append(';');
sb.Append(rot.y.ToString("F4"));
sb.Append(';');
sb.Append(rot.z.ToString("F4"));
}
sb.Append(';');
sb.Append((poseData.IsDataValid ? 1 : 0).ToString());
sb.Append(';');
sb.Append((poseData.IsDataHighConfidence ? 1 : 0).ToString());
sb.Append(';');
sb.Append(poseData.BoneTranslations.Length.ToString());
foreach (OVRPlugin.Vector3f trans in poseData.BoneTranslations)
{
sb.Append(';');
sb.Append(trans.x.ToString("F4"));
sb.Append(';');
sb.Append(trans.y.ToString("F4"));
sb.Append(';');
sb.Append(trans.z.ToString("F4"));
}
sb.Append(';');
sb.Append(poseData.SkeletonChangedCount.ToString());
sb.Append(';');
sb.Append(leftEye.localRotation.w.ToString("F4"));
sb.Append(';');
sb.Append(leftEye.localRotation.x.ToString("F4"));
sb.Append(';');
sb.Append(leftEye.localRotation.y.ToString("F4"));
sb.Append(';');
sb.Append(leftEye.localRotation.z.ToString("F4"));
sb.Append(';');
sb.Append(rightEye.localRotation.w.ToString("F4"));
sb.Append(';');
sb.Append(rightEye.localRotation.x.ToString("F4"));
sb.Append(';');
sb.Append(rightEye.localRotation.y.ToString("F4"));
sb.Append(';');
sb.Append(rightEye.localRotation.z.ToString("F4"));
// foreach (var bone in bones)
// {
// sb.Append(';');
// sb.Append(bone.localPosition.x.ToString("F4"));
// sb.Append(';');
// sb.Append(bone.localPosition.y.ToString("F4"));
// sb.Append(';');
// sb.Append(bone.localPosition.z.ToString("F4"));
// sb.Append(';');
// sb.Append(bone.localRotation.w.ToString("F4"));
// sb.Append(';');
// sb.Append(bone.localRotation.x.ToString("F4"));
// sb.Append(';');
// sb.Append(bone.localRotation.y.ToString("F4"));
// sb.Append(';');
// sb.Append(bone.localRotation.z.ToString("F4"));
// sb.Append(';');
// sb.Append(bone.localScale.x.ToString("F4"));
// sb.Append(';');
// sb.Append(bone.localScale.y.ToString("F4"));
// sb.Append(';');
// sb.Append(bone.localScale.z.ToString("F4"));
// }
// sendQueue.Enqueue(sb.ToString());
byte[] data = Encoding.UTF8.GetBytes(sb.ToString());
sender.Send(data, data.Length, serverEndpoint);
}
// text.text = "";
// for (int i = (int)FaceExpression.BrowLowererL; i < (int)FaceExpression.Max; i++)
// {
// _allWeights[i] = _ovrFaceExpressions[(FaceExpression)i];
// text.text += $"{_weightNames[i]}: {_allWeights[i]:F3}\n";
// }
}
void OnApplicationQuit()
{
senderThread?.Abort();
sender?.Close();
}
}

2
vrcharades-unity/vrcharades-master/Assets/Scripts/TrackWeights.cs.meta Normal file
View File

@ -0,0 +1,2 @@
fileFormatVersion: 2
guid: 03cad4b9548071a44b702e91a43af0cc