/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* Licensed under the Oculus SDK License Agreement (the "License");
* you may not use the Oculus SDK except in compliance with the License,
* which is provided at the time of installation or download, or which
* otherwise accompanies this software in either electronic or hard copy form.
*
* You may obtain a copy of the License at
*
* https://developer.oculus.com/licenses/oculussdk/
*
* Unless required by applicable law or agreed to in writing, the Oculus SDK
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
///
/// This class manages the face expressions data.
///
///
/// Refers to the enum for the list of face expressions.
///
[HelpURL("https://developer.oculus.com/reference/unity/latest/class_o_v_r_face_expressions")]
public class OVRFaceExpressions : MonoBehaviour, IReadOnlyCollection, OVRFaceExpressions.WeightProvider
{
///
/// True if face tracking is enabled, otherwise false.
///
public bool FaceTrackingEnabled => OVRPlugin.faceTrackingEnabled;
public interface WeightProvider
{
float GetWeight(FaceExpression expression);
}
///
/// True if the facial expressions are valid, otherwise false.
///
///
/// This value gets updated in every frame. You should check this
/// value before querying for face expressions.
///
public bool ValidExpressions { get; private set; }
///
/// True if the eye look-related blend shapes are valid, otherwise false.
///
///
/// This property affects the behavior of two sets of blend shapes.
///
/// **EyesLook:**
/// -
/// -
/// -
/// -
/// -
/// -
/// -
/// -
///
/// **EyesClosed:**
/// -
/// -
///
/// **When is `false`:**
/// - The `EyesLook` blend shapes are set to zero.
/// - The `EyesClosed` blend shapes range from 0..1, and represent the true state of the eyelids.
///
/// **When is `true`:**
/// - The `EyesLook` blend shapes are valid.
/// - The `EyesClosed` blend shapes are modified so that the sum of the `EyesClosedX` and `EyesLookDownX` blend shapes
/// range from 0..1. This helps avoid double deformation of the avatar's eye lids when they may be driven by both
/// the `EyesClosed` and `EyesLookDown` blend shapes. To recover the true `EyesClosed` values, add the
/// minimum of `EyesLookDownL` and `EyesLookDownR` blend shapes back using the following formula:
/// `EyesClosedL` += min(`EyesLookDownL`, `EyesLookDownR`)
/// `EyesClosedR` += min(`EyesLookDownL`, `EyesLookDownR`)
///
public bool EyeFollowingBlendshapesValid { get; private set; }
private OVRPlugin.FaceState _currentFaceState;
private const OVRPermissionsRequester.Permission FaceTrackingPermission =
OVRPermissionsRequester.Permission.FaceTracking;
private Action _onPermissionGranted;
private static int _trackingInstanceCount;
private void Awake()
{
_onPermissionGranted = OnPermissionGranted;
}
private void OnEnable()
{
_trackingInstanceCount++;
if (!StartFaceTracking())
{
enabled = false;
}
}
private void OnPermissionGranted(string permissionId)
{
if (permissionId == OVRPermissionsRequester.GetPermissionId(FaceTrackingPermission))
{
OVRPermissionsRequester.PermissionGranted -= _onPermissionGranted;
enabled = true;
}
}
private bool StartFaceTracking()
{
if (!OVRPermissionsRequester.IsPermissionGranted(FaceTrackingPermission))
{
OVRPermissionsRequester.PermissionGranted -= _onPermissionGranted;
OVRPermissionsRequester.PermissionGranted += _onPermissionGranted;
return false;
}
if (!OVRPlugin.StartFaceTracking())
{
Debug.LogWarning($"[{nameof(OVRFaceExpressions)}] Failed to start face tracking.");
return false;
}
return true;
}
private void OnDisable()
{
if (--_trackingInstanceCount == 0)
{
OVRPlugin.StopFaceTracking();
}
}
private void OnDestroy()
{
OVRPermissionsRequester.PermissionGranted -= _onPermissionGranted;
}
private void Update()
{
ValidExpressions = OVRPlugin.GetFaceState(OVRPlugin.Step.Render, -1, ref _currentFaceState) &&
_currentFaceState.Status.IsValid;
EyeFollowingBlendshapesValid = ValidExpressions && _currentFaceState.Status.IsEyeFollowingBlendshapesValid;
}
///
/// This will return the weight of the given expression.
///
/// Returns weight of expression ranged between 0.0 to 100.0.
///
/// Thrown when is false.
///
///
/// Thrown when value is not in range.
///
public float this[FaceExpression expression]
{
get
{
CheckValidity();
if (expression < 0 || expression >= FaceExpression.Max)
{
throw new ArgumentOutOfRangeException(nameof(expression),
expression,
$"Value must be between 0 to {(int)FaceExpression.Max}");
}
return _currentFaceState.ExpressionWeights[(int)expression];
}
}
public float GetWeight(FaceExpression expression) => this[expression];
///
/// This method tries to gets the weight of the given expression if it's available.
///
/// The expression to get the weight of.
/// The output argument that will contain the expression weight or 0.0 if it's not available.
/// Returns true if the expression weight is available, false otherwise
public bool TryGetFaceExpressionWeight(FaceExpression expression, out float weight)
{
if (!ValidExpressions || expression < 0 || expression >= FaceExpression.Max)
{
weight = 0;
return false;
}
weight = _currentFaceState.ExpressionWeights[(int)expression];
return true;
}
///
/// List of face parts used for getting the face tracking confidence weight in .
///
public enum FaceRegionConfidence
{
///
/// Represents the lower part of the face. It includes the mouth, chin and a portion of the nose and cheek.
///
Lower = OVRPlugin.FaceRegionConfidence.Lower,
///
/// Represents the upper part of the face. It includes the eyes, eye brows and a portion of the nose and cheek.
///
Upper = OVRPlugin.FaceRegionConfidence.Upper,
///
/// Used to determine the size of the enum.
///
Max = OVRPlugin.FaceRegionConfidence.Max
}
///
/// This method tries to gets the confidence weight of the given face part if it's available.
///
/// The part of the face to get the confidence weight of.
/// The output argument that will contain the weight confidence or 0.0 if it's not available.
/// Returns true if the weight confidence is available, false otherwise
public bool TryGetWeightConfidence(FaceRegionConfidence region, out float weightConfidence)
{
if (!ValidExpressions || region < 0 || region >= FaceRegionConfidence.Max)
{
weightConfidence = 0;
return false;
}
weightConfidence = _currentFaceState.ExpressionWeightConfidences[(int)region];
return true;
}
internal void CheckValidity()
{
if (!ValidExpressions)
{
throw new InvalidOperationException(
$"Face expressions are not valid at this time. Use {nameof(ValidExpressions)} to check for validity.");
}
}
///
/// Copies expression weights to a pre-allocated array.
///
/// Pre-allocated destination array for expression weights
/// Starting index in the destination array
///
/// Thrown when is null.
///
///
/// Thrown when there is not enough capacity to copy weights to at index.
///
///
/// Thrown when value is out of bounds.
///
///
/// Thrown when is false.
///
public void CopyTo(float[] array, int startIndex = 0)
{
if (array == null)
{
throw new ArgumentNullException(nameof(array));
}
if (startIndex < 0 || startIndex >= array.Length)
{
throw new ArgumentOutOfRangeException(nameof(startIndex),
startIndex,
$"Value must be between 0 to {array.Length - 1}");
}
if (array.Length - startIndex < (int)FaceExpression.Max)
{
throw new ArgumentException(
$"Capacity is too small - required {(int)FaceExpression.Max}, available {array.Length - startIndex}.",
nameof(array));
}
CheckValidity();
for (int i = 0; i < (int)FaceExpression.Max; i++)
{
array[i + startIndex] = _currentFaceState.ExpressionWeights[i];
}
}
///
/// Allocates a float array and copies expression weights to it.
///
public float[] ToArray()
{
var array = new float[(int)OVRFaceExpressions.FaceExpression.Max];
this.CopyTo(array);
return array;
}
///
/// List of face expressions.
///
public enum FaceExpression
{
[InspectorName("None")]
Invalid = OVRPlugin.FaceExpression.Invalid,
BrowLowererL = OVRPlugin.FaceExpression.Brow_Lowerer_L,
BrowLowererR = OVRPlugin.FaceExpression.Brow_Lowerer_R,
CheekPuffL = OVRPlugin.FaceExpression.Cheek_Puff_L,
CheekPuffR = OVRPlugin.FaceExpression.Cheek_Puff_R,
CheekRaiserL = OVRPlugin.FaceExpression.Cheek_Raiser_L,
CheekRaiserR = OVRPlugin.FaceExpression.Cheek_Raiser_R,
CheekSuckL = OVRPlugin.FaceExpression.Cheek_Suck_L,
CheekSuckR = OVRPlugin.FaceExpression.Cheek_Suck_R,
ChinRaiserB = OVRPlugin.FaceExpression.Chin_Raiser_B,
ChinRaiserT = OVRPlugin.FaceExpression.Chin_Raiser_T,
DimplerL = OVRPlugin.FaceExpression.Dimpler_L,
DimplerR = OVRPlugin.FaceExpression.Dimpler_R,
EyesClosedL = OVRPlugin.FaceExpression.Eyes_Closed_L,
EyesClosedR = OVRPlugin.FaceExpression.Eyes_Closed_R,
EyesLookDownL = OVRPlugin.FaceExpression.Eyes_Look_Down_L,
EyesLookDownR = OVRPlugin.FaceExpression.Eyes_Look_Down_R,
EyesLookLeftL = OVRPlugin.FaceExpression.Eyes_Look_Left_L,
EyesLookLeftR = OVRPlugin.FaceExpression.Eyes_Look_Left_R,
EyesLookRightL = OVRPlugin.FaceExpression.Eyes_Look_Right_L,
EyesLookRightR = OVRPlugin.FaceExpression.Eyes_Look_Right_R,
EyesLookUpL = OVRPlugin.FaceExpression.Eyes_Look_Up_L,
EyesLookUpR = OVRPlugin.FaceExpression.Eyes_Look_Up_R,
InnerBrowRaiserL = OVRPlugin.FaceExpression.Inner_Brow_Raiser_L,
InnerBrowRaiserR = OVRPlugin.FaceExpression.Inner_Brow_Raiser_R,
JawDrop = OVRPlugin.FaceExpression.Jaw_Drop,
JawSidewaysLeft = OVRPlugin.FaceExpression.Jaw_Sideways_Left,
JawSidewaysRight = OVRPlugin.FaceExpression.Jaw_Sideways_Right,
JawThrust = OVRPlugin.FaceExpression.Jaw_Thrust,
LidTightenerL = OVRPlugin.FaceExpression.Lid_Tightener_L,
LidTightenerR = OVRPlugin.FaceExpression.Lid_Tightener_R,
LipCornerDepressorL = OVRPlugin.FaceExpression.Lip_Corner_Depressor_L,
LipCornerDepressorR = OVRPlugin.FaceExpression.Lip_Corner_Depressor_R,
LipCornerPullerL = OVRPlugin.FaceExpression.Lip_Corner_Puller_L,
LipCornerPullerR = OVRPlugin.FaceExpression.Lip_Corner_Puller_R,
LipFunnelerLB = OVRPlugin.FaceExpression.Lip_Funneler_LB,
LipFunnelerLT = OVRPlugin.FaceExpression.Lip_Funneler_LT,
LipFunnelerRB = OVRPlugin.FaceExpression.Lip_Funneler_RB,
LipFunnelerRT = OVRPlugin.FaceExpression.Lip_Funneler_RT,
LipPressorL = OVRPlugin.FaceExpression.Lip_Pressor_L,
LipPressorR = OVRPlugin.FaceExpression.Lip_Pressor_R,
LipPuckerL = OVRPlugin.FaceExpression.Lip_Pucker_L,
LipPuckerR = OVRPlugin.FaceExpression.Lip_Pucker_R,
LipStretcherL = OVRPlugin.FaceExpression.Lip_Stretcher_L,
LipStretcherR = OVRPlugin.FaceExpression.Lip_Stretcher_R,
LipSuckLB = OVRPlugin.FaceExpression.Lip_Suck_LB,
LipSuckLT = OVRPlugin.FaceExpression.Lip_Suck_LT,
LipSuckRB = OVRPlugin.FaceExpression.Lip_Suck_RB,
LipSuckRT = OVRPlugin.FaceExpression.Lip_Suck_RT,
LipTightenerL = OVRPlugin.FaceExpression.Lip_Tightener_L,
LipTightenerR = OVRPlugin.FaceExpression.Lip_Tightener_R,
LipsToward = OVRPlugin.FaceExpression.Lips_Toward,
LowerLipDepressorL = OVRPlugin.FaceExpression.Lower_Lip_Depressor_L,
LowerLipDepressorR = OVRPlugin.FaceExpression.Lower_Lip_Depressor_R,
MouthLeft = OVRPlugin.FaceExpression.Mouth_Left,
MouthRight = OVRPlugin.FaceExpression.Mouth_Right,
NoseWrinklerL = OVRPlugin.FaceExpression.Nose_Wrinkler_L,
NoseWrinklerR = OVRPlugin.FaceExpression.Nose_Wrinkler_R,
OuterBrowRaiserL = OVRPlugin.FaceExpression.Outer_Brow_Raiser_L,
OuterBrowRaiserR = OVRPlugin.FaceExpression.Outer_Brow_Raiser_R,
UpperLidRaiserL = OVRPlugin.FaceExpression.Upper_Lid_Raiser_L,
UpperLidRaiserR = OVRPlugin.FaceExpression.Upper_Lid_Raiser_R,
UpperLipRaiserL = OVRPlugin.FaceExpression.Upper_Lip_Raiser_L,
UpperLipRaiserR = OVRPlugin.FaceExpression.Upper_Lip_Raiser_R,
[InspectorName(null)]
Max = OVRPlugin.FaceExpression.Max
}
#region Face expressions enumerator
public FaceExpressionsEnumerator GetEnumerator() =>
new FaceExpressionsEnumerator(_currentFaceState.ExpressionWeights);
IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
public int Count => _currentFaceState.ExpressionWeights?.Length ?? 0;
public struct FaceExpressionsEnumerator : IEnumerator
{
private float[] _faceExpressions;
private int _index;
private int _count;
internal FaceExpressionsEnumerator(float[] array)
{
_faceExpressions = array;
_index = -1;
_count = _faceExpressions?.Length ?? 0;
}
public bool MoveNext() => ++_index < _count;
public float Current => _faceExpressions[_index];
object IEnumerator.Current => Current;
public void Reset() => _index = -1;
public void Dispose()
{
}
}
#endregion
}