// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using Microsoft.MixedReality.Toolkit.Input;
using Microsoft.MixedReality.Toolkit.Utilities;
using System;
#if WINDOWS_UWP
using Unity.Profiling;
using UnityEngine;
using Windows.Perception.People;
using Windows.UI.Input.Spatial;
#endif // WINDOWS_UWP
namespace Microsoft.MixedReality.Toolkit.WindowsMixedReality
{
///
/// Queries the hand mesh data that an articulated hand on HoloLens 2 can provide.
///
public class WindowsMixedRealityHandMeshProvider
{
///
/// The user's left hand.
///
public static WindowsMixedRealityHandMeshProvider Left { get; } = new WindowsMixedRealityHandMeshProvider(Handedness.Left);
///
/// The user's right hand.
///
public static WindowsMixedRealityHandMeshProvider Right { get; } = new WindowsMixedRealityHandMeshProvider(Handedness.Right);
private WindowsMixedRealityHandMeshProvider(Handedness handedness) => this.handedness = handedness;
[Obsolete("WindowsMixedRealityHandMeshProvider(IMixedRealityController) is obsolete. Please use either the static Left or Right members and call SetInputSource()")]
public WindowsMixedRealityHandMeshProvider(IMixedRealityController controller) : this(controller.ControllerHandedness)
{
SetInputSource(controller.InputSource);
}
private readonly Handedness handedness;
private IMixedRealityInputSource inputSource = null;
///
/// Sets the that represents the current hand for this mesh.
///
/// Implementation of the hand input source.
public void SetInputSource(IMixedRealityInputSource inputSource)
{
this.inputSource = inputSource;
#if WINDOWS_UWP
hasRequestedHandMeshObserver = false;
handMeshObserver = null;
#endif // WINDOWS_UWP
}
#if WINDOWS_UWP
private HandMeshObserver handMeshObserver = null;
private bool hasRequestedHandMeshObserver = false;
private int handMeshModelId = -1;
private int neutralPoseVersion = -1;
private ushort[] handMeshTriangleIndices = null;
private HandMeshVertex[] vertexAndNormals = null;
private Vector3[] handMeshVerticesUnity = null;
private Vector3[] handMeshNormalsUnity = null;
private int[] handMeshTriangleIndicesUnity = null;
private Vector2[] handMeshUVsUnity = null;
private async void SetHandMeshObserver(SpatialInteractionSourceState sourceState)
{
handMeshObserver = await sourceState.Source.TryCreateHandMeshObserverAsync();
}
private void InitializeUVs(Vector3[] poseVertices)
{
if (poseVertices.Length == 0)
{
Debug.LogError("Loaded 0 vertices for poseVertices");
handMeshUVsUnity = Array.Empty();
return;
}
float minY = poseVertices[0].y;
float maxY = minY;
for (int ix = 1; ix < poseVertices.Length; ix++)
{
Vector3 p = poseVertices[ix];
if (p.y < minY)
{
minY = p.y;
}
else if (p.y > maxY)
{
maxY = p.y;
}
}
float scale = 1.0f / (maxY - minY);
if ((handMeshUVsUnity == null) ||
(handMeshUVsUnity.Length != poseVertices.Length))
{
handMeshUVsUnity = new Vector2[poseVertices.Length];
}
for (int ix = 0; ix < poseVertices.Length; ix++)
{
Vector3 p = poseVertices[ix];
handMeshUVsUnity[ix] = new Vector2(p.x * scale + 0.5f, (p.y - minY) * scale);
}
}
private static readonly ProfilerMarker UpdateHandMeshPerfMarker = new ProfilerMarker($"[MRTK] {nameof(WindowsMixedRealityHandMeshProvider)}.UpdateHandMesh");
private HandMeshInfo handMeshInfo = new HandMeshInfo();
private Vector3[] neutralPoseVertices = null;
private HandMeshVertex[] neutralVertexAndNormals = null;
///
/// Updates the current hand mesh based on the passed in state of the hand.
///
/// The current hand state.
public void UpdateHandMesh(SpatialInteractionSourceState sourceState)
{
using (UpdateHandMeshPerfMarker.Auto())
{
MixedRealityInputSystemProfile inputSystemProfile = CoreServices.InputSystem?.InputSystemProfile;
MixedRealityHandTrackingProfile handTrackingProfile = inputSystemProfile != null ? inputSystemProfile.HandTrackingProfile : null;
if (handTrackingProfile == null || !handTrackingProfile.EnableHandMeshVisualization)
{
// If hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created
if (handMeshObserver != null)
{
// Notify that hand mesh has been updated (cleared)
handMeshInfo = new HandMeshInfo();
CoreServices.InputSystem?.RaiseHandMeshUpdated(inputSource, handedness, handMeshInfo);
hasRequestedHandMeshObserver = false;
handMeshObserver = null;
}
return;
}
// Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it.
if (handMeshObserver == null && !hasRequestedHandMeshObserver)
{
SetHandMeshObserver(sourceState);
hasRequestedHandMeshObserver = true;
}
HandPose handPose = sourceState.TryGetHandPose();
if (handMeshObserver != null && handPose != null)
{
uint triangleIndexCount = handMeshObserver.TriangleIndexCount;
if (handMeshTriangleIndices == null ||
handMeshTriangleIndices.Length != triangleIndexCount)
{
handMeshTriangleIndices = new ushort[triangleIndexCount];
handMeshTriangleIndicesUnity = new int[triangleIndexCount];
}
int modelId = handMeshObserver.ModelId;
if (handMeshModelId != modelId)
{
handMeshObserver.GetTriangleIndices(handMeshTriangleIndices);
handMeshModelId = modelId;
Array.Copy(handMeshTriangleIndices, handMeshTriangleIndicesUnity, triangleIndexCount);
}
int poseVersion = handMeshObserver.NeutralPoseVersion;
if (neutralPoseVersion != poseVersion)
{
// Compute neutral pose
if ((neutralPoseVertices == null) ||
(neutralPoseVertices.Length != handMeshObserver.VertexCount))
{
neutralPoseVertices = new Vector3[handMeshObserver.VertexCount];
}
HandPose neutralPose = handMeshObserver.NeutralPose;
if ((neutralVertexAndNormals == null) ||
(neutralVertexAndNormals.Length != handMeshObserver.VertexCount))
{
neutralVertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
}
HandMeshVertexState handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose);
handMeshVertexState.GetVertices(neutralVertexAndNormals);
for (int i = 0; i < handMeshObserver.VertexCount; i++)
{
neutralVertexAndNormals[i].Position.ConvertToUnityVector3(ref neutralPoseVertices[i]);
};
neutralPoseVersion = poseVersion;
// Compute UV mapping
InitializeUVs(neutralPoseVertices);
}
if (vertexAndNormals == null)
{
vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
handMeshVerticesUnity = new Vector3[handMeshObserver.VertexCount];
handMeshNormalsUnity = new Vector3[handMeshObserver.VertexCount];
}
if (vertexAndNormals != null && handMeshTriangleIndices != null)
{
HandMeshVertexState handMeshVertexState = null;
try
{
handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose);
}
catch (ArgumentException)
{
Debug.Log($"{nameof(WindowsMixedRealityHandMeshProvider)} failed to update the hand mesh. This might happen if a source was detected and lost rapidly. Otherwise, the mesh might be stale this frame.");
}
if (handMeshVertexState == null)
{
return;
}
handMeshVertexState.GetVertices(vertexAndNormals);
var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem);
if (meshTransform.HasValue)
{
System.Numerics.Matrix4x4.Decompose(meshTransform.Value,
out _,
out System.Numerics.Quaternion rotation,
out System.Numerics.Vector3 translation);
for (int i = 0; i < handMeshObserver.VertexCount; i++)
{
vertexAndNormals[i].Position.ConvertToUnityVector3(ref handMeshVerticesUnity[i]);
vertexAndNormals[i].Normal.ConvertToUnityVector3(ref handMeshNormalsUnity[i]);
};
// Hands should follow the Playspace to accommodate teleporting, so fold in the Playspace transform.
Vector3 positionUnity = MixedRealityPlayspace.TransformPoint(translation.ToUnityVector3());
Quaternion rotationUnity = MixedRealityPlayspace.Rotation * rotation.ToUnityQuaternion();
handMeshInfo.vertices = handMeshVerticesUnity;
handMeshInfo.normals = handMeshNormalsUnity;
handMeshInfo.triangles = handMeshTriangleIndicesUnity;
handMeshInfo.uvs = handMeshUVsUnity;
handMeshInfo.position = positionUnity;
handMeshInfo.rotation = rotationUnity;
CoreServices.InputSystem?.RaiseHandMeshUpdated(inputSource, handedness, handMeshInfo);
}
}
}
}
}
#endif // WINDOWS_UWP
}
}