// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using Microsoft.MixedReality.Toolkit.Input;
using Microsoft.MixedReality.Toolkit.Utilities;
using Microsoft.MixedReality.Toolkit.Windows.Utilities;
using System.Collections.Generic;
#if UNITY_WSA
using Unity.Profiling;
using UnityEngine.XR.WSA.Input;
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
using Microsoft.MixedReality.Toolkit.Windows.Input;
using UnityEngine;
#if WINDOWS_UWP
using Windows.Perception.People;
using Windows.UI.Input.Spatial;
#elif DOTNETWINRT_PRESENT
using Microsoft.Windows.Perception.People;
using Microsoft.Windows.UI.Input.Spatial;
#endif
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
#endif // UNITY_WSA
namespace Microsoft.MixedReality.Toolkit.WindowsMixedReality.Input
{
///
/// A Windows Mixed Reality articulated hand instance.
///
[MixedRealityController(
SupportedControllerType.ArticulatedHand,
new[] { Handedness.Left, Handedness.Right },
supportedUnityXRPipelines: SupportedUnityXRPipelines.LegacyXR)]
public class WindowsMixedRealityArticulatedHand : BaseWindowsMixedRealitySource, IMixedRealityHand
{
///
/// Constructor.
///
public WindowsMixedRealityArticulatedHand(
TrackingState trackingState,
Handedness controllerHandedness,
IMixedRealityInputSource inputSource = null,
MixedRealityInteractionMapping[] interactions = null)
: base(trackingState, controllerHandedness, inputSource, interactions, new ArticulatedHandDefinition(inputSource, controllerHandedness))
{
handDefinition = Definition as ArticulatedHandDefinition;
handMeshProvider = (controllerHandedness == Handedness.Left) ? WindowsMixedRealityHandMeshProvider.Left : WindowsMixedRealityHandMeshProvider.Right;
handMeshProvider.SetInputSource(inputSource);
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
articulatedHandApiAvailable = WindowsApiChecker.IsMethodAvailable(
"Windows.UI.Input.Spatial",
"SpatialInteractionSourceState",
"TryGetHandPose");
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
}
private readonly Dictionary unityJointPoses = new Dictionary();
private readonly ArticulatedHandDefinition handDefinition;
private readonly WindowsMixedRealityHandMeshProvider handMeshProvider;
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
private readonly bool articulatedHandApiAvailable = false;
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
#region IMixedRealityHand Implementation
///
public bool TryGetJoint(TrackedHandJoint joint, out MixedRealityPose pose) => unityJointPoses.TryGetValue(joint, out pose);
#endregion IMixedRealityHand Implementation
///
public override bool IsInPointingPose => handDefinition.IsInPointingPose;
#if UNITY_WSA
#region Update data functions
private static readonly ProfilerMarker UpdateControllerPerfMarker = new ProfilerMarker("[MRTK] WindowsMixedRealityArticulatedHand.UpdateController");
///
public override void UpdateController(InteractionSourceState interactionSourceState)
{
using (UpdateControllerPerfMarker.Auto())
{
if (!Enabled) { return; }
base.UpdateController(interactionSourceState);
UpdateHandData(interactionSourceState);
for (int i = 0; i < Interactions?.Length; i++)
{
switch (Interactions[i].InputType)
{
case DeviceInputType.IndexFinger:
handDefinition?.UpdateCurrentIndexPose(Interactions[i]);
break;
case DeviceInputType.ThumbStick:
handDefinition?.UpdateCurrentTeleportPose(Interactions[i]);
break;
}
}
}
}
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
private static readonly ProfilerMarker UpdateHandDataPerfMarker = new ProfilerMarker("[MRTK] WindowsMixedRealityArticulatedHand.UpdateHandData");
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
///
/// Update the hand data from the device.
///
/// The InteractionSourceState retrieved from the platform.
private void UpdateHandData(InteractionSourceState interactionSourceState)
{
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
using (UpdateHandDataPerfMarker.Auto())
{
// Articulated hand support is only present in the 18362 version and beyond Windows
// SDK (which contains the V8 drop of the Universal API Contract). In particular,
// the HandPose related APIs are only present on this version and above.
if (!articulatedHandApiAvailable)
{
return;
}
SpatialInteractionSourceState sourceState = interactionSourceState.source.GetSpatialInteractionSourceState();
if (sourceState == null)
{
return;
}
#if WINDOWS_UWP
handMeshProvider?.UpdateHandMesh(sourceState);
#endif // WINDOWS_UWP
HandPose handPose = sourceState.TryGetHandPose();
if (handPose != null && handPose.TryGetJoints(WindowsMixedRealityUtilities.SpatialCoordinateSystem, jointIndices, jointPoses))
{
for (int i = 0; i < jointPoses.Length; i++)
{
Vector3 position = jointPoses[i].Position.ToUnityVector3();
Quaternion rotation = jointPoses[i].Orientation.ToUnityQuaternion();
// We want the joints to follow the playspace, so fold in the playspace transform here to
// put the joint pose into world space.
position = MixedRealityPlayspace.TransformPoint(position);
rotation = MixedRealityPlayspace.Rotation * rotation;
TrackedHandJoint trackedHandJoint = ConvertHandJointKindToTrackedHandJoint(jointIndices[i]);
if (trackedHandJoint == TrackedHandJoint.IndexTip)
{
lastIndexTipRadius = jointPoses[i].Radius;
}
unityJointPoses[trackedHandJoint] = new MixedRealityPose(position, rotation);
}
handDefinition?.UpdateHandJoints(unityJointPoses);
}
}
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
}
#endregion Update data functions
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
private static readonly HandJointKind[] jointIndices = new HandJointKind[]
{
HandJointKind.Palm,
HandJointKind.Wrist,
HandJointKind.ThumbMetacarpal,
HandJointKind.ThumbProximal,
HandJointKind.ThumbDistal,
HandJointKind.ThumbTip,
HandJointKind.IndexMetacarpal,
HandJointKind.IndexProximal,
HandJointKind.IndexIntermediate,
HandJointKind.IndexDistal,
HandJointKind.IndexTip,
HandJointKind.MiddleMetacarpal,
HandJointKind.MiddleProximal,
HandJointKind.MiddleIntermediate,
HandJointKind.MiddleDistal,
HandJointKind.MiddleTip,
HandJointKind.RingMetacarpal,
HandJointKind.RingProximal,
HandJointKind.RingIntermediate,
HandJointKind.RingDistal,
HandJointKind.RingTip,
HandJointKind.LittleMetacarpal,
HandJointKind.LittleProximal,
HandJointKind.LittleIntermediate,
HandJointKind.LittleDistal,
HandJointKind.LittleTip
};
private readonly JointPose[] jointPoses = new JointPose[jointIndices.Length];
private float lastIndexTipRadius = 0;
private TrackedHandJoint ConvertHandJointKindToTrackedHandJoint(HandJointKind handJointKind)
{
switch (handJointKind)
{
case HandJointKind.Palm: return TrackedHandJoint.Palm;
case HandJointKind.Wrist: return TrackedHandJoint.Wrist;
case HandJointKind.ThumbMetacarpal: return TrackedHandJoint.ThumbMetacarpalJoint;
case HandJointKind.ThumbProximal: return TrackedHandJoint.ThumbProximalJoint;
case HandJointKind.ThumbDistal: return TrackedHandJoint.ThumbDistalJoint;
case HandJointKind.ThumbTip: return TrackedHandJoint.ThumbTip;
case HandJointKind.IndexMetacarpal: return TrackedHandJoint.IndexMetacarpal;
case HandJointKind.IndexProximal: return TrackedHandJoint.IndexKnuckle;
case HandJointKind.IndexIntermediate: return TrackedHandJoint.IndexMiddleJoint;
case HandJointKind.IndexDistal: return TrackedHandJoint.IndexDistalJoint;
case HandJointKind.IndexTip: return TrackedHandJoint.IndexTip;
case HandJointKind.MiddleMetacarpal: return TrackedHandJoint.MiddleMetacarpal;
case HandJointKind.MiddleProximal: return TrackedHandJoint.MiddleKnuckle;
case HandJointKind.MiddleIntermediate: return TrackedHandJoint.MiddleMiddleJoint;
case HandJointKind.MiddleDistal: return TrackedHandJoint.MiddleDistalJoint;
case HandJointKind.MiddleTip: return TrackedHandJoint.MiddleTip;
case HandJointKind.RingMetacarpal: return TrackedHandJoint.RingMetacarpal;
case HandJointKind.RingProximal: return TrackedHandJoint.RingKnuckle;
case HandJointKind.RingIntermediate: return TrackedHandJoint.RingMiddleJoint;
case HandJointKind.RingDistal: return TrackedHandJoint.RingDistalJoint;
case HandJointKind.RingTip: return TrackedHandJoint.RingTip;
case HandJointKind.LittleMetacarpal: return TrackedHandJoint.PinkyMetacarpal;
case HandJointKind.LittleProximal: return TrackedHandJoint.PinkyKnuckle;
case HandJointKind.LittleIntermediate: return TrackedHandJoint.PinkyMiddleJoint;
case HandJointKind.LittleDistal: return TrackedHandJoint.PinkyDistalJoint;
case HandJointKind.LittleTip: return TrackedHandJoint.PinkyTip;
default: return TrackedHandJoint.None;
}
}
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
#endif // UNITY_WSA
}
}