// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. using System; using System.Collections.Generic; using System.IO; using UnityEngine; #if UNITY_EDITOR using UnityEditor; #endif namespace Microsoft.MixedReality.Toolkit.Utilities { /// /// Shape of an articulated hand defined by joint poses. /// public class ArticulatedHandPose { private static readonly TrackedHandJoint[] Joints = Enum.GetValues(typeof(TrackedHandJoint)) as TrackedHandJoint[]; /// /// Represents the maximum number of tracked hand joints. /// public static int JointCount { get; } = Joints.Length; /// /// Joint poses are stored as right-hand poses in camera space. /// Output poses are computed in world space, and mirroring on the x axis for the left hand. /// private readonly MixedRealityPose[] localJointPoses; public ArticulatedHandPose() { localJointPoses = new MixedRealityPose[JointCount]; SetZero(); } public ArticulatedHandPose(MixedRealityPose[] localJointPoses) { this.localJointPoses = new MixedRealityPose[JointCount]; Array.Copy(localJointPoses, this.localJointPoses, JointCount); } public MixedRealityPose GetLocalJointPose(TrackedHandJoint joint, Handedness handedness) { MixedRealityPose pose = localJointPoses[(int)joint]; // Pose offset are for right hand, mirror on X axis if left hand is needed if (handedness == Handedness.Left) { pose = new MixedRealityPose( new Vector3(-pose.Position.x, pose.Position.y, pose.Position.z), new Quaternion(pose.Rotation.x, -pose.Rotation.y, -pose.Rotation.z, pose.Rotation.w)); } return pose; } /// /// Compute world space poses from camera-space joint data. /// /// Handedness of the resulting pose /// Rotational offset of the resulting pose /// Translational offset of the resulting pose /// Output array of joint poses public void ComputeJointPoses( Handedness handedness, Quaternion rotation, Vector3 position, MixedRealityPose[] jointsOut) { for (int i = 0; i < JointCount; i++) { // Initialize from local offsets MixedRealityPose pose = GetLocalJointPose(Joints[i], handedness); Vector3 p = pose.Position; Quaternion r = pose.Rotation; // Apply external transform p = position + rotation * p; r = rotation * r; jointsOut[i] = new MixedRealityPose(p, r); } } /// /// Take world space joint poses from any hand and convert into right-hand, camera-space poses. /// /// Input joint poses /// Handedness of the input data /// Rotational offset of the input data /// Translational offset of the input data public void ParseFromJointPoses( MixedRealityPose[] joints, Handedness handedness, Quaternion rotation, Vector3 position) { Quaternion invRotation = Quaternion.Inverse(rotation); Quaternion invCameraRotation = Quaternion.Inverse(CameraCache.Main.transform.rotation); for (int i = 0; i < JointCount; i++) { Vector3 p = joints[i].Position; Quaternion r = joints[i].Rotation; // Apply inverse external transform p = invRotation * (p - position); r = invRotation * r; // To camera space p = invCameraRotation * p; r = invCameraRotation * r; // Pose offset are for right hand, mirror on X axis if left hand is given if (handedness == Handedness.Left) { p.x = -p.x; r.y = -r.y; r.z = -r.z; } localJointPoses[i] = new MixedRealityPose(p, r); } } /// /// Set all poses to zero. /// public void SetZero() { for (int i = 0; i < JointCount; i++) { localJointPoses[i] = MixedRealityPose.ZeroIdentity; } } /// /// Copy data from another articulated hand pose. /// public void Copy(ArticulatedHandPose other) { Array.Copy(other.localJointPoses, localJointPoses, JointCount); } /// /// Blend between two hand poses. /// public void InterpolateOffsets(ArticulatedHandPose poseA, ArticulatedHandPose poseB, float value) { for (int i = 0; i < JointCount; i++) { var p = Vector3.Lerp(poseA.localJointPoses[i].Position, poseB.localJointPoses[i].Position, value); var r = Quaternion.Slerp(poseA.localJointPoses[i].Rotation, poseB.localJointPoses[i].Rotation, value); localJointPoses[i] = new MixedRealityPose(p, r); } } /// /// Supported hand gestures. /// public enum GestureId { /// /// Unspecified hand shape /// None = 0, /// /// Flat hand with fingers spread out /// Flat, /// /// Relaxed hand pose /// Open, /// /// Index finger and Thumb touching, grab point does not move /// Pinch, /// /// Index finger and Thumb touching, wrist does not move /// PinchSteadyWrist, /// /// Index finger stretched out /// Poke, /// /// Grab with whole hand, fist shape /// Grab, /// /// OK sign /// ThumbsUp, /// /// Victory sign /// Victory, /// /// Relaxed hand pose, grab point does not move /// OpenSteadyGrabPoint, /// /// Hand facing upwards, Index and Thumb stretched out to start a teleport /// TeleportStart, /// /// Hand facing upwards, Index curled in to finish a teleport /// TeleportEnd, } [Obsolete("Use SimulatedArticulatedHandPoses class or other custom class")] private static readonly Dictionary handPoses = new Dictionary(); /// /// Get pose data for a supported gesture. /// [Obsolete("Use SimulatedArticulatedHandPoses.GetGesturePose() or other custom class")] public static ArticulatedHandPose GetGesturePose(GestureId gesture) { if (handPoses.TryGetValue(gesture, out ArticulatedHandPose pose)) { return pose; } return null; } #if UNITY_EDITOR /// /// Load pose data from files. /// [Obsolete("Use SimulatedArticulatedHandPoses or other custom class")] public static void LoadGesturePoses() { string[] gestureNames = Enum.GetNames(typeof(GestureId)); for (int i = 0; i < gestureNames.Length; ++i) { string gestureFileName = string.Format("ArticulatedHandPose_{0}", gestureNames[i]); string[] gestureGuids = AssetDatabase.FindAssets(gestureFileName); string gesturePath = string.Empty; foreach (string guid in gestureGuids) { string tempPath = AssetDatabase.GUIDToAssetPath(guid); if (tempPath.Contains("InputSimulation") && tempPath.Contains("ArticulatedHandPoses") && tempPath.Contains(gestureFileName + ".json")) { gesturePath = tempPath; break; } } if (!string.IsNullOrWhiteSpace(gesturePath)) { LoadGesturePose((GestureId)i, gesturePath); } } } [Obsolete("Use SimulatedArticulatedHandPoses class or other custom class")] private static ArticulatedHandPose LoadGesturePose(GestureId gesture, string filePath) { if (!string.IsNullOrWhiteSpace(filePath)) { var pose = new ArticulatedHandPose(); pose.FromJson(File.ReadAllText(filePath)); handPoses.Add(gesture, pose); return pose; } return null; } [Obsolete("Use SimulatedArticulatedHandPoses class or other custom class")] public static void ResetGesturePoses() { handPoses.Clear(); } #endif /// Utility class to serialize hand pose as a dictionary with full joint names [Serializable] internal struct ArticulatedHandPoseItem { private static readonly string[] jointNames = Enum.GetNames(typeof(TrackedHandJoint)); public string joint; public MixedRealityPose pose; public TrackedHandJoint JointIndex { get { int nameIndex = Array.FindIndex(jointNames, IsJointName); if (nameIndex < 0) { Debug.LogError($"Joint name {joint} not in TrackedHandJoint enum"); return TrackedHandJoint.None; } return (TrackedHandJoint)nameIndex; } set { joint = jointNames[(int)value]; } } private bool IsJointName(string s) { return s == joint; } public ArticulatedHandPoseItem(TrackedHandJoint joint, MixedRealityPose pose) { this.joint = jointNames[(int)joint]; this.pose = pose; } } /// Utility class to serialize hand pose as a dictionary with full joint names [Serializable] internal class ArticulatedHandPoseDictionary { public ArticulatedHandPoseItem[] items = null; public void FromJointPoses(MixedRealityPose[] jointPoses) { items = new ArticulatedHandPoseItem[JointCount]; for (int i = 0; i < JointCount; ++i) { items[i].JointIndex = (TrackedHandJoint)i; items[i].pose = jointPoses[i]; } } public void ToJointPoses(MixedRealityPose[] jointPoses) { for (int i = 0; i < JointCount; ++i) { jointPoses[i] = MixedRealityPose.ZeroIdentity; } foreach (var item in items) { jointPoses[(int)item.JointIndex] = item.pose; } } } /// /// Serialize pose data to JSON format. /// public string ToJson() { var dict = new ArticulatedHandPoseDictionary(); dict.FromJointPoses(localJointPoses); return JsonUtility.ToJson(dict, true); } /// /// Deserialize pose data from JSON format. /// public void FromJson(string json) { var dict = JsonUtility.FromJson(json); dict.ToJointPoses(localJointPoses); } } }