// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. using Microsoft.MixedReality.Toolkit.Input; using Microsoft.MixedReality.Toolkit.Utilities; using System; using System.Collections.Generic; using Unity.Profiling; using UnityEngine; #if LEAPMOTIONCORE_PRESENT using Leap; using Leap.Unity; using Leap.Unity.Attachments; #endif namespace Microsoft.MixedReality.Toolkit.LeapMotion.Input { [MixedRealityDataProvider( typeof(IMixedRealityInputSystem), SupportedPlatforms.WindowsStandalone | SupportedPlatforms.WindowsEditor, "Leap Motion Device Manager", "LeapMotion/Profiles/LeapMotionDeviceManagerProfile.asset", "MixedRealityToolkit.Providers", true)] /// /// Class that detects the tracking state of leap motion hands. This class will only run if the Leap Motion Core Assets are in the project and the Leap Motion Device /// Manager data provider has been added in the input system configuration profile. /// public class LeapMotionDeviceManager : BaseInputDeviceManager, IMixedRealityCapabilityCheck { /// /// Constructor. /// /// The instance that receives data from this provider. /// Friendly name of the service. /// Service priority. Used to determine order of instantiation. /// The service's configuration profile. public LeapMotionDeviceManager( IMixedRealityInputSystem inputSystem, string name = null, uint priority = DefaultPriority, BaseMixedRealityProfile profile = null) : base(inputSystem, name, priority, profile) { } #region IMixedRealityCapabilityCheck Implementation /// public bool CheckCapability(MixedRealityCapability capability) { // Leap Motion only supports Articulated Hands return (capability == MixedRealityCapability.ArticulatedHand); } #endregion IMixedRealityCapabilityCheck Implementation #if LEAPMOTIONCORE_PRESENT /// /// The profile that contains settings for the Leap Motion Device Manager input data provider. This profile is nested under /// Input > Input Data Providers > Leap Motion Device Manager in the MixedRealityToolkit object in the hierarchy. /// public LeapMotionDeviceManagerProfile SettingsProfile => ConfigurationProfile as LeapMotionDeviceManagerProfile; /// /// The LeapServiceProvider is added to the scene at runtime in OnEnable. /// public LeapServiceProvider LeapMotionServiceProvider { get; protected set; } /// /// The distance between the index finger tip and the thumb tip required to enter the pinch/air tap selection gesture. /// The pinch gesture enter will be registered for all values less than the EnterPinchDistance. The default EnterPinchDistance value is 0.02 and must be between 0.015 and 0.1. /// private float EnterPinchDistance => SettingsProfile.EnterPinchDistance; /// /// The distance between the index finger tip and the thumb tip required to exit the pinch/air tap gesture. /// The pinch gesture exit will be registered for all values greater than the ExitPinchDistance. The default ExitPinchDistance value is 0.05 and must be between 0.015 and 0.1. /// private float ExitPinchDistance => SettingsProfile.ExitPinchDistance; /// /// If true, the leap motion controller is connected and detected. /// private bool IsLeapConnected => LeapMotionServiceProvider.IsConnected(); /// /// The Leap attachment hands, used to determine which hand is currently tracked by leap. /// private AttachmentHands leapAttachmentHands = null; /// /// List of hands that are currently in frame and detected by the leap motion controller. If there are no hands in the current frame, this list will be empty. /// private List CurrentHandsDetectedByLeap => LeapMotionServiceProvider.CurrentFrame.Hands; // This value can only be set in the profile, the default is LeapControllerOrientation.Headset. private LeapControllerOrientation LeapControllerOrientation => SettingsProfile.LeapControllerOrientation; /// /// Adds an offset to the game object with LeapServiceProvider attached. This offset is only applied if the leapControllerOrientation /// is LeapControllerOrientation.Desk and is necessary for the hand to appear in front of the main camera. If the leap controller is on the /// desk, the LeapServiceProvider is added to the scene instead of the LeapXRServiceProvider. The anchor point for the position of the leap hands is /// the position of the game object with the LeapServiceProvider attached. /// private Vector3 LeapHandsOffset => SettingsProfile.LeapControllerOffset; // If the Leap Controller Orientation is the Headset, controller offset settings will be exposed in the inspector while using the LeapXRServiceProvider private LeapVRDeviceOffsetMode LeapVRDeviceOffsetMode => SettingsProfile.LeapVRDeviceOffsetMode; /// /// Dictionary to capture all active leap motion hands detected. /// private readonly Dictionary trackedHands = new Dictionary(); private AttachmentHand leftAttachmentHand = null; private AttachmentHand rightAttachmentHand = null; private static readonly ProfilerMarker UpdatePerfMarker = new ProfilerMarker("[MRTK] LeapMotionDeviceManager.Update"); /// public override void Enable() { base.Enable(); if (LeapControllerOrientation == LeapControllerOrientation.Headset) { // As of the Unity Plugin (>V5.0.0), the leap service provider needs to know what is the main camera, // it will pick this up from the MainCameraProvider. This needs to be done before the LeapXRServiceProvider is created #if LEAPMOTIONPLUGIN_PRESENT MainCameraProvider.mainCamera = CameraCache.Main; #endif // If the leap controller is mounted on a headset then add the LeapXRServiceProvider to the scene LeapXRServiceProvider leapXRServiceProvider; // This is a different type than the LeapMotionServiceProvider property for access to specific settings below. LeapMotionServiceProvider = leapXRServiceProvider = CameraCache.Main.gameObject.AddComponent(); // Allow modification of VR specific offset modes if the leapControllerOrientation is Headset // These settings mirror the modification of the properties exposed in the inspector within the LeapXRServiceProvider attached // to the main camera if (LeapVRDeviceOffsetMode == LeapVRDeviceOffsetMode.ManualHeadOffset) { // Change the offset mode before setting the properties leapXRServiceProvider.deviceOffsetMode = LeapXRServiceProvider.DeviceOffsetMode.ManualHeadOffset; leapXRServiceProvider.deviceOffsetYAxis = SettingsProfile.LeapVRDeviceOffsetY; leapXRServiceProvider.deviceOffsetZAxis = SettingsProfile.LeapVRDeviceOffsetZ; leapXRServiceProvider.deviceTiltXAxis = SettingsProfile.LeapVRDeviceOffsetTiltX; } else if (LeapVRDeviceOffsetMode == LeapVRDeviceOffsetMode.Transform) { if (SettingsProfile.LeapVRDeviceOrigin != null) { leapXRServiceProvider.deviceOffsetMode = LeapXRServiceProvider.DeviceOffsetMode.Transform; leapXRServiceProvider.deviceOrigin = SettingsProfile.LeapVRDeviceOrigin; } else { Debug.LogError("The Leap VR Device Origin Transform was not set in the LeapMotionDeviceManagerProfile and is null."); } } } if (LeapControllerOrientation == LeapControllerOrientation.Desk) { // Create a separate gameobject if the leap controller is on the desk GameObject leapProvider = new GameObject("LeapProvider"); // The LeapServiceProvider does not need to be attached to a camera, but the location of this gameobject is the anchor for the desk hands LeapMotionServiceProvider = leapProvider.AddComponent(); // Follow the transform of the main camera by adding the service provider as a child of the main camera leapProvider.transform.parent = CameraCache.Main.transform; // Apply hand position offset, an offset is required to render the hands in view and in front of the camera LeapMotionServiceProvider.transform.position += LeapHandsOffset; } // Add the attachment hands to the scene for the purpose of getting the tracking state of each hand and joint positions GameObject leapAttachmentHandsGameObject = new GameObject("LeapAttachmentHands"); leapAttachmentHands = leapAttachmentHandsGameObject.AddComponent(); // The first hand in attachmentHands.attachmentHands is always left leftAttachmentHand = leapAttachmentHands.attachmentHands[0]; // The second hand in attachmentHands.attachmentHands is always right rightAttachmentHand = leapAttachmentHands.attachmentHands[1]; // Enable all attachment point flags in the leap hand. By default, only the wrist and the palm are enabled. foreach (TrackedHandJoint joint in Enum.GetValues(typeof(TrackedHandJoint))) { leapAttachmentHands.attachmentPoints |= LeapMotionArticulatedHand.ConvertMRTKJointToLeapJoint(joint); } } /// public override void Disable() { base.Disable(); // Only destroy the objects if the application is playing because the objects are added to the scene at runtime if (Application.isPlaying) { // Destroy AttachmentHands GameObject if (leapAttachmentHands != null) { GameObject.Destroy(leapAttachmentHands.gameObject); } if (LeapMotionServiceProvider != null) { // Destroy the LeapProvider GameObject if the controller orientation is the desk if (LeapControllerOrientation == LeapControllerOrientation.Desk) { GameObject.Destroy(LeapMotionServiceProvider.gameObject); } // Destroy the LeapXRServiceProvider attached to the main camera if the controller orientation is headset else if (LeapControllerOrientation == LeapControllerOrientation.Headset) { GameObject.Destroy(LeapMotionServiceProvider); } } } } /// /// Adds a new LeapMotionArticulatedHand to the scene. /// /// The handedness (Handedness.Left or Handedness.Right) of the hand to be added private void OnHandDetected(Handedness handedness) { // Only create a new hand if the hand does not exist if (!trackedHands.ContainsKey(handedness)) { var pointers = RequestPointers(SupportedControllerType.ArticulatedHand, handedness); var inputSource = CoreServices.InputSystem?.RequestNewGenericInputSource($"Leap {handedness} Controller", pointers, InputSourceType.Hand); var leapHand = new LeapMotionArticulatedHand(TrackingState.Tracked, handedness, inputSource); // Set pinch thresholds leapHand.HandDefinition.EnterPinchDistance = EnterPinchDistance; leapHand.HandDefinition.ExitPinchDistance = ExitPinchDistance; // Set the leap attachment hand to the corresponding handedness if (handedness == Handedness.Left) { leapHand.SetAttachmentHands(leftAttachmentHand, LeapMotionServiceProvider); } else // handedness == Handedness.Right { leapHand.SetAttachmentHands(rightAttachmentHand, LeapMotionServiceProvider); } // Set the pointers for an articulated hand to the leap hand foreach (var pointer in pointers) { pointer.Controller = leapHand; } trackedHands.Add(handedness, leapHand); CoreServices.InputSystem.RaiseSourceDetected(inputSource, leapHand); } } /// /// Removes the LeapMotionArticulated hand from the scene when the tracking is lost. /// /// The handedness (Handedness.Left or Handedness.Right) of the hand to be removed private void OnHandDetectionLost(Handedness handedness) { if (CoreServices.InputSystem != null) { CoreServices.InputSystem.RaiseSourceLost(trackedHands[handedness].InputSource, trackedHands[handedness]); } // Disable the pointers if the hand is not tracking RecyclePointers(trackedHands[handedness].InputSource); // Remove hand from tracked hands trackedHands.Remove(trackedHands[handedness].ControllerHandedness); } /// /// Update the number of tracked leap hands. /// /// The tracking state of the left leap hand /// The tracking state of the right leap hand private void UpdateLeapTrackedHands(bool isLeftTracked, bool isRightTracked) { // Left Hand Update if (isLeftTracked && !trackedHands.ContainsKey(Handedness.Left)) { OnHandDetected(Handedness.Left); } else if (!isLeftTracked && trackedHands.ContainsKey(Handedness.Left)) { OnHandDetectionLost(Handedness.Left); } // Right Hand Update if (isRightTracked && !trackedHands.ContainsKey(Handedness.Right)) { OnHandDetected(Handedness.Right); } else if (!isRightTracked && trackedHands.ContainsKey(Handedness.Right)) { OnHandDetectionLost(Handedness.Right); } } /// public override void Update() { base.Update(); using (UpdatePerfMarker.Auto()) { if (IsLeapConnected) { // if the number of tracked hands in frame has changed if (CurrentHandsDetectedByLeap.Count != trackedHands.Count) { UpdateLeapTrackedHands(leftAttachmentHand.isTracked, rightAttachmentHand.isTracked); } // Update the hand/hands that are in trackedhands foreach (KeyValuePair hand in trackedHands) { hand.Value.UpdateState(); } } } } #endif } }