// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. using Microsoft.MixedReality.Toolkit.Physics; using Microsoft.MixedReality.Toolkit.Utilities; using System; using Unity.Profiling; using UnityEngine; using UnityPhysics = UnityEngine.Physics; namespace Microsoft.MixedReality.Toolkit.Input { /// /// This class provides Gaze as an Input Source so users can interact with objects using their head. /// [DisallowMultipleComponent] [AddComponentMenu("Scripts/MRTK/Services/GazeProvider")] public class GazeProvider : InputSystemGlobalHandlerListener, IMixedRealityGazeProviderHeadOverride, IMixedRealityEyeGazeProvider, IMixedRealityInputHandler { private const float VelocityThreshold = 0.1f; private const float MovementThreshold = 0.01f; /// /// Used on Gaze Pointer initialization. To make the object lock/not lock when focus locked during runtime, use the IsTargetPositionLockedOnFocusLock /// attribute of /// [SerializeField] [Tooltip("If true, initializes the gaze cursor to stay locked on the object when the cursor's focus is locked, otherwise it will continue following the head's direction")] internal bool lockCursorWhenFocusLocked = true; [SerializeField] [Tooltip("If true, the gaze cursor will disappear when the pointer's focus is locked, to prevent the cursor from floating idly in the world.")] internal bool setCursorInvisibleWhenFocusLocked = false; [SerializeField] [Tooltip("Maximum distance at which the gaze can hit a GameObject.")] internal float maxGazeCollisionDistance = 10.0f; /// /// The LayerMasks, in prioritized order, that are used to determine the GazeTarget when raycasting. /// /// Allow the cursor to hit SR, but first prioritize any DefaultRaycastLayers (potentially behind SR) /// /// /// [SerializeField] [Tooltip("The LayerMasks, in prioritized order, that are used to determine the GazeTarget when raycasting.")] internal LayerMask[] raycastLayerMasks = { UnityPhysics.DefaultRaycastLayers }; /// /// Current stabilization method, used to smooth out the gaze ray data. /// If left null, no stabilization will be performed. /// [SerializeField] [Tooltip("Stabilizer, if any, used to smooth out the gaze ray data.")] internal GazeStabilizer stabilizer = null; /// /// Transform that should be used as the source of the gaze position and rotation. /// Defaults to the main camera. /// [SerializeField] [Tooltip("Transform that should be used to represent the gaze position and rotation. Defaults to CameraCache.Main")] internal Transform gazeTransform = null; [SerializeField] [Range(0.01f, 1f)] [Tooltip("Minimum head velocity threshold")] private float minHeadVelocityThreshold = 0.5f; [SerializeField] [Range(0.1f, 5f)] [Tooltip("Maximum head velocity threshold")] private float maxHeadVelocityThreshold = 2f; #region IMixedRealityGazeProvider Implementation /// public bool Enabled { get { return enabled; } set { enabled = value; } } /// public virtual IMixedRealityInputSource GazeInputSource { get { if (gazeInputSource == null) { gazeInputSource = new BaseGenericInputSource("Gaze", sourceType: InputSourceType.Head); gazePointer.SetGazeInputSourceParent(gazeInputSource); } return gazeInputSource; } } internal BaseGenericInputSource gazeInputSource; /// public virtual IMixedRealityPointer GazePointer => gazePointer ?? InitializeGazePointer(); private InternalGazePointer gazePointer = null; /// public GameObject GazeCursorPrefab { internal get; set; } /// public IMixedRealityCursor GazeCursor => GazePointer.BaseCursor; /// public GameObject GazeTarget { get; private set; } /// public MixedRealityRaycastHit HitInfo { get; private set; } /// public Vector3 HitPosition { get; private set; } /// public Vector3 HitNormal { get; private set; } /// public Vector3 GazeOrigin => GazePointer != null ? GazePointer.Rays[0].Origin : Vector3.zero; /// public Vector3 GazeDirection => GazePointer != null ? GazePointer.Rays[0].Direction : Vector3.forward; /// public Vector3 HeadVelocity { get; private set; } /// public Vector3 HeadMovementDirection { get; private set; } /// public GameObject GameObjectReference => gameObject; #endregion IMixedRealityGazeProvider Implementation private float lastHitDistance = 2.0f; private bool delayInitialization = true; private Vector3 lastHeadPosition = Vector3.zero; public Vector3? overrideHeadPosition { get; private set; } public Vector3? overrideHeadForward { get; private set; } #region InternalGazePointer Class private class InternalGazePointer : GenericPointer { private readonly Transform gazeTransform; private readonly BaseRayStabilizer stabilizer; private readonly GazeProvider gazeProvider; public InternalGazePointer(GazeProvider gazeProvider, string pointerName, IMixedRealityInputSource inputSourceParent, LayerMask[] raycastLayerMasks, float pointerExtent, Transform gazeTransform, BaseRayStabilizer stabilizer) : base(pointerName, inputSourceParent) { this.gazeProvider = gazeProvider; PrioritizedLayerMasksOverride = raycastLayerMasks; this.pointerExtent = pointerExtent; this.gazeTransform = gazeTransform; this.stabilizer = stabilizer; IsInteractionEnabled = true; } #region IMixedRealityPointer Implementation /// public override IMixedRealityController Controller { get; set; } /// public override IMixedRealityInputSource InputSourceParent { get; protected set; } private float pointerExtent; /// public override float PointerExtent { get => pointerExtent; set => pointerExtent = value; } // Is the pointer currently down private bool isDown = false; // Input source that raised pointer down private IMixedRealityInputSource currentInputSource; // Handedness of the input source that raised pointer down private Handedness currentHandedness = Handedness.None; /// /// Only for use when initializing Gaze Pointer on startup. /// internal void SetGazeInputSourceParent(IMixedRealityInputSource gazeInputSource) { InputSourceParent = gazeInputSource; } private static readonly ProfilerMarker OnPreSceneQueryPerfMarker = new ProfilerMarker("[MRTK] InternalGazePointer.OnPreSceneQuery"); /// /// On pre-scene query, the gaze pointer will set up it's raycast ray to use either the eye gaze ray or the head gaze ray, depending on IsEyeTrackingEnabledAndValid public override void OnPreSceneQuery() { using (OnPreSceneQueryPerfMarker.Auto()) { Vector3 newGazeOrigin; Vector3 newGazeNormal; if (gazeProvider.IsEyeTrackingEnabledAndValid) { gazeProvider.gazeInputSource.SourceType = InputSourceType.Eyes; newGazeOrigin = gazeProvider.LatestEyeGaze.origin; newGazeNormal = gazeProvider.LatestEyeGaze.direction; } else { gazeProvider.gazeInputSource.SourceType = InputSourceType.Head; if (gazeProvider.UseHeadGazeOverride && gazeProvider.overrideHeadPosition.HasValue && gazeProvider.overrideHeadForward.HasValue) { newGazeOrigin = gazeProvider.overrideHeadPosition.Value; newGazeNormal = gazeProvider.overrideHeadForward.Value; // Reset values in case the override source is removed gazeProvider.overrideHeadPosition = null; gazeProvider.overrideHeadForward = null; } else { newGazeOrigin = gazeTransform.position; newGazeNormal = gazeTransform.forward; } // Update gaze info from stabilizer if (stabilizer != null) { stabilizer.UpdateStability(MixedRealityPlayspace.InverseTransformPoint(newGazeOrigin), MixedRealityPlayspace.InverseTransformDirection(newGazeNormal)); newGazeOrigin = MixedRealityPlayspace.TransformPoint(stabilizer.StablePosition); newGazeNormal = MixedRealityPlayspace.TransformDirection(stabilizer.StableRay.direction); } } Vector3 endPoint = newGazeOrigin + (newGazeNormal * pointerExtent); Rays[0].UpdateRayStep(ref newGazeOrigin, ref endPoint); } } private static readonly ProfilerMarker OnPostSceneQueryPerfMarker = new ProfilerMarker("[MRTK] InternalGazePointer.OnPostSceneQuery"); public override void OnPostSceneQuery() { using (OnPostSceneQueryPerfMarker.Auto()) { if (isDown) { CoreServices.InputSystem.RaisePointerDragged(this, MixedRealityInputAction.None, currentHandedness, currentInputSource); } } } /// public override void OnPreCurrentPointerTargetChange() { } /// public override Vector3 Position => gazeTransform.position; /// public override Quaternion Rotation => gazeTransform.rotation; /// public override void Reset() { Controller = null; } #endregion IMixedRealityPointer Implementation /// /// Press this pointer. This sends a pointer down event across the input system. /// /// The input action that corresponds to the pressed button or axis. /// Optional handedness of the source that pressed the pointer. public void RaisePointerDown(MixedRealityInputAction mixedRealityInputAction, Handedness handedness = Handedness.None, IMixedRealityInputSource inputSource = null) { isDown = true; currentHandedness = handedness; currentInputSource = inputSource; CoreServices.InputSystem?.RaisePointerDown(this, mixedRealityInputAction, handedness, inputSource); } /// /// Release this pointer. This sends pointer clicked and pointer up events across the input system. /// /// The input action that corresponds to the released button or axis. /// Optional handedness of the source that released the pointer. public void RaisePointerUp(MixedRealityInputAction mixedRealityInputAction, Handedness handedness = Handedness.None, IMixedRealityInputSource inputSource = null) { isDown = false; currentHandedness = Handedness.None; currentInputSource = null; CoreServices.InputSystem?.RaisePointerClicked(this, mixedRealityInputAction, 0, handedness, inputSource); CoreServices.InputSystem?.RaisePointerUp(this, mixedRealityInputAction, handedness, inputSource); } } #endregion InternalGazePointer Class #region MonoBehaviour Implementation private void OnValidate() { if (minHeadVelocityThreshold > maxHeadVelocityThreshold) { Debug.LogWarning("Minimum head velocity threshold should be less than the maximum velocity threshold. Changing now."); minHeadVelocityThreshold = maxHeadVelocityThreshold; } } /// protected override void OnEnable() { base.OnEnable(); if (!delayInitialization) { // The first time we call OnEnable we skip this. RaiseSourceDetected(); } } /// protected override async void Start() { base.Start(); await EnsureInputSystemValid(); if (this.IsNull()) { // We've been destroyed during the await. return; } GazePointer.BaseCursor?.SetVisibility(true); if (delayInitialization) { delayInitialization = false; RaiseSourceDetected(); } } private static readonly ProfilerMarker UpdatePerfMarker = new ProfilerMarker("[MRTK] GazeProvider.Update"); private void Update() { using (UpdatePerfMarker.Auto()) { if (MixedRealityRaycaster.DebugEnabled && gazeTransform != null) { Debug.DrawRay(GazeOrigin, (HitPosition - GazeOrigin), Color.white); } // If flagged to do so (setCursorInvisibleWhenFocusLocked) and active (IsInteractionEnabled), set the visibility to !IsFocusLocked, // but don't touch the visibility when not active or not flagged. if (setCursorInvisibleWhenFocusLocked && gazePointer != null && gazePointer.IsInteractionEnabled && GazeCursor != null && gazePointer.IsFocusLocked == GazeCursor.IsVisible) { GazeCursor.SetVisibility(!gazePointer.IsFocusLocked); } // Handle toggling the input source's SourceType based on the current eyetracking mode if (IsEyeTrackingEnabledAndValid) { gazeInputSource.SourceType = InputSourceType.Eyes; } else { gazeInputSource.SourceType = InputSourceType.Head; } } } private static readonly ProfilerMarker LateUpdatePerfMarker = new ProfilerMarker("[MRTK] GazeProvider.LateUpdate"); private void LateUpdate() { using (LateUpdatePerfMarker.Auto()) { // Update head velocity. Vector3 headPosition = GazeOrigin; Vector3 headDelta = headPosition - lastHeadPosition; if (headDelta.sqrMagnitude < MovementThreshold * MovementThreshold) { headDelta = Vector3.zero; } if (Time.fixedDeltaTime > 0) { float velocityAdjustmentRate = 3f * Time.fixedDeltaTime; HeadVelocity = HeadVelocity * (1f - velocityAdjustmentRate) + headDelta * velocityAdjustmentRate / Time.fixedDeltaTime; if (HeadVelocity.sqrMagnitude < VelocityThreshold * VelocityThreshold) { HeadVelocity = Vector3.zero; } } // Update Head Movement Direction float multiplier = Mathf.Clamp01(Mathf.InverseLerp(minHeadVelocityThreshold, maxHeadVelocityThreshold, HeadVelocity.magnitude)); Vector3 newHeadMoveDirection = Vector3.Lerp(headPosition, HeadVelocity, multiplier).normalized; lastHeadPosition = headPosition; float directionAdjustmentRate = Mathf.Clamp01(5f * Time.fixedDeltaTime); HeadMovementDirection = Vector3.Slerp(HeadMovementDirection, newHeadMoveDirection, directionAdjustmentRate); if (MixedRealityRaycaster.DebugEnabled && gazeTransform != null) { Debug.DrawLine(lastHeadPosition, lastHeadPosition + HeadMovementDirection * 10f, Color.Lerp(Color.red, Color.green, multiplier)); Debug.DrawLine(lastHeadPosition, lastHeadPosition + HeadVelocity, Color.yellow); } } } /// protected override void OnDisable() { base.OnDisable(); GazePointer?.BaseCursor?.SetVisibility(false); // if true, component has never started and never fired onSourceDetected event if (!delayInitialization) { CoreServices.InputSystem?.RaiseSourceLost(GazeInputSource); } } /// private void OnDestroy() { // Because GazeCursor is not derived from UnityEngine.Object, we need to manually perform null check against Unity's null if (GazeCursor.TryGetMonoBehaviour(out MonoBehaviour gazeCursor)) { Destroy(gazeCursor.gameObject); } } #endregion MonoBehaviour Implementation #region InputSystemGlobalHandlerListener Implementation /// protected override void RegisterHandlers() { CoreServices.InputSystem?.RegisterHandler(this); } /// protected override void UnregisterHandlers() { CoreServices.InputSystem?.UnregisterHandler(this); } #endregion InputSystemGlobalHandlerListener Implementation #region IMixedRealityInputHandler Implementation public virtual void OnInputUp(InputEventData eventData) { for (int i = 0; i < eventData.InputSource.Pointers.Length; i++) { if (eventData.InputSource.Pointers[i].PointerId == GazePointer.PointerId) { gazePointer.RaisePointerUp(eventData.MixedRealityInputAction, eventData.Handedness, eventData.InputSource); return; } } } public virtual void OnInputDown(InputEventData eventData) { for (int i = 0; i < eventData.InputSource.Pointers.Length; i++) { if (eventData.InputSource.Pointers[i].PointerId == GazePointer.PointerId) { gazePointer.RaisePointerDown(eventData.MixedRealityInputAction, eventData.Handedness, eventData.InputSource); return; } } } #endregion IMixedRealityInputHandler Implementation #region Utilities private static readonly ProfilerMarker InitializeGazePointerPerfMarker = new ProfilerMarker("[MRTK] GazeProvider.InitializeGazePointer"); internal virtual IMixedRealityPointer InitializeGazePointer() { using (InitializeGazePointerPerfMarker.Auto()) { if (gazeTransform == null) { gazeTransform = CameraCache.Main.transform; } Debug.Assert(gazeTransform != null, "No gaze transform to raycast from!"); gazePointer = new InternalGazePointer(this, "Gaze Pointer", null, raycastLayerMasks, maxGazeCollisionDistance, gazeTransform, stabilizer); if ((GazeCursor == null) && (GazeCursorPrefab != null)) { GameObject cursor = Instantiate(GazeCursorPrefab); MixedRealityPlayspace.AddChild(cursor.transform); SetGazeCursor(cursor); } gazePointer.IsTargetPositionLockedOnFocusLock = lockCursorWhenFocusLocked; return gazePointer; } } private static readonly ProfilerMarker RaiseSourceDetectedPerfMarker = new ProfilerMarker("[MRTK] GazeProvider.RaiseSourceDetected"); private async void RaiseSourceDetected() { using (RaiseSourceDetectedPerfMarker.Auto()) { await EnsureInputSystemValid(); if (this.IsNull()) { // We've been destroyed during the await. return; } CoreServices.InputSystem?.RaiseSourceDetected(GazeInputSource); GazePointer.BaseCursor?.SetVisibility(true); } } /// public void UpdateGazeInfoFromHit(MixedRealityRaycastHit raycastHit) { HitInfo = raycastHit; if (IsEyeTrackingEnabledAndValid) { UpdateEyeGaze(null, GazePointer.Rays[0], DateTime.UtcNow); } if (raycastHit.transform != null) { GazeTarget = raycastHit.transform.gameObject; var ray = GazePointer.Rays[0]; var lhd = (raycastHit.point - ray.Origin).magnitude; lastHitDistance = lhd; HitPosition = ray.Origin + lhd * ray.Direction; HitNormal = raycastHit.normal; } else { GazeTarget = null; HitPosition = Vector3.zero; HitNormal = Vector3.zero; } } /// /// Set the gaze cursor. /// public void SetGazeCursor(GameObject cursor) { Debug.Assert(cursor != null); cursor.transform.parent = transform.parent; GazePointer.BaseCursor = cursor.GetComponent(); Debug.Assert(GazePointer.BaseCursor != null, "Failed to load cursor"); GazePointer.BaseCursor.SetVisibilityOnSourceDetected = false; GazePointer.BaseCursor.Pointer = GazePointer; } #endregion Utilities #region IMixedRealityEyeGazeProvider Implementation private DateTime latestEyeTrackingUpdate = DateTime.MinValue; private static readonly float maxEyeTrackingTimeoutInSeconds = 2.0f; /// public bool IsEyeTrackingEnabledAndValid => IsEyeTrackingDataValid && IsEyeTrackingEnabled; /// public bool IsEyeTrackingDataValid => (DateTime.UtcNow - latestEyeTrackingUpdate).TotalSeconds <= maxEyeTrackingTimeoutInSeconds; /// public bool? IsEyeCalibrationValid { get; private set; } = null; /// public Ray LatestEyeGaze { get; private set; } = default(Ray); /// public bool IsEyeTrackingEnabled { get; set; } /// public DateTime Timestamp { get; private set; } /// public void UpdateEyeGaze(IMixedRealityEyeGazeDataProvider provider, Ray eyeRay, DateTime timestamp) { LatestEyeGaze = eyeRay; latestEyeTrackingUpdate = DateTime.UtcNow; Timestamp = timestamp; } /// public void UpdateEyeTrackingStatus(IMixedRealityEyeGazeDataProvider provider, bool userIsEyeCalibrated) { IsEyeCalibrationValid = userIsEyeCalibrated; } #endregion IMixedRealityEyeGazeProvider Implementation #region IMixedRealityGazeProviderHeadOverride Implementation /// public bool UseHeadGazeOverride { get; set; } /// public void OverrideHeadGaze(Vector3 position, Vector3 forward) { overrideHeadPosition = position; overrideHeadForward = forward; } #endregion IMixedRealityGazeProviderHeadOverride Implementation } }