// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. using Microsoft.MixedReality.Toolkit.Utilities; using System; using UnityEngine; namespace Microsoft.MixedReality.Toolkit.Input { /// /// Utility struct that provides mouse delta in pixels (screen space), normalized viewport coordinates, and world units. /// public class MouseDelta { public Vector3 screenDelta = Vector3.zero; public Vector3 viewportDelta = Vector3.zero; public Vector3 worldDelta = Vector3.zero; /// /// Resets all vector contents to zero vector values /// public void Reset() { screenDelta = Vector3.zero; viewportDelta = Vector3.zero; worldDelta = Vector3.zero; } } /// /// Service that provides simulated mixed reality input information based on mouse and keyboard input in editor /// [MixedRealityDataProvider( typeof(IMixedRealityInputSystem), SupportedPlatforms.WindowsEditor | SupportedPlatforms.MacEditor | SupportedPlatforms.LinuxEditor, "Input Simulation Service", "Profiles/DefaultMixedRealityInputSimulationProfile.asset", "MixedRealityToolkit.SDK", true)] [HelpURL("https://docs.microsoft.com/windows/mixed-reality/mrtk-unity/features/input-simulation/input-simulation-service")] public class InputSimulationService : BaseInputSimulationService, IInputSimulationService, IMixedRealityEyeGazeDataProvider, IMixedRealityCapabilityCheck { private ManualCameraControl cameraControl = null; private SimulatedControllerDataProvider dataProvider = null; /// public ControllerSimulationMode ControllerSimulationMode { get; set; } /// public SimulatedHandData HandDataLeft { get; } = new SimulatedHandData(); /// public SimulatedHandData HandDataRight { get; } = new SimulatedHandData(); /// private SimulatedHandData HandDataGaze { get; } = new SimulatedHandData(); /// public SimulatedMotionControllerData MotionControllerDataLeft { get; } = new SimulatedMotionControllerData(); /// public SimulatedMotionControllerData MotionControllerDataRight { get; } = new SimulatedMotionControllerData(); /// public bool IsSimulatingControllerLeft => dataProvider != null && dataProvider.IsSimulatingLeft; /// public bool IsSimulatingControllerRight => dataProvider != null && dataProvider.IsSimulatingRight; /// public bool IsAlwaysVisibleControllerLeft { get { return dataProvider != null && dataProvider.IsAlwaysVisibleLeft; } set { if (dataProvider != null) { dataProvider.IsAlwaysVisibleLeft = value; } } } /// public bool IsAlwaysVisibleControllerRight { get { return dataProvider != null && dataProvider.IsAlwaysVisibleRight; } set { if (dataProvider != null) { dataProvider.IsAlwaysVisibleRight = value; } } } /// public Vector3 ControllerPositionLeft { get { return dataProvider != null ? dataProvider.InputStateLeft.ViewportPosition : Vector3.zero; } set { if (dataProvider != null) { dataProvider.InputStateLeft.ViewportPosition = value; } } } /// public Vector3 ControllerPositionRight { get { return dataProvider != null ? dataProvider.InputStateRight.ViewportPosition : Vector3.zero; } set { if (dataProvider != null) { dataProvider.InputStateRight.ViewportPosition = value; } } } /// public Vector3 ControllerRotationLeft { get { return dataProvider != null ? dataProvider.InputStateLeft.ViewportRotation : Vector3.zero; } set { if (dataProvider != null) { dataProvider.InputStateLeft.ViewportRotation = value; } } } /// public Vector3 ControllerRotationRight { get { return dataProvider != null ? dataProvider.InputStateRight.ViewportRotation : Vector3.zero; } set { if (dataProvider != null) { dataProvider.InputStateRight.ViewportRotation = value; } } } /// public void ResetControllerLeft() { if (dataProvider != null) { dataProvider.ResetInput(Handedness.Left); } } /// public void ResetControllerRight() { if (dataProvider != null) { dataProvider.ResetInput(Handedness.Right); } } /// /// If true then camera forward direction is used to simulate eye tracking data. /// [Obsolete("Check the EyeGazeSimulationMode instead")] public bool SimulateEyePosition { get { return EyeGazeSimulationMode != EyeGazeSimulationMode.Disabled; } set { EyeGazeSimulationMode = value ? EyeGazeSimulationMode.CameraForwardAxis : EyeGazeSimulationMode.Disabled; } } /// public EyeGazeSimulationMode EyeGazeSimulationMode { get; set; } /// /// If true then keyboard and mouse input are used to simulate controllers. /// public bool UserInputEnabled { get; set; } = true; /// /// Timestamp of the last controller device update /// private long lastControllerUpdateTimestamp = 0; /// /// Indicators to show input simulation state in the viewport. /// private GameObject indicators; /// /// Tracks mouse movement delta information in different coordinate system spaces between updates /// private MouseDelta mouseDelta = new MouseDelta(); private Vector3 lastMousePosition; private bool wasFocused; private bool wasCursorLocked; #region BaseInputDeviceManager Implementation /// /// Constructor. /// /// The instance that loaded the data provider. /// The instance that receives data from this provider. /// Friendly name of the service. /// Service priority. Used to determine order of instantiation. /// The service's configuration profile. [System.Obsolete("This constructor is obsolete (registrar parameter is no longer required) and will be removed in a future version of the Microsoft Mixed Reality Toolkit.")] public InputSimulationService( IMixedRealityServiceRegistrar registrar, IMixedRealityInputSystem inputSystem, string name, uint priority, BaseMixedRealityProfile profile) : this(inputSystem, name, priority, profile) { Registrar = registrar; } /// /// Constructor. /// /// The instance that receives data from this provider. /// Friendly name of the service. /// Service priority. Used to determine order of instantiation. /// The service's configuration profile. public InputSimulationService( IMixedRealityInputSystem inputSystem, string name, uint priority, BaseMixedRealityProfile profile) : base(inputSystem, name, priority, profile) { } /// public bool CheckCapability(MixedRealityCapability capability) { switch (capability) { case MixedRealityCapability.ArticulatedHand: return (ControllerSimulationMode == ControllerSimulationMode.ArticulatedHand); case MixedRealityCapability.GGVHand: // If any hand simulation is enabled, GGV interactions are supported. return (ControllerSimulationMode != ControllerSimulationMode.Disabled); case MixedRealityCapability.EyeTracking: return EyeGazeSimulationMode != EyeGazeSimulationMode.Disabled; case MixedRealityCapability.MotionController: return ControllerSimulationMode == ControllerSimulationMode.MotionController; } return false; } /// public override void Initialize() { base.Initialize(); ControllerSimulationMode = InputSimulationProfile.DefaultControllerSimulationMode; EyeGazeSimulationMode = InputSimulationProfile.DefaultEyeGazeSimulationMode; } /// public override void Destroy() { base.Destroy(); } /// public override void Enable() { base.Enable(); var profile = InputSimulationProfile; if (indicators == null && profile.IndicatorsPrefab) { indicators = GameObject.Instantiate(profile.IndicatorsPrefab); } ResetMouseDelta(); } /// public override void Disable() { base.Disable(); if (indicators) { UnityEngine.Object.Destroy(indicators); } DisableCameraControl(); DisableControllerSimulation(); } /// public override void Update() { base.Update(); var profile = InputSimulationProfile; switch (ControllerSimulationMode) { case ControllerSimulationMode.Disabled: DisableControllerSimulation(); break; case ControllerSimulationMode.ArticulatedHand: case ControllerSimulationMode.HandGestures: EnableHandSimulation(); break; case ControllerSimulationMode.MotionController: EnableMotionControllerSimulation(); break; } // If an XRDevice is present, the user will not be able to control the camera // as it is controlled by the device. We therefore disable camera controls in // this case. // This was causing issues while simulating in editor for VR, as the UpDown // camera movement is mapped to controller AXIS_3, which happens to be the // select trigger for WMR controllers. if (profile.IsCameraControlEnabled && !DeviceUtility.IsPresent) { EnableCameraControl(); } else { DisableCameraControl(); } UpdateMouseDelta(); if (UserInputEnabled) { if (dataProvider != null) { if (dataProvider is SimulatedHandDataProvider handDataProvider) { handDataProvider.UpdateHandData(HandDataLeft, HandDataRight, HandDataGaze, mouseDelta); } else if (dataProvider is SimulatedMotionControllerDataProvider controllerDataProvider) { controllerDataProvider.UpdateControllerData(MotionControllerDataLeft, MotionControllerDataRight, mouseDelta); } } if (cameraControl != null && CameraCache.Main != null) { cameraControl.UpdateTransform(CameraCache.Main.transform, mouseDelta); } } switch (EyeGazeSimulationMode) { case EyeGazeSimulationMode.Disabled: break; case EyeGazeSimulationMode.CameraForwardAxis: // In the simulated eye gaze condition, let's set the eye tracking calibration status automatically to true Service?.EyeGazeProvider?.UpdateEyeTrackingStatus(this, true); Service?.EyeGazeProvider?.UpdateEyeGaze(this, new Ray(CameraCache.Main.transform.position, CameraCache.Main.transform.forward), DateTime.UtcNow); break; case EyeGazeSimulationMode.Mouse: // In the simulated eye gaze condition, let's set the eye tracking calibration status automatically to true Service?.EyeGazeProvider?.UpdateEyeTrackingStatus(this, true); Service?.EyeGazeProvider?.UpdateEyeGaze(this, CameraCache.Main.ScreenPointToRay(UnityEngine.Input.mousePosition), DateTime.UtcNow); break; } } /// public override void LateUpdate() { base.LateUpdate(); var profile = InputSimulationProfile; // Apply hand data in LateUpdate to ensure external changes are applied. // HandDataLeft/Right can be modified after the services Update() call. if (ControllerSimulationMode == ControllerSimulationMode.Disabled) { RemoveAllControllerDevices(); } else { DateTime currentTime = DateTime.UtcNow; double msSinceLastControllerUpdate = currentTime.Subtract(new DateTime(lastControllerUpdateTimestamp)).TotalMilliseconds; // TODO implement custom hand device update frequency here, use 1000/fps instead of 0 if (msSinceLastControllerUpdate > 0) { object controllerDataLeft = null; object controllerDataRight = null; switch (ControllerSimulationMode) { case ControllerSimulationMode.ArticulatedHand: case ControllerSimulationMode.HandGestures: controllerDataLeft = HandDataLeft; controllerDataRight = HandDataRight; break; case ControllerSimulationMode.MotionController: controllerDataLeft = MotionControllerDataLeft; controllerDataRight = MotionControllerDataRight; break; } UpdateControllerDevice(ControllerSimulationMode, Handedness.Left, controllerDataLeft); UpdateControllerDevice(ControllerSimulationMode, Handedness.Right, controllerDataRight); // HandDataGaze is only enabled if the user is simulating via mouse and keyboard if (UserInputEnabled && profile.IsHandsFreeInputEnabled) UpdateControllerDevice(ControllerSimulationMode.HandGestures, Handedness.None, HandDataGaze); lastControllerUpdateTimestamp = currentTime.Ticks; } } } #endregion BaseInputDeviceManager Implementation private MixedRealityInputSimulationProfile inputSimulationProfile = null; /// public MixedRealityInputSimulationProfile InputSimulationProfile { get { if (inputSimulationProfile == null) { inputSimulationProfile = ConfigurationProfile as MixedRealityInputSimulationProfile; } return inputSimulationProfile; } set { inputSimulationProfile = value; } } /// IMixedRealityEyeSaccadeProvider IMixedRealityEyeGazeDataProvider.SaccadeProvider => null; /// bool IMixedRealityEyeGazeDataProvider.SmoothEyeTracking { get; set; } private void EnableCameraControl() { if (cameraControl == null) { cameraControl = new ManualCameraControl(InputSimulationProfile); if (CameraCache.Main != null) { cameraControl.SetInitialTransform(CameraCache.Main.transform); } } } private void DisableCameraControl() { if (cameraControl != null) { cameraControl = null; } } private void EnableHandSimulation() { if (dataProvider == null) { DebugUtilities.LogVerbose("Creating a new hand simulation data provider"); dataProvider = new SimulatedHandDataProvider(InputSimulationProfile); } else if (dataProvider is SimulatedMotionControllerDataProvider) { DebugUtilities.LogVerbose("Replacing motion controller simulation data provider with hand simulation data provider"); RemoveAllControllerDevices(); dataProvider = new SimulatedHandDataProvider(InputSimulationProfile); } } private void EnableMotionControllerSimulation() { if (dataProvider == null) { DebugUtilities.LogVerbose("Creating a new motion controller simulation data provider"); dataProvider = new SimulatedMotionControllerDataProvider(InputSimulationProfile); } else if (dataProvider is SimulatedHandDataProvider) { DebugUtilities.LogVerbose("Replacing hand simulation data provider with motion controller simulation data provider"); RemoveAllControllerDevices(); dataProvider = new SimulatedMotionControllerDataProvider(InputSimulationProfile); } } private void DisableControllerSimulation() { RemoveAllControllerDevices(); if (dataProvider != null) { DebugUtilities.LogVerbose("Destroying the controller simulation data provider"); dataProvider = null; } } private void ResetMouseDelta() { lastMousePosition = UnityEngine.Input.mousePosition; mouseDelta.Reset(); } private void UpdateMouseDelta() { var profile = InputSimulationProfile; bool isFocused = Application.isFocused; bool gainedFocus = !wasFocused && isFocused; wasFocused = isFocused; bool isCursorLocked = UnityEngine.Cursor.lockState != CursorLockMode.None; bool cursorLockChanged = wasCursorLocked != isCursorLocked; wasCursorLocked = isCursorLocked; // Reset in cases where mouse position is jumping if (gainedFocus || cursorLockChanged) { ResetMouseDelta(); } else { Vector3 screenDelta; Vector3 worldDelta; if (UnityEngine.Cursor.lockState == CursorLockMode.Locked) { screenDelta.x = UnityEngine.Input.GetAxis(profile.MouseX); screenDelta.y = UnityEngine.Input.GetAxis(profile.MouseY); worldDelta.z = UnityEngine.Input.GetAxis(profile.MouseScroll); } else { // Use frame-to-frame mouse delta in pixels to determine mouse rotation. // The traditional GetAxis("Mouse X") method doesn't work under Remote Desktop. screenDelta.x = (UnityEngine.Input.mousePosition.x - lastMousePosition.x); screenDelta.y = (UnityEngine.Input.mousePosition.y - lastMousePosition.y); worldDelta.z = UnityEngine.Input.mouseScrollDelta.y; } // Interpret scroll values as world space delta worldDelta.z *= profile.ControllerDepthMultiplier; Vector2 worldDepthDelta = new Vector2(worldDelta.z, 0); // Convert world space scroll delta into screen space pixels screenDelta.z = WorldToScreen(worldDepthDelta).x; // Convert screen space x/y delta into world space Vector2 worldDelta2D = ScreenToWorld(screenDelta); worldDelta.x = worldDelta2D.x; worldDelta.y = worldDelta2D.y; // Viewport delta x and y can be computed from screen x/y. // Note that the conversion functions do not change Z, it is expected to always be in world space units. Vector3 viewportDelta = CameraCache.Main.ScreenToViewportPoint(screenDelta); // Compute viewport-scale z delta viewportDelta.z = WorldToViewport(worldDepthDelta).x; lastMousePosition = UnityEngine.Input.mousePosition; mouseDelta.screenDelta = screenDelta; mouseDelta.worldDelta = worldDelta; mouseDelta.viewportDelta = viewportDelta; } } // Default world-space distance for converting screen/viewport scroll offsets into world space depth offset. // The pixel-to-world-unit ratio changes with depth, so have to chose a fixed distance for conversion. // Center of the viewport is at (0.5, 0.5) private const float mouseWorldDepth = 0.5f; private Vector2 ScreenToWorld(Vector3 screenDelta) { Vector3 deltaViewport3D = new Vector3( screenDelta.x / (0.5f * CameraCache.Main.pixelWidth), screenDelta.y / (0.5f * CameraCache.Main.pixelHeight), 1) * mouseWorldDepth; var invProjMat = Matrix4x4.Inverse(CameraCache.Main.projectionMatrix); Vector3 deltaWorld3D = invProjMat * deltaViewport3D; return new Vector2(deltaWorld3D.x, deltaWorld3D.y); } private Vector2 WorldToScreen(Vector2 deltaWorld) { Vector3 deltaWorld3D = new Vector3(deltaWorld.x, deltaWorld.y, mouseWorldDepth); Vector4 proj = CameraCache.Main.projectionMatrix * deltaWorld3D; Vector3 deltaViewport3D = -proj / proj.w; return new Vector2( deltaViewport3D.x * CameraCache.Main.pixelWidth, deltaViewport3D.y * CameraCache.Main.pixelHeight); } private Vector2 WorldToViewport(Vector2 deltaWorld) { Vector3 deltaWorld3D = new Vector3(deltaWorld.x, deltaWorld.y, mouseWorldDepth); Vector4 proj = CameraCache.Main.projectionMatrix * deltaWorld3D; Vector3 deltaViewport3D = -proj / proj.w; return new Vector2(deltaViewport3D.x, deltaViewport3D.y); } #region Obsolete Properties and Methods /// [Obsolete("Use ControllerSimulationMode instead.")] public HandSimulationMode HandSimulationMode { get => (HandSimulationMode)ControllerSimulationMode; set { ControllerSimulationMode = (ControllerSimulationMode)value; } } /// [Obsolete("Use IsSimulatingControllerLeft instead.")] public bool IsSimulatingHandLeft => IsSimulatingControllerLeft; /// [Obsolete("Use IsSimulatingControllerRight instead.")] public bool IsSimulatingHandRight => IsSimulatingControllerRight; /// [Obsolete("Use IsAlwaysVisibleControllerLeft instead.")] public bool IsAlwaysVisibleHandLeft { get => IsAlwaysVisibleControllerLeft; set { IsAlwaysVisibleControllerLeft = value; } } /// [Obsolete("Use IsAlwaysVisibleControllerRight instead.")] public bool IsAlwaysVisibleHandRight { get => IsAlwaysVisibleControllerRight; set { IsAlwaysVisibleControllerRight = value; } } /// [Obsolete("Use ControllerPositionLeft instead.")] public Vector3 HandPositionLeft { get => ControllerPositionLeft; set { ControllerPositionLeft = value; } } /// [Obsolete("Use ControllerPositionRight instead.")] public Vector3 HandPositionRight { get => ControllerPositionRight; set { ControllerPositionRight = value; } } /// [Obsolete("Use ControllerRotationLeft instead.")] public Vector3 HandRotationLeft { get => ControllerRotationLeft; set { ControllerRotationLeft = value; } } /// [Obsolete("Use ControllerRotationRight instead.")] public Vector3 HandRotationRight { get => ControllerRotationRight; set { ControllerRotationRight = value; } } /// [Obsolete("Use ResetControllerLeft instead.")] public void ResetHandLeft() { ResetControllerLeft(); } /// [Obsolete("Use ResetControllerRight instead.")] public void ResetHandRight() { ResetControllerRight(); } #endregion } }