// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. using Microsoft.MixedReality.Toolkit.Input; using Microsoft.MixedReality.Toolkit.Utilities; using System; using TMPro; using UnityEngine; using UnityEngine.UI; namespace Microsoft.MixedReality.Toolkit.Experimental.UI { /// /// A simple general use keyboard that is ideal for AR/VR applications that do not provide a native keyboard. /// /// /// NOTE: This keyboard will not automatically appear when you select an InputField in your /// Canvas. In order for the keyboard to appear you must call Keyboard.Instance.PresentKeyboard(string). /// To retrieve the input from the Keyboard, subscribe to the textEntered event. Note that /// tapping 'Close' on the Keyboard will not fire the textEntered event. You must tap 'Enter' to /// get the textEntered event. public class NonNativeKeyboard : InputSystemGlobalHandlerListener, IMixedRealityDictationHandler { public static NonNativeKeyboard Instance { get; private set; } /// /// Layout type enum for the type of keyboard layout to use. /// This is used when spawning to enable the correct keys based on layout type. /// public enum LayoutType { Alpha, Symbol, URL, Email, } #region Callbacks /// /// Sent when the 'Enter' button is pressed. To retrieve the text from the event, /// cast the sender to 'Keyboard' and get the text from the TextInput field. /// (Cleared when keyboard is closed.) /// public event EventHandler OnTextSubmitted = delegate { }; /// /// Fired every time the text in the InputField changes. /// (Cleared when keyboard is closed.) /// public event Action OnTextUpdated = delegate { }; /// /// Fired every time the close button is pressed. /// (Cleared when keyboard is closed.) /// public event EventHandler OnClosed = delegate { }; /// /// Sent when the 'Previous' button is pressed. Ideally you would use this event /// to set your targeted text input to the previous text field in your document. /// (Cleared when keyboard is closed.) /// public event EventHandler OnPrevious = delegate { }; /// /// Sent when the 'Next' button is pressed. Ideally you would use this event /// to set your targeted text input to the next text field in your document. /// (Cleared when keyboard is closed.) /// public event EventHandler OnNext = delegate { }; /// /// Sent when the keyboard is placed. This allows listener to know when someone else is co-opting the keyboard. /// public event EventHandler OnPlacement = delegate { }; #endregion Callbacks /// /// The InputField that the keyboard uses to show the currently edited text. /// If you are using the Keyboard prefab you can ignore this field as it will /// be already assigned. /// [Experimental] public TMP_InputField InputField = null; /// /// Move the axis slider based on the camera forward and the keyboard plane projection. /// public AxisSlider InputFieldSlide = null; /// /// Bool for toggling the slider being enabled. /// public bool SliderEnabled = true; /// /// Bool to flag submitting on enter /// public bool SubmitOnEnter = true; /// /// The panel that contains the alpha keys. /// public Image AlphaKeyboard = null; /// /// The panel that contains the number and symbol keys. /// public Image SymbolKeyboard = null; /// /// References abc bottom panel. /// public Image AlphaSubKeys = null; /// /// References .com bottom panel. /// public Image AlphaWebKeys = null; /// /// References @ bottom panel. /// public Image AlphaMailKeys = null; private LayoutType m_LastKeyboardLayout = LayoutType.Alpha; /// /// The scale the keyboard should be at its maximum distance. /// [Header("Positioning")] [SerializeField] private float m_MaxScale = 1.0f; /// /// The scale the keyboard should be at its minimum distance. /// [SerializeField] private float m_MinScale = 1.0f; /// /// The maximum distance the keyboard should be from the user. /// [SerializeField] private float m_MaxDistance = 3.5f; /// /// The minimum distance the keyboard needs to be away from the user. /// [SerializeField] private float m_MinDistance = 0.25f; /// /// Make the keyboard disappear automatically after a timeout /// public bool CloseOnInactivity = true; /// /// Inactivity time that makes the keyboard disappear automatically. /// public float CloseOnInactivityTime = 15; /// /// Time on which the keyboard should close on inactivity /// private float _closingTime; /// /// Event fired when shift key on keyboard is pressed. /// public event Action OnKeyboardShifted = delegate { }; /// /// Event fired when char key on keyboard is pressed. /// public event Action OnKeyboardValueKeyPressed = delegate { }; /// /// Event fired when function key on keyboard is pressed. /// Fires before internal keyboard state is updated. /// public event Action OnKeyboardFunctionKeyPressed = delegate { }; /// /// Current shift state of keyboard. /// private bool m_IsShifted = false; /// /// Current caps lock state of keyboard. /// private bool m_IsCapslocked = false; /// /// Accessor reporting shift state of keyboard. /// public bool IsShifted { get { return m_IsShifted; } } /// /// Accessor reporting caps lock state of keyboard. /// public bool IsCapsLocked { get { return m_IsCapslocked; } } /// /// The position of the caret in the text field. /// private int m_CaretPosition = 0; /// /// The starting scale of the keyboard. /// private Vector3 m_StartingScale = Vector3.one; /// /// The default bounds of the keyboard. /// private Vector3 m_ObjectBounds; /// /// The default color of the mike key. /// private Color _defaultColor; /// /// The image on the mike key. /// private Image _recordImage; /// /// User can add an audio source to the keyboard to have a click be heard on tapping a key /// private AudioSource _audioSource; /// /// Dictation System /// private IMixedRealityDictationSystem dictationSystem; /// /// Deactivate on Awake. /// void Awake() { Instance = this; m_StartingScale = transform.localScale; Bounds canvasBounds = RectTransformUtility.CalculateRelativeRectTransformBounds(transform); RectTransform rect = GetComponent(); m_ObjectBounds = new Vector3(canvasBounds.size.x * rect.localScale.x, canvasBounds.size.y * rect.localScale.y, canvasBounds.size.z * rect.localScale.z); // Actually find microphone key in the keyboard var dictationButton = TransformExtensions.GetChildRecursive(gameObject.transform, "Dictation"); if (dictationButton != null) { var dictationIcon = dictationButton.Find("keyboard_closeIcon"); if (dictationIcon != null) { _recordImage = dictationIcon.GetComponentInChildren(); var material = new Material(_recordImage.material); _defaultColor = material.color; _recordImage.material = material; } } // Setting the keyboardType to an undefined TouchScreenKeyboardType, // which prevents the MRTK keyboard from triggering the system keyboard itself. InputField.keyboardType = (TouchScreenKeyboardType)(int.MaxValue); // Keep keyboard deactivated until needed gameObject.SetActive(false); } /// /// Set up Dictation, CanvasEX, and automatically select the TextInput object. /// protected override void Start() { base.Start(); dictationSystem = CoreServices.GetInputSystemDataProvider(); // Delegate Subscription InputField.onValueChanged.AddListener(DoTextUpdated); } protected override void RegisterHandlers() { CoreServices.InputSystem?.RegisterHandler(this); } protected override void UnregisterHandlers() { CoreServices.InputSystem?.UnregisterHandler(this); } /// /// Intermediary function for text update events. /// Workaround for strange leftover reference when unsubscribing. /// /// String value. private void DoTextUpdated(string value) => OnTextUpdated?.Invoke(value); /// /// Makes sure the input field is always selected while the keyboard is up. /// private void LateUpdate() { // Axis Slider if (SliderEnabled) { Vector3 nearPoint = Vector3.ProjectOnPlane(CameraCache.Main.transform.forward, transform.forward); Vector3 relPos = transform.InverseTransformPoint(nearPoint); InputFieldSlide.TargetPoint = relPos; } CheckForCloseOnInactivityTimeExpired(); } private void UpdateCaretPosition(int newPos) => InputField.caretPosition = newPos; /// /// Called whenever the keyboard is disabled or deactivated. /// protected override void OnDisable() { base.OnDisable(); m_LastKeyboardLayout = LayoutType.Alpha; Clear(); } /// /// Called when dictation hypothesis is found. Not used here /// /// Dictation event data public void OnDictationHypothesis(DictationEventData eventData) { } /// /// Called when dictation result is obtained /// /// Dictation event data public void OnDictationResult(DictationEventData eventData) { if (eventData.used) { return; } var text = eventData.DictationResult; ResetClosingTime(); if (text != null) { m_CaretPosition = InputField.caretPosition; InputField.text = InputField.text.Insert(m_CaretPosition, text); m_CaretPosition += text.Length; UpdateCaretPosition(m_CaretPosition); eventData.Use(); } } /// /// Called when dictation is completed /// /// Dictation event data public void OnDictationComplete(DictationEventData eventData) { ResetClosingTime(); SetMicrophoneDefault(); } /// /// Called on dictation error. Not used here. /// /// Dictation event data public void OnDictationError(DictationEventData eventData) { } /// /// Destroy unmanaged memory links. /// void OnDestroy() { if (dictationSystem != null && IsMicrophoneActive()) { dictationSystem.StopRecording(); } Instance = null; } #region Present Functions /// /// Present the default keyboard to the camera. /// public void PresentKeyboard() { ResetClosingTime(); gameObject.SetActive(true); ActivateSpecificKeyboard(LayoutType.Alpha); OnPlacement(this, EventArgs.Empty); // todo: if the app is built for xaml, our prefab and the system keyboard may be displayed. InputField.ActivateInputField(); SetMicrophoneDefault(); } /// /// Presents the default keyboard to the camera, with start text. /// /// The initial text to show in the keyboard's input field. public void PresentKeyboard(string startText) { PresentKeyboard(); Clear(); InputField.text = startText; } /// /// Presents a specific keyboard to the camera. /// /// Specify the keyboard type. public void PresentKeyboard(LayoutType keyboardType) { PresentKeyboard(); ActivateSpecificKeyboard(keyboardType); } /// /// Presents a specific keyboard to the camera, with start text. /// /// The initial text to show in the keyboard's input field. /// Specify the keyboard type. public void PresentKeyboard(string startText, LayoutType keyboardType) { PresentKeyboard(startText); ActivateSpecificKeyboard(keyboardType); } #endregion Present Functions /// /// Function to reposition the Keyboard based on target position and vertical offset /// /// World position for keyboard /// Optional vertical offset of keyboard public void RepositionKeyboard(Vector3 kbPos, float verticalOffset = 0.0f) { transform.position = kbPos; ScaleToSize(); LookAtTargetOrigin(); } /// /// Function to reposition the keyboard based on target transform and collider information /// /// Transform of target object to remain relative to /// Optional collider information for offset placement /// Optional vertical offset from the target public void RepositionKeyboard(Transform objectTransform, BoxCollider aCollider = null, float verticalOffset = 0.0f) { transform.position = objectTransform.position; if (aCollider != null) { float yTranslation = -((aCollider.bounds.size.y * 0.5f) + verticalOffset); transform.Translate(0.0f, yTranslation, -0.6f, objectTransform); } else { float yTranslation = -((m_ObjectBounds.y * 0.5f) + verticalOffset); transform.Translate(0.0f, yTranslation, -0.6f, objectTransform); } ScaleToSize(); LookAtTargetOrigin(); } /// /// Function to scale keyboard to the appropriate size based on distance /// private void ScaleToSize() { float distance = (transform.position - CameraCache.Main.transform.position).magnitude; float distancePercent = (distance - m_MinDistance) / (m_MaxDistance - m_MinDistance); float scale = m_MinScale + (m_MaxScale - m_MinScale) * distancePercent; scale = Mathf.Clamp(scale, m_MinScale, m_MaxScale); transform.localScale = m_StartingScale * scale; Debug.LogFormat("Setting scale: {0} for distance: {1}", scale, distance); } /// /// Look at function to have the keyboard face the user /// private void LookAtTargetOrigin() { transform.LookAt(CameraCache.Main.transform.position); transform.Rotate(Vector3.up, 180.0f); } /// /// Activates a specific keyboard layout, and any sub keys. /// /// The keyboard layout type that should be activated private void ActivateSpecificKeyboard(LayoutType keyboardType) { DisableAllKeyboards(); ResetKeyboardState(); switch (keyboardType) { case LayoutType.URL: { ShowAlphaKeyboard(); TryToShowURLSubkeys(); break; } case LayoutType.Email: { ShowAlphaKeyboard(); TryToShowEmailSubkeys(); break; } case LayoutType.Symbol: { ShowSymbolKeyboard(); break; } case LayoutType.Alpha: default: { ShowAlphaKeyboard(); TryToShowAlphaSubkeys(); break; } } } #region Keyboard Functions #region Dictation /// /// Initialize dictation mode. /// private void BeginDictation() { ResetClosingTime(); dictationSystem.StartRecording(gameObject); SetMicrophoneRecording(); } private bool IsMicrophoneActive() { var result = _recordImage.color != _defaultColor; return result; } /// /// Set mike default look /// private void SetMicrophoneDefault() { _recordImage.color = _defaultColor; } /// /// Set mike recording look (red) /// private void SetMicrophoneRecording() { _recordImage.color = Color.red; } /// /// Terminate dictation mode. /// public void EndDictation() { dictationSystem.StopRecording(); SetMicrophoneDefault(); } #endregion Dictation /// /// Primary method for typing individual characters to a text field. /// /// The valueKey of the pressed key. public void AppendValue(KeyboardValueKey valueKey) { IndicateActivity(); string value = ""; OnKeyboardValueKeyPressed(valueKey); // Shift value should only be applied if a shift value is present. if (m_IsShifted && !string.IsNullOrEmpty(valueKey.ShiftValue)) { value = valueKey.ShiftValue; } else { value = valueKey.Value; } if (!m_IsCapslocked) { Shift(false); } m_CaretPosition = InputField.caretPosition; InputField.text = InputField.text.Insert(m_CaretPosition, value); m_CaretPosition += value.Length; UpdateCaretPosition(m_CaretPosition); } /// /// Trigger specific keyboard functionality. /// /// The functionKey of the pressed key. public void FunctionKey(KeyboardKeyFunc functionKey) { IndicateActivity(); OnKeyboardFunctionKeyPressed(functionKey); switch (functionKey.ButtonFunction) { case KeyboardKeyFunc.Function.Enter: { Enter(); break; } case KeyboardKeyFunc.Function.Tab: { Tab(); break; } case KeyboardKeyFunc.Function.ABC: { ActivateSpecificKeyboard(m_LastKeyboardLayout); break; } case KeyboardKeyFunc.Function.Symbol: { ActivateSpecificKeyboard(LayoutType.Symbol); break; } case KeyboardKeyFunc.Function.Previous: { MoveCaretLeft(); break; } case KeyboardKeyFunc.Function.Next: { MoveCaretRight(); break; } case KeyboardKeyFunc.Function.Close: { Close(); break; } case KeyboardKeyFunc.Function.Dictate: { if (dictationSystem == null) { break; } if (IsMicrophoneActive()) { EndDictation(); } else { BeginDictation(); } break; } case KeyboardKeyFunc.Function.Shift: { Shift(!m_IsShifted); break; } case KeyboardKeyFunc.Function.CapsLock: { CapsLock(!m_IsCapslocked); break; } case KeyboardKeyFunc.Function.Space: { Space(); break; } case KeyboardKeyFunc.Function.Backspace: { Backspace(); break; } case KeyboardKeyFunc.Function.UNDEFINED: { Debug.LogErrorFormat("The {0} key on this keyboard hasn't been assigned a function.", functionKey.name); break; } default: throw new ArgumentOutOfRangeException(); } } /// /// Delete the character before the caret. /// public void Backspace() { // check if text is selected if (InputField.selectionFocusPosition != InputField.caretPosition || InputField.selectionAnchorPosition != InputField.caretPosition) { if (InputField.selectionAnchorPosition > InputField.selectionFocusPosition) // right to left { InputField.text = InputField.text.Substring(0, InputField.selectionFocusPosition) + InputField.text.Substring(InputField.selectionAnchorPosition); InputField.caretPosition = InputField.selectionFocusPosition; } else // left to right { InputField.text = InputField.text.Substring(0, InputField.selectionAnchorPosition) + InputField.text.Substring(InputField.selectionFocusPosition); InputField.caretPosition = InputField.selectionAnchorPosition; } m_CaretPosition = InputField.caretPosition; InputField.selectionAnchorPosition = m_CaretPosition; InputField.selectionFocusPosition = m_CaretPosition; } else { m_CaretPosition = InputField.caretPosition; if (m_CaretPosition > 0) { --m_CaretPosition; InputField.text = InputField.text.Remove(m_CaretPosition, 1); UpdateCaretPosition(m_CaretPosition); } } } /// /// Send the "previous" event. /// public void Previous() { OnPrevious(this, EventArgs.Empty); } /// /// Send the "next" event. /// public void Next() { OnNext(this, EventArgs.Empty); } /// /// Fire the text entered event for objects listening to keyboard. /// Immediately closes keyboard. /// public void Enter() { if (SubmitOnEnter) { // Send text entered event and close the keyboard OnTextSubmitted?.Invoke(this, EventArgs.Empty); Close(); } else { string enterString = "\n"; m_CaretPosition = InputField.caretPosition; InputField.text = InputField.text.Insert(m_CaretPosition, enterString); m_CaretPosition += enterString.Length; UpdateCaretPosition(m_CaretPosition); } } /// /// Set the keyboard to a single action shift state. /// /// value the shift key should have after calling the method public void Shift(bool newShiftState) { m_IsShifted = newShiftState; OnKeyboardShifted(m_IsShifted); if (m_IsCapslocked && !newShiftState) { m_IsCapslocked = false; } } /// /// Set the keyboard to a permanent shift state. /// /// Caps lock state the method is switching to public void CapsLock(bool newCapsLockState) { m_IsCapslocked = newCapsLockState; Shift(newCapsLockState); } /// /// Insert a space character. /// public void Space() { m_CaretPosition = InputField.caretPosition; InputField.text = InputField.text.Insert(m_CaretPosition++, " "); UpdateCaretPosition(m_CaretPosition); } /// /// Insert a tab character. /// public void Tab() { string tabString = "\t"; m_CaretPosition = InputField.caretPosition; InputField.text = InputField.text.Insert(m_CaretPosition, tabString); m_CaretPosition += tabString.Length; UpdateCaretPosition(m_CaretPosition); } /// /// Move caret to the left. /// public void MoveCaretLeft() { m_CaretPosition = InputField.caretPosition; if (m_CaretPosition > 0) { --m_CaretPosition; UpdateCaretPosition(m_CaretPosition); } } /// /// Move caret to the right. /// public void MoveCaretRight() { m_CaretPosition = InputField.caretPosition; if (m_CaretPosition < InputField.text.Length) { ++m_CaretPosition; UpdateCaretPosition(m_CaretPosition); } } /// /// Close the keyboard. /// (Clears all event subscriptions.) /// public void Close() { if (IsMicrophoneActive()) { dictationSystem.StopRecording(); } SetMicrophoneDefault(); OnClosed(this, EventArgs.Empty); gameObject.SetActive(false); } /// /// Clear the text input field. /// public void Clear() { ResetKeyboardState(); if (InputField.caretPosition != 0) { InputField.MoveTextStart(false); } InputField.text = ""; m_CaretPosition = InputField.caretPosition; } #endregion /// /// Method to set the sizes by code, as the properties are private. /// Useful for scaling 'from the outside', for instance taking care of differences between /// immersive headsets and HoloLens /// /// Min scale factor /// Max scale factor /// Min distance from camera /// Max distance from camera public void SetScaleSizeValues(float minScale, float maxScale, float minDistance, float maxDistance) { m_MinScale = minScale; m_MaxScale = maxScale; m_MinDistance = minDistance; m_MaxDistance = maxDistance; } #region Keyboard Layout Modes /// /// Enable the alpha keyboard. /// public void ShowAlphaKeyboard() { AlphaKeyboard.gameObject.SetActive(true); m_LastKeyboardLayout = LayoutType.Alpha; } /// /// Show the default subkeys only on the Alphanumeric keyboard. /// /// Returns true if default subkeys were activated, false if alphanumeric keyboard isn't active private bool TryToShowAlphaSubkeys() { if (AlphaKeyboard.IsActive()) { AlphaSubKeys.gameObject.SetActive(true); return true; } else { return false; } } /// /// Show the email subkeys only on the Alphanumeric keyboard. /// /// Returns true if the email subkey was activated, false if alphanumeric keyboard is not active and key can't be activated private bool TryToShowEmailSubkeys() { if (AlphaKeyboard.IsActive()) { AlphaMailKeys.gameObject.SetActive(true); m_LastKeyboardLayout = LayoutType.Email; return true; } else { return false; } } /// /// Show the URL subkeys only on the Alphanumeric keyboard. /// /// Returns true if the URL subkey was activated, false if alphanumeric keyboard is not active and key can't be activated private bool TryToShowURLSubkeys() { if (AlphaKeyboard.IsActive()) { AlphaWebKeys.gameObject.SetActive(true); m_LastKeyboardLayout = LayoutType.URL; return true; } else { return false; } } /// /// Enable the symbol keyboard. /// public void ShowSymbolKeyboard() { SymbolKeyboard.gameObject.SetActive(true); } /// /// Disable GameObjects for all keyboard elements. /// private void DisableAllKeyboards() { AlphaKeyboard.gameObject.SetActive(false); SymbolKeyboard.gameObject.SetActive(false); AlphaWebKeys.gameObject.SetActive(false); AlphaMailKeys.gameObject.SetActive(false); AlphaSubKeys.gameObject.SetActive(false); } /// /// Reset temporary states of keyboard. /// private void ResetKeyboardState() { CapsLock(false); } #endregion Keyboard Layout Modes /// /// Respond to keyboard activity: reset timeout timer, play sound /// private void IndicateActivity() { ResetClosingTime(); if (_audioSource == null) { _audioSource = GetComponent(); } if (_audioSource != null) { _audioSource.Play(); } } /// /// Reset inactivity closing timer /// private void ResetClosingTime() { if (CloseOnInactivity) { _closingTime = Time.time + CloseOnInactivityTime; } } /// /// Check if the keyboard has been left alone for too long and close /// private void CheckForCloseOnInactivityTimeExpired() { if (Time.time > _closingTime && CloseOnInactivity) { Close(); } } } }