// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using Microsoft.MixedReality.Toolkit.Input;
using Microsoft.MixedReality.Toolkit.Utilities;
using System;
using TMPro;
using UnityEngine;
using UnityEngine.UI;
namespace Microsoft.MixedReality.Toolkit.Experimental.UI
{
///
/// A simple general use keyboard that is ideal for AR/VR applications that do not provide a native keyboard.
///
///
/// NOTE: This keyboard will not automatically appear when you select an InputField in your
/// Canvas. In order for the keyboard to appear you must call Keyboard.Instance.PresentKeyboard(string).
/// To retrieve the input from the Keyboard, subscribe to the textEntered event. Note that
/// tapping 'Close' on the Keyboard will not fire the textEntered event. You must tap 'Enter' to
/// get the textEntered event.
public class NonNativeKeyboard : InputSystemGlobalHandlerListener, IMixedRealityDictationHandler
{
public static NonNativeKeyboard Instance { get; private set; }
///
/// Layout type enum for the type of keyboard layout to use.
/// This is used when spawning to enable the correct keys based on layout type.
///
public enum LayoutType
{
Alpha,
Symbol,
URL,
Email,
}
#region Callbacks
///
/// Sent when the 'Enter' button is pressed. To retrieve the text from the event,
/// cast the sender to 'Keyboard' and get the text from the TextInput field.
/// (Cleared when keyboard is closed.)
///
public event EventHandler OnTextSubmitted = delegate { };
///
/// Fired every time the text in the InputField changes.
/// (Cleared when keyboard is closed.)
///
public event Action OnTextUpdated = delegate { };
///
/// Fired every time the close button is pressed.
/// (Cleared when keyboard is closed.)
///
public event EventHandler OnClosed = delegate { };
///
/// Sent when the 'Previous' button is pressed. Ideally you would use this event
/// to set your targeted text input to the previous text field in your document.
/// (Cleared when keyboard is closed.)
///
public event EventHandler OnPrevious = delegate { };
///
/// Sent when the 'Next' button is pressed. Ideally you would use this event
/// to set your targeted text input to the next text field in your document.
/// (Cleared when keyboard is closed.)
///
public event EventHandler OnNext = delegate { };
///
/// Sent when the keyboard is placed. This allows listener to know when someone else is co-opting the keyboard.
///
public event EventHandler OnPlacement = delegate { };
#endregion Callbacks
///
/// The InputField that the keyboard uses to show the currently edited text.
/// If you are using the Keyboard prefab you can ignore this field as it will
/// be already assigned.
///
[Experimental]
public TMP_InputField InputField = null;
///
/// Move the axis slider based on the camera forward and the keyboard plane projection.
///
public AxisSlider InputFieldSlide = null;
///
/// Bool for toggling the slider being enabled.
///
public bool SliderEnabled = true;
///
/// Bool to flag submitting on enter
///
public bool SubmitOnEnter = true;
///
/// The panel that contains the alpha keys.
///
public Image AlphaKeyboard = null;
///
/// The panel that contains the number and symbol keys.
///
public Image SymbolKeyboard = null;
///
/// References abc bottom panel.
///
public Image AlphaSubKeys = null;
///
/// References .com bottom panel.
///
public Image AlphaWebKeys = null;
///
/// References @ bottom panel.
///
public Image AlphaMailKeys = null;
private LayoutType m_LastKeyboardLayout = LayoutType.Alpha;
///
/// The scale the keyboard should be at its maximum distance.
///
[Header("Positioning")]
[SerializeField]
private float m_MaxScale = 1.0f;
///
/// The scale the keyboard should be at its minimum distance.
///
[SerializeField]
private float m_MinScale = 1.0f;
///
/// The maximum distance the keyboard should be from the user.
///
[SerializeField]
private float m_MaxDistance = 3.5f;
///
/// The minimum distance the keyboard needs to be away from the user.
///
[SerializeField]
private float m_MinDistance = 0.25f;
///
/// Make the keyboard disappear automatically after a timeout
///
public bool CloseOnInactivity = true;
///
/// Inactivity time that makes the keyboard disappear automatically.
///
public float CloseOnInactivityTime = 15;
///
/// Time on which the keyboard should close on inactivity
///
private float _closingTime;
///
/// Event fired when shift key on keyboard is pressed.
///
public event Action OnKeyboardShifted = delegate { };
///
/// Event fired when char key on keyboard is pressed.
///
public event Action OnKeyboardValueKeyPressed = delegate { };
///
/// Event fired when function key on keyboard is pressed.
/// Fires before internal keyboard state is updated.
///
public event Action OnKeyboardFunctionKeyPressed = delegate { };
///
/// Current shift state of keyboard.
///
private bool m_IsShifted = false;
///
/// Current caps lock state of keyboard.
///
private bool m_IsCapslocked = false;
///
/// Accessor reporting shift state of keyboard.
///
public bool IsShifted
{
get { return m_IsShifted; }
}
///
/// Accessor reporting caps lock state of keyboard.
///
public bool IsCapsLocked
{
get { return m_IsCapslocked; }
}
///
/// The position of the caret in the text field.
///
private int m_CaretPosition = 0;
///
/// The starting scale of the keyboard.
///
private Vector3 m_StartingScale = Vector3.one;
///
/// The default bounds of the keyboard.
///
private Vector3 m_ObjectBounds;
///
/// The default color of the mike key.
///
private Color _defaultColor;
///
/// The image on the mike key.
///
private Image _recordImage;
///
/// User can add an audio source to the keyboard to have a click be heard on tapping a key
///
private AudioSource _audioSource;
///
/// Dictation System
///
private IMixedRealityDictationSystem dictationSystem;
///
/// Deactivate on Awake.
///
void Awake()
{
Instance = this;
m_StartingScale = transform.localScale;
Bounds canvasBounds = RectTransformUtility.CalculateRelativeRectTransformBounds(transform);
RectTransform rect = GetComponent();
m_ObjectBounds = new Vector3(canvasBounds.size.x * rect.localScale.x, canvasBounds.size.y * rect.localScale.y, canvasBounds.size.z * rect.localScale.z);
// Actually find microphone key in the keyboard
var dictationButton = TransformExtensions.GetChildRecursive(gameObject.transform, "Dictation");
if (dictationButton != null)
{
var dictationIcon = dictationButton.Find("keyboard_closeIcon");
if (dictationIcon != null)
{
_recordImage = dictationIcon.GetComponentInChildren();
var material = new Material(_recordImage.material);
_defaultColor = material.color;
_recordImage.material = material;
}
}
// Setting the keyboardType to an undefined TouchScreenKeyboardType,
// which prevents the MRTK keyboard from triggering the system keyboard itself.
InputField.keyboardType = (TouchScreenKeyboardType)(int.MaxValue);
// Keep keyboard deactivated until needed
gameObject.SetActive(false);
}
///
/// Set up Dictation, CanvasEX, and automatically select the TextInput object.
///
protected override void Start()
{
base.Start();
dictationSystem = CoreServices.GetInputSystemDataProvider();
// Delegate Subscription
InputField.onValueChanged.AddListener(DoTextUpdated);
}
protected override void RegisterHandlers()
{
CoreServices.InputSystem?.RegisterHandler(this);
}
protected override void UnregisterHandlers()
{
CoreServices.InputSystem?.UnregisterHandler(this);
}
///
/// Intermediary function for text update events.
/// Workaround for strange leftover reference when unsubscribing.
///
/// String value.
private void DoTextUpdated(string value) => OnTextUpdated?.Invoke(value);
///
/// Makes sure the input field is always selected while the keyboard is up.
///
private void LateUpdate()
{
// Axis Slider
if (SliderEnabled)
{
Vector3 nearPoint = Vector3.ProjectOnPlane(CameraCache.Main.transform.forward, transform.forward);
Vector3 relPos = transform.InverseTransformPoint(nearPoint);
InputFieldSlide.TargetPoint = relPos;
}
CheckForCloseOnInactivityTimeExpired();
}
private void UpdateCaretPosition(int newPos) => InputField.caretPosition = newPos;
///
/// Called whenever the keyboard is disabled or deactivated.
///
protected override void OnDisable()
{
base.OnDisable();
m_LastKeyboardLayout = LayoutType.Alpha;
Clear();
}
///
/// Called when dictation hypothesis is found. Not used here
///
/// Dictation event data
public void OnDictationHypothesis(DictationEventData eventData) { }
///
/// Called when dictation result is obtained
///
/// Dictation event data
public void OnDictationResult(DictationEventData eventData)
{
if (eventData.used)
{
return;
}
var text = eventData.DictationResult;
ResetClosingTime();
if (text != null)
{
m_CaretPosition = InputField.caretPosition;
InputField.text = InputField.text.Insert(m_CaretPosition, text);
m_CaretPosition += text.Length;
UpdateCaretPosition(m_CaretPosition);
eventData.Use();
}
}
///
/// Called when dictation is completed
///
/// Dictation event data
public void OnDictationComplete(DictationEventData eventData)
{
ResetClosingTime();
SetMicrophoneDefault();
}
///
/// Called on dictation error. Not used here.
///
/// Dictation event data
public void OnDictationError(DictationEventData eventData) { }
///
/// Destroy unmanaged memory links.
///
void OnDestroy()
{
if (dictationSystem != null && IsMicrophoneActive())
{
dictationSystem.StopRecording();
}
Instance = null;
}
#region Present Functions
///
/// Present the default keyboard to the camera.
///
public void PresentKeyboard()
{
ResetClosingTime();
gameObject.SetActive(true);
ActivateSpecificKeyboard(LayoutType.Alpha);
OnPlacement(this, EventArgs.Empty);
// todo: if the app is built for xaml, our prefab and the system keyboard may be displayed.
InputField.ActivateInputField();
SetMicrophoneDefault();
}
///
/// Presents the default keyboard to the camera, with start text.
///
/// The initial text to show in the keyboard's input field.
public void PresentKeyboard(string startText)
{
PresentKeyboard();
Clear();
InputField.text = startText;
}
///
/// Presents a specific keyboard to the camera.
///
/// Specify the keyboard type.
public void PresentKeyboard(LayoutType keyboardType)
{
PresentKeyboard();
ActivateSpecificKeyboard(keyboardType);
}
///
/// Presents a specific keyboard to the camera, with start text.
///
/// The initial text to show in the keyboard's input field.
/// Specify the keyboard type.
public void PresentKeyboard(string startText, LayoutType keyboardType)
{
PresentKeyboard(startText);
ActivateSpecificKeyboard(keyboardType);
}
#endregion Present Functions
///
/// Function to reposition the Keyboard based on target position and vertical offset
///
/// World position for keyboard
/// Optional vertical offset of keyboard
public void RepositionKeyboard(Vector3 kbPos, float verticalOffset = 0.0f)
{
transform.position = kbPos;
ScaleToSize();
LookAtTargetOrigin();
}
///
/// Function to reposition the keyboard based on target transform and collider information
///
/// Transform of target object to remain relative to
/// Optional collider information for offset placement
/// Optional vertical offset from the target
public void RepositionKeyboard(Transform objectTransform, BoxCollider aCollider = null, float verticalOffset = 0.0f)
{
transform.position = objectTransform.position;
if (aCollider != null)
{
float yTranslation = -((aCollider.bounds.size.y * 0.5f) + verticalOffset);
transform.Translate(0.0f, yTranslation, -0.6f, objectTransform);
}
else
{
float yTranslation = -((m_ObjectBounds.y * 0.5f) + verticalOffset);
transform.Translate(0.0f, yTranslation, -0.6f, objectTransform);
}
ScaleToSize();
LookAtTargetOrigin();
}
///
/// Function to scale keyboard to the appropriate size based on distance
///
private void ScaleToSize()
{
float distance = (transform.position - CameraCache.Main.transform.position).magnitude;
float distancePercent = (distance - m_MinDistance) / (m_MaxDistance - m_MinDistance);
float scale = m_MinScale + (m_MaxScale - m_MinScale) * distancePercent;
scale = Mathf.Clamp(scale, m_MinScale, m_MaxScale);
transform.localScale = m_StartingScale * scale;
Debug.LogFormat("Setting scale: {0} for distance: {1}", scale, distance);
}
///
/// Look at function to have the keyboard face the user
///
private void LookAtTargetOrigin()
{
transform.LookAt(CameraCache.Main.transform.position);
transform.Rotate(Vector3.up, 180.0f);
}
///
/// Activates a specific keyboard layout, and any sub keys.
///
/// The keyboard layout type that should be activated
private void ActivateSpecificKeyboard(LayoutType keyboardType)
{
DisableAllKeyboards();
ResetKeyboardState();
switch (keyboardType)
{
case LayoutType.URL:
{
ShowAlphaKeyboard();
TryToShowURLSubkeys();
break;
}
case LayoutType.Email:
{
ShowAlphaKeyboard();
TryToShowEmailSubkeys();
break;
}
case LayoutType.Symbol:
{
ShowSymbolKeyboard();
break;
}
case LayoutType.Alpha:
default:
{
ShowAlphaKeyboard();
TryToShowAlphaSubkeys();
break;
}
}
}
#region Keyboard Functions
#region Dictation
///
/// Initialize dictation mode.
///
private void BeginDictation()
{
ResetClosingTime();
dictationSystem.StartRecording(gameObject);
SetMicrophoneRecording();
}
private bool IsMicrophoneActive()
{
var result = _recordImage.color != _defaultColor;
return result;
}
///
/// Set mike default look
///
private void SetMicrophoneDefault()
{
_recordImage.color = _defaultColor;
}
///
/// Set mike recording look (red)
///
private void SetMicrophoneRecording()
{
_recordImage.color = Color.red;
}
///
/// Terminate dictation mode.
///
public void EndDictation()
{
dictationSystem.StopRecording();
SetMicrophoneDefault();
}
#endregion Dictation
///
/// Primary method for typing individual characters to a text field.
///
/// The valueKey of the pressed key.
public void AppendValue(KeyboardValueKey valueKey)
{
IndicateActivity();
string value = "";
OnKeyboardValueKeyPressed(valueKey);
// Shift value should only be applied if a shift value is present.
if (m_IsShifted && !string.IsNullOrEmpty(valueKey.ShiftValue))
{
value = valueKey.ShiftValue;
}
else
{
value = valueKey.Value;
}
if (!m_IsCapslocked)
{
Shift(false);
}
m_CaretPosition = InputField.caretPosition;
InputField.text = InputField.text.Insert(m_CaretPosition, value);
m_CaretPosition += value.Length;
UpdateCaretPosition(m_CaretPosition);
}
///
/// Trigger specific keyboard functionality.
///
/// The functionKey of the pressed key.
public void FunctionKey(KeyboardKeyFunc functionKey)
{
IndicateActivity();
OnKeyboardFunctionKeyPressed(functionKey);
switch (functionKey.ButtonFunction)
{
case KeyboardKeyFunc.Function.Enter:
{
Enter();
break;
}
case KeyboardKeyFunc.Function.Tab:
{
Tab();
break;
}
case KeyboardKeyFunc.Function.ABC:
{
ActivateSpecificKeyboard(m_LastKeyboardLayout);
break;
}
case KeyboardKeyFunc.Function.Symbol:
{
ActivateSpecificKeyboard(LayoutType.Symbol);
break;
}
case KeyboardKeyFunc.Function.Previous:
{
MoveCaretLeft();
break;
}
case KeyboardKeyFunc.Function.Next:
{
MoveCaretRight();
break;
}
case KeyboardKeyFunc.Function.Close:
{
Close();
break;
}
case KeyboardKeyFunc.Function.Dictate:
{
if (dictationSystem == null) { break; }
if (IsMicrophoneActive())
{
EndDictation();
}
else
{
BeginDictation();
}
break;
}
case KeyboardKeyFunc.Function.Shift:
{
Shift(!m_IsShifted);
break;
}
case KeyboardKeyFunc.Function.CapsLock:
{
CapsLock(!m_IsCapslocked);
break;
}
case KeyboardKeyFunc.Function.Space:
{
Space();
break;
}
case KeyboardKeyFunc.Function.Backspace:
{
Backspace();
break;
}
case KeyboardKeyFunc.Function.UNDEFINED:
{
Debug.LogErrorFormat("The {0} key on this keyboard hasn't been assigned a function.", functionKey.name);
break;
}
default:
throw new ArgumentOutOfRangeException();
}
}
///
/// Delete the character before the caret.
///
public void Backspace()
{
// check if text is selected
if (InputField.selectionFocusPosition != InputField.caretPosition || InputField.selectionAnchorPosition != InputField.caretPosition)
{
if (InputField.selectionAnchorPosition > InputField.selectionFocusPosition) // right to left
{
InputField.text = InputField.text.Substring(0, InputField.selectionFocusPosition) + InputField.text.Substring(InputField.selectionAnchorPosition);
InputField.caretPosition = InputField.selectionFocusPosition;
}
else // left to right
{
InputField.text = InputField.text.Substring(0, InputField.selectionAnchorPosition) + InputField.text.Substring(InputField.selectionFocusPosition);
InputField.caretPosition = InputField.selectionAnchorPosition;
}
m_CaretPosition = InputField.caretPosition;
InputField.selectionAnchorPosition = m_CaretPosition;
InputField.selectionFocusPosition = m_CaretPosition;
}
else
{
m_CaretPosition = InputField.caretPosition;
if (m_CaretPosition > 0)
{
--m_CaretPosition;
InputField.text = InputField.text.Remove(m_CaretPosition, 1);
UpdateCaretPosition(m_CaretPosition);
}
}
}
///
/// Send the "previous" event.
///
public void Previous()
{
OnPrevious(this, EventArgs.Empty);
}
///
/// Send the "next" event.
///
public void Next()
{
OnNext(this, EventArgs.Empty);
}
///
/// Fire the text entered event for objects listening to keyboard.
/// Immediately closes keyboard.
///
public void Enter()
{
if (SubmitOnEnter)
{
// Send text entered event and close the keyboard
OnTextSubmitted?.Invoke(this, EventArgs.Empty);
Close();
}
else
{
string enterString = "\n";
m_CaretPosition = InputField.caretPosition;
InputField.text = InputField.text.Insert(m_CaretPosition, enterString);
m_CaretPosition += enterString.Length;
UpdateCaretPosition(m_CaretPosition);
}
}
///
/// Set the keyboard to a single action shift state.
///
/// value the shift key should have after calling the method
public void Shift(bool newShiftState)
{
m_IsShifted = newShiftState;
OnKeyboardShifted(m_IsShifted);
if (m_IsCapslocked && !newShiftState)
{
m_IsCapslocked = false;
}
}
///
/// Set the keyboard to a permanent shift state.
///
/// Caps lock state the method is switching to
public void CapsLock(bool newCapsLockState)
{
m_IsCapslocked = newCapsLockState;
Shift(newCapsLockState);
}
///
/// Insert a space character.
///
public void Space()
{
m_CaretPosition = InputField.caretPosition;
InputField.text = InputField.text.Insert(m_CaretPosition++, " ");
UpdateCaretPosition(m_CaretPosition);
}
///
/// Insert a tab character.
///
public void Tab()
{
string tabString = "\t";
m_CaretPosition = InputField.caretPosition;
InputField.text = InputField.text.Insert(m_CaretPosition, tabString);
m_CaretPosition += tabString.Length;
UpdateCaretPosition(m_CaretPosition);
}
///
/// Move caret to the left.
///
public void MoveCaretLeft()
{
m_CaretPosition = InputField.caretPosition;
if (m_CaretPosition > 0)
{
--m_CaretPosition;
UpdateCaretPosition(m_CaretPosition);
}
}
///
/// Move caret to the right.
///
public void MoveCaretRight()
{
m_CaretPosition = InputField.caretPosition;
if (m_CaretPosition < InputField.text.Length)
{
++m_CaretPosition;
UpdateCaretPosition(m_CaretPosition);
}
}
///
/// Close the keyboard.
/// (Clears all event subscriptions.)
///
public void Close()
{
if (IsMicrophoneActive())
{
dictationSystem.StopRecording();
}
SetMicrophoneDefault();
OnClosed(this, EventArgs.Empty);
gameObject.SetActive(false);
}
///
/// Clear the text input field.
///
public void Clear()
{
ResetKeyboardState();
if (InputField.caretPosition != 0)
{
InputField.MoveTextStart(false);
}
InputField.text = "";
m_CaretPosition = InputField.caretPosition;
}
#endregion
///
/// Method to set the sizes by code, as the properties are private.
/// Useful for scaling 'from the outside', for instance taking care of differences between
/// immersive headsets and HoloLens
///
/// Min scale factor
/// Max scale factor
/// Min distance from camera
/// Max distance from camera
public void SetScaleSizeValues(float minScale, float maxScale, float minDistance, float maxDistance)
{
m_MinScale = minScale;
m_MaxScale = maxScale;
m_MinDistance = minDistance;
m_MaxDistance = maxDistance;
}
#region Keyboard Layout Modes
///
/// Enable the alpha keyboard.
///
public void ShowAlphaKeyboard()
{
AlphaKeyboard.gameObject.SetActive(true);
m_LastKeyboardLayout = LayoutType.Alpha;
}
///
/// Show the default subkeys only on the Alphanumeric keyboard.
///
/// Returns true if default subkeys were activated, false if alphanumeric keyboard isn't active
private bool TryToShowAlphaSubkeys()
{
if (AlphaKeyboard.IsActive())
{
AlphaSubKeys.gameObject.SetActive(true);
return true;
}
else
{
return false;
}
}
///
/// Show the email subkeys only on the Alphanumeric keyboard.
///
/// Returns true if the email subkey was activated, false if alphanumeric keyboard is not active and key can't be activated
private bool TryToShowEmailSubkeys()
{
if (AlphaKeyboard.IsActive())
{
AlphaMailKeys.gameObject.SetActive(true);
m_LastKeyboardLayout = LayoutType.Email;
return true;
}
else
{
return false;
}
}
///
/// Show the URL subkeys only on the Alphanumeric keyboard.
///
/// Returns true if the URL subkey was activated, false if alphanumeric keyboard is not active and key can't be activated
private bool TryToShowURLSubkeys()
{
if (AlphaKeyboard.IsActive())
{
AlphaWebKeys.gameObject.SetActive(true);
m_LastKeyboardLayout = LayoutType.URL;
return true;
}
else
{
return false;
}
}
///
/// Enable the symbol keyboard.
///
public void ShowSymbolKeyboard()
{
SymbolKeyboard.gameObject.SetActive(true);
}
///
/// Disable GameObjects for all keyboard elements.
///
private void DisableAllKeyboards()
{
AlphaKeyboard.gameObject.SetActive(false);
SymbolKeyboard.gameObject.SetActive(false);
AlphaWebKeys.gameObject.SetActive(false);
AlphaMailKeys.gameObject.SetActive(false);
AlphaSubKeys.gameObject.SetActive(false);
}
///
/// Reset temporary states of keyboard.
///
private void ResetKeyboardState()
{
CapsLock(false);
}
#endregion Keyboard Layout Modes
///
/// Respond to keyboard activity: reset timeout timer, play sound
///
private void IndicateActivity()
{
ResetClosingTime();
if (_audioSource == null)
{
_audioSource = GetComponent();
}
if (_audioSource != null)
{
_audioSource.Play();
}
}
///
/// Reset inactivity closing timer
///
private void ResetClosingTime()
{
if (CloseOnInactivity)
{
_closingTime = Time.time + CloseOnInactivityTime;
}
}
///
/// Check if the keyboard has been left alone for too long and close
///
private void CheckForCloseOnInactivityTimeExpired()
{
if (Time.time > _closingTime && CloseOnInactivity)
{
Close();
}
}
}
}