// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. using Microsoft.MixedReality.Toolkit.Experimental.SpatialAwareness; using Microsoft.MixedReality.Toolkit.SpatialAwareness; using Microsoft.MixedReality.Toolkit.Utilities; using System.Collections.Generic; using UnityEngine; #if SCENE_UNDERSTANDING_PRESENT && UNITY_WSA using System; using System.Collections.Concurrent; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.MixedReality.SceneUnderstanding; #if MSFT_OPENXR using Microsoft.MixedReality.OpenXR; using Microsoft.MixedReality.OpenXR.Remoting; using Microsoft.MixedReality.Toolkit.XRSDK; using UnityEngine.XR.OpenXR; #endif // MSFT_OPENXR #if WINDOWS_UWP using Windows.Perception.Spatial; using Windows.Perception.Spatial.Preview; #endif // WINDOWS_UWP using UnityEngine.Assertions; using UnityEngine.EventSystems; #endif // SCENE_UNDERSTANDING_PRESENT && UNITY_WSA #if WINDOWS_UWP using Windows.Storage; #endif namespace Microsoft.MixedReality.Toolkit.WindowsSceneUnderstanding.Experimental { /// /// A Spatial Awareness observer with Scene Understanding capabilities. /// /// /// Only works with HoloLens 2 and Unity 2019.4+ /// [MixedRealityDataProvider( typeof(IMixedRealitySpatialAwarenessSystem), SupportedPlatforms.WindowsUniversal, "Windows Scene Understanding Observer", "Experimental/WindowsSceneUnderstanding/Profiles/DefaultSceneUnderstandingObserverProfile.asset", "MixedRealityToolkit.Providers", true)] [HelpURL("https://docs.microsoft.com/windows/mixed-reality/mrtk-unity/features/spatial-awareness/scene-understanding")] public class WindowsSceneUnderstandingObserver : BaseSpatialObserver, IMixedRealitySceneUnderstandingObserver { /// /// Constructor. /// /// Friendly name of the service. /// Service priority. Used to determine order of instantiation. /// The service's configuration profile. public WindowsSceneUnderstandingObserver( IMixedRealitySpatialAwarenessSystem spatialAwarenessSystem, string name = null, uint priority = DefaultPriority, BaseMixedRealityProfile profile = null) : base(spatialAwarenessSystem, name, priority, profile) { ReadProfile(); } /// /// Reads the observer's configuration profile. /// private void ReadProfile() { if (ConfigurationProfile == null) { return; } SceneUnderstandingObserverProfile profile = ConfigurationProfile as SceneUnderstandingObserverProfile; if (profile == null) { Debug.LogError("Windows Scene Understanding Observer's configuration profile must be a SceneUnderstandingObserverProfile."); return; } AutoUpdate = profile.AutoUpdate; UpdateOnceInitialized = profile.UpdateOnceInitialized; DefaultPhysicsLayer = profile.DefaultPhysicsLayer; DefaultMaterial = profile.DefaultMaterial; DefaultWorldMeshMaterial = profile.DefaultWorldMeshMaterial; SurfaceTypes = profile.SurfaceTypes; RequestMeshData = profile.RequestMeshData; RequestPlaneData = profile.RequestPlaneData; InferRegions = profile.InferRegions; CreateGameObjects = profile.CreateGameObjects; UsePersistentObjects = profile.UsePersistentObjects; UpdateInterval = profile.UpdateInterval; FirstAutoUpdateDelay = profile.FirstAutoUpdateDelay; ShouldLoadFromFile = profile.ShouldLoadFromFile; SerializedScene = profile.SerializedScene; WorldMeshLevelOfDetail = profile.WorldMeshLevelOfDetail; InstantiationBatchRate = profile.InstantiationBatchRate; ObservationExtents = profile.ObservationExtents; QueryRadius = profile.QueryRadius; RequestOcclusionMask = profile.RequestOcclusionMask; OcclusionMaskResolution = profile.OcclusionMaskResolution; OrientScene = profile.OrientScene; } #region IMixedRealityService #if SCENE_UNDERSTANDING_PRESENT && UNITY_WSA /// public override void Reset() { CleanupObserver(); Initialize(); } #endif // SCENE_UNDERSTANDING_PRESENT && UNITY_WSA /// public override void Initialize() { #if !(UNITY_WSA && SCENE_UNDERSTANDING_PRESENT) if (Application.isPlaying) { Debug.LogWarning("The required package Microsoft.MixedReality.SceneUnderstanding is not installed or properly configured. Please visit https://docs.microsoft.com/windows/mixed-reality/mrtk-unity/features/spatial-awareness/scene-understanding for more information."); } #else base.Initialize(); #if MSFT_OPENXR isOpenXRLoaderActive = LoaderHelpers.IsLoaderActive() ?? false; isOpenXRRemotingConnected = AppRemoting.TryGetConnectionState(out ConnectionState state, out _) && state == ConnectionState.Connected; #elif WINDOWS_UWP isOpenXRLoaderActive = false; #endif // MSFT_OPENXR sceneEventData = new MixedRealitySpatialAwarenessEventData(EventSystem.current); CreateQuadFromExtents(normalizedQuadMesh, 1, 1); SceneObserverAccessStatus accessStatus = Task.Run(RequestAccess).GetAwaiter().GetResult(); if (accessStatus == SceneObserverAccessStatus.Allowed) { IsRunning = true; observerState = ObserverState.Idle; StartUpdateTimers(); } else { Debug.LogError("Something went wrong getting scene observer access!"); } if (UpdateOnceInitialized) { observerState = ObserverState.GetScene; } #endif // !(UNITY_WSA && SCENE_UNDERSTANDING_PRESENT) } /// public override void Enable() { #if SCENE_UNDERSTANDING_PRESENT && UNITY_WSA base.Enable(); // Terminate the background thread when we stop in editor. cancelToken = cancelTokenSource.Token; task = Task.Run(() => RunObserverAsync(cancelToken)).ContinueWith(t => { Debug.LogError($"{t.Exception.InnerException.GetType().Name}: {t.Exception.InnerException.Message} {t.Exception.InnerException.StackTrace}"); }, TaskContinuationOptions.OnlyOnFaulted); #else IsEnabled = false; #endif // SCENE_UNDERSTANDING_PRESENT && UNITY_WSA } #if SCENE_UNDERSTANDING_PRESENT && UNITY_WSA /// public override void Update() { if (instantiationQueue.Count > 0) { // Make our new objects in batches and tell observers about it int batchCount = CreateGameObjects ? Math.Min(InstantiationBatchRate, instantiationQueue.Count) : instantiationQueue.Count; for (int i = 0; i < batchCount; ++i) { if (instantiationQueue.TryDequeue(out SpatialAwarenessSceneObject saso) && CreateGameObjects) { InstantiateSceneObject(saso); SendSceneObjectAdded(saso, saso.Id); } } } } /// public override void Destroy() { cancelTokenSource.Cancel(); CleanupObserver(); } #endif // SCENE_UNDERSTANDING_PRESENT && UNITY_WSA #endregion IMixedRealityService #if SCENE_UNDERSTANDING_PRESENT && UNITY_WSA #region BaseService /// protected override void Dispose(bool disposing) { if (disposed) { return; } base.Dispose(disposing); Suspend(); if (disposing) { CleanupInstantiatedSceneObjects(); } disposed = true; } #endregion BaseService #endif // SCENE_UNDERSTANDING_PRESENT && UNITY_WSA #region IMixedRealitySpatialAwarenessObserver /// public override void Resume() { #if SCENE_UNDERSTANDING_PRESENT && UNITY_WSA updateTimer.Enabled = true; #endif // SCENE_UNDERSTANDING_PRESENT && UNITY_WSA } /// public override void Suspend() { #if SCENE_UNDERSTANDING_PRESENT && UNITY_WSA if (updateTimer != null) { updateTimer.Enabled = false; } #endif // SCENE_UNDERSTANDING_PRESENT && UNITY_WSA } #endregion IMixedRealitySpatialAwarenessObserver #region IMixedRealitySpatialAwarenessSceneUnderstandingObserver /// public IReadOnlyDictionary SceneObjects { #if SCENE_UNDERSTANDING_PRESENT && UNITY_WSA get => sceneObjects; #else // SCENE_UNDERSTANDING_PRESENT && UNITY_WSA get => null; #endif // SCENE_UNDERSTANDING_PRESENT && UNITY_WSA } /// public SpatialAwarenessSurfaceTypes SurfaceTypes { get; set; } /// public bool ShouldLoadFromFile { get; set; } /// public int InstantiationBatchRate { get; set; } /// public bool InferRegions { get; set; } /// public bool RequestMeshData { get; set; } /// public bool RequestPlaneData { get; set; } /// public bool RequestOcclusionMask { get; set; } /// public bool UsePersistentObjects { get; set; } /// public float QueryRadius { get; set; } /// public Vector2Int OcclusionMaskResolution { get; set; } /// public bool CreateGameObjects { get; set; } /// public bool AutoUpdate { get; set; } /// public bool OrientScene { get; set; } /// public SpatialAwarenessMeshLevelOfDetail WorldMeshLevelOfDetail { get; set; } /// public void SaveScene(string filenamePrefix) { #if WINDOWS_UWP && SCENE_UNDERSTANDING_PRESENT Task.Run(() => SaveToFile(filenamePrefix)).ContinueWith(t => { Debug.LogError($"{t.Exception.InnerException.GetType().Name}: {t.Exception.InnerException.Message} {t.Exception.InnerException.StackTrace}"); }, TaskContinuationOptions.OnlyOnFaulted); #else // WINDOWS_UWP && SCENE_UNDERSTANDING_PRESENT Debug.LogWarning("SaveScene() only supported at runtime! Ignoring request."); #endif // WINDOWS_UWP && SCENE_UNDERSTANDING_PRESENT } #endregion IMixedRealitySpatialAwarenessSceneUnderstandingObserver #region IMixedRealityOnDemandObserver /// public float FirstAutoUpdateDelay { get; set; } /// public void UpdateOnDemand() { #if SCENE_UNDERSTANDING_PRESENT && UNITY_WSA if (!MixedRealityToolkit.Instance.ActiveProfile.IsSpatialAwarenessSystemEnabled || (Service == null)) { return; } observerState = ObserverState.GetScene; #endif // SCENE_UNDERSTANDING_PRESENT && UNITY_WSA } /// public bool UpdateOnceInitialized { get; set; } #endregion IMixedRealityOnDemandObserver #region Public Profile /// /// /// public Material DefaultMaterial { get; set; } // Need references so they are included for runtime /// /// /// public Material DefaultWorldMeshMaterial { get; set; } // Need references so they are included for runtime private byte[] sceneBytes; private TextAsset serializedScene = null; /// /// The saved scene understanding file /// public TextAsset SerializedScene { get { return serializedScene; } set { if (serializedScene != value) { serializedScene = value; sceneBytes = serializedScene.bytes; } } } #endregion Profile #if SCENE_UNDERSTANDING_PRESENT && UNITY_WSA #region Private Fields private Task task; private readonly Dictionary sceneObjects = new Dictionary(256); private System.Timers.Timer firstUpdateTimer = null; private System.Timers.Timer updateTimer = null; private Dictionary> cachedSceneQuads = new Dictionary>(256); private ConcurrentQueue instantiationQueue = new ConcurrentQueue(); private readonly Mesh normalizedQuadMesh = new Mesh(); private string surfaceTypeName; private readonly CancellationTokenSource cancelTokenSource = new CancellationTokenSource(); private System.Numerics.Matrix4x4 correctOrientation = System.Numerics.Matrix4x4.Identity; private readonly List convertedObjects = new List(256); private readonly Dictionary IdToGuidLookup = new Dictionary(); private enum ObserverState { Idle = 0, GetScene, GetSceneTransform, Working } private ObserverState observerState; private CancellationToken cancelToken; private Guid sceneOriginId; private System.Numerics.Matrix4x4 sceneToWorldTransformMatrix; private List filteredSelectedSurfaceTypesResult = new List(128); private Texture defaultTexture; #if WINDOWS_UWP || MSFT_OPENXR private bool isOpenXRLoaderActive; #endif // WINDOWS_UWP || MSFT_OPENXR #if MSFT_OPENXR private bool isOpenXRRemotingConnected; #endif // MSFT_OPENXR #endregion Private Fields #endif // SCENE_UNDERSTANDING_PRESENT && UNITY_WSA #region Public Methods /// /// Gets the occlusion mask from a scene quad /// /// Guid of the quad /// Width of the mask /// Height of the mask /// Mask result /// Returns false if fails to get the mask public bool TryGetOcclusionMask( int quadId, ushort textureWidth, ushort textureHeight, out byte[] mask) { mask = null; #if SCENE_UNDERSTANDING_PRESENT && UNITY_WSA Tuple result; if (!cachedSceneQuads.TryGetValue(quadId, out result)) { return false; } SceneQuad quad = result.Item1; SceneObject sceneObject = result.Item2; byte[] maskResult = new byte[textureWidth * textureHeight]; quad.GetSurfaceMask(textureWidth, textureHeight, maskResult); mask = maskResult; return true; #else return false; #endif // SCENE_UNDERSTANDING_PRESENT && UNITY_WSA } /// public bool TryFindCentermostPlacement( int quadId, Vector2 objExtents, out Vector3 placementPosOnQuad) { placementPosOnQuad = Vector3.zero; #if SCENE_UNDERSTANDING_PRESENT && UNITY_WSA Tuple result; if (!cachedSceneQuads.TryGetValue(quadId, out result)) { placementPosOnQuad = Vector2.zero; return false; } SceneQuad quad = result.Item1; SceneObject sceneObject = result.Item2; System.Numerics.Vector2 ext = new System.Numerics.Vector2(objExtents.x, objExtents.y); System.Numerics.Vector2 centerPosition = new System.Numerics.Vector2(); quad.FindCentermostPlacement(ext, out centerPosition); // best placement origin is top left (2d sheet of paper) centerPosition -= quad.Extents / new System.Numerics.Vector2(2.0f); var centerUnity = new Vector3(centerPosition.X, centerPosition.Y, 0); placementPosOnQuad = (sceneObject.GetLocationAsMatrix() * sceneToWorldTransformMatrix * correctOrientation).ToUnity().MultiplyPoint(centerUnity); return true; #else return false; #endif // SCENE_UNDERSTANDING_PRESENT && UNITY_WSA } #endregion Public Methods #if SCENE_UNDERSTANDING_PRESENT && UNITY_WSA #region Private private MixedRealitySpatialAwarenessEventData sceneEventData = null; private static readonly ExecuteEvents.EventFunction> OnSceneObjectAdded = delegate (IMixedRealitySpatialAwarenessObservationHandler handler, BaseEventData eventData) { MixedRealitySpatialAwarenessEventData spatialEventData = ExecuteEvents.ValidateEventData>(eventData); handler.OnObservationAdded(spatialEventData); }; private static readonly ExecuteEvents.EventFunction> OnSceneObjectUpdated = delegate (IMixedRealitySpatialAwarenessObservationHandler handler, BaseEventData eventData) { MixedRealitySpatialAwarenessEventData spatialEventData = ExecuteEvents.ValidateEventData>(eventData); handler.OnObservationUpdated(spatialEventData); }; private static readonly ExecuteEvents.EventFunction> OnSceneObjectRemoved = delegate (IMixedRealitySpatialAwarenessObservationHandler handler, BaseEventData eventData) { MixedRealitySpatialAwarenessEventData spatialEventData = ExecuteEvents.ValidateEventData>(eventData); handler.OnObservationRemoved(spatialEventData); }; /// /// Sends SceneObject Added event via /// /// The SpatialAwarenessSceneObject being created /// the id associated with the protected virtual void SendSceneObjectAdded(SpatialAwarenessSceneObject sceneObj, int id) { sceneEventData.Initialize(this, id, sceneObj); Service?.HandleEvent(sceneEventData, OnSceneObjectAdded); } /// /// Sends SceneObject Updated event via /// /// The SpatialAwarenessSceneObject being updated /// the id associated with the protected virtual void SendSceneObjectUpdated(SpatialAwarenessSceneObject sceneObj, int id) { sceneEventData.Initialize(this, id, sceneObj); Service?.HandleEvent(sceneEventData, OnSceneObjectUpdated); } /// /// Sends SceneObject Removed event via /// /// the id associated with the removed SceneObject protected virtual void SendSceneObjectRemoved(int id) { sceneEventData.Initialize(this, id, null); Service?.HandleEvent(sceneEventData, OnSceneObjectRemoved); } private async Task RequestAccess() { return await SceneObserver.RequestAccessAsync(); } /// /// Sets up and starts update timers /// private void StartUpdateTimers() { updateTimer = new System.Timers.Timer { Interval = Math.Max(UpdateInterval, Mathf.Epsilon) * 1000.0, // convert to milliseconds }; updateTimer.Elapsed += (sender, e) => { if (AutoUpdate) { observerState = ObserverState.GetScene; } }; // If AutoUpdate we set up the timer to wait until we pass FirstAutoUpdateDelay before starting the first automatic update if (AutoUpdate) { firstUpdateTimer = new System.Timers.Timer() { Interval = Math.Max(FirstAutoUpdateDelay, Mathf.Epsilon) * 1000.0, // convert to milliseconds AutoReset = false }; // After an initial delay, start a load once or the auto update firstUpdateTimer.Elapsed += (sender, e) => { updateTimer.Start(); observerState = ObserverState.GetScene; }; firstUpdateTimer.Start(); } // If AutoUpdate is false then can we start the update timer right away. // Note we still want to start this timer in case the user set AutoUpdate to true later else { updateTimer.Start(); } } /// /// Creates a quad based on extents /// /// Mesh to contain the quad /// Length of the quad /// Width of the quad private void CreateQuadFromExtents(Mesh mesh, float x, float y) { List vertices = new List() { new Vector3(-x / 2, -y / 2, 0), new Vector3( x / 2, -y / 2, 0), new Vector3(-x / 2, y / 2, 0), new Vector3( x / 2, y / 2, 0) }; Vector2[] quadUVs = new Vector2[] { new Vector2(0, 0), new Vector2(1, 0), new Vector2(0, 1), new Vector2(1, 1) }; int[] quadTriangles = new int[] { 0, 3, 1, 0, 2, 3, // 1, 3, 0, // 3, 2, 0 }; mesh.SetVertices(vertices); mesh.SetIndices(quadTriangles, MeshTopology.Triangles, 0); mesh.SetUVs(0, new List(quadUVs)); } /// /// Runs the observer asynchronously /// /// CancellationToken of the task /// The async task private async Task RunObserverAsync(CancellationToken cancellationToken) { Scene scene = null; Scene previousScene = null; List sasos; while (!cancellationToken.IsCancellationRequested) { switch (observerState) { case ObserverState.Idle: await new WaitForUpdate(); continue; case ObserverState.GetScene: observerState = ObserverState.Working; while (CreateGameObjects && instantiationQueue.Count > 0) { await new WaitForUpdate(); } await new WaitForBackgroundThread(); { scene = await GetSceneAsync(previousScene); previousScene = scene; sceneOriginId = scene.OriginSpatialGraphNodeId; } await new WaitForUpdate(); System.Numerics.Matrix4x4? transformResult = GetSceneToWorldTransform(); if (transformResult.HasValue) { sceneToWorldTransformMatrix = transformResult.Value; } else { await new WaitForUpdate(); observerState = ObserverState.GetScene; continue; } if (!UsePersistentObjects) { ClearObservations(); } if (OrientScene && Application.isEditor) { System.Numerics.Vector3 toUp; await new WaitForBackgroundThread(); { toUp = ToUpFromBiggestFloor(scene.SceneObjects); } await new WaitForUpdate(); var floorNormalUnity = new Vector3(toUp.X, toUp.Y, toUp.Z); // Get the rotation between the floor normal and Unity world's up vector. var upRotation = Quaternion.FromToRotation(floorNormalUnity, Vector3.down); correctOrientation = Matrix4x4.TRS(Vector3.zero, upRotation, Vector3.one).ToSystemNumerics(); } await new WaitForBackgroundThread(); { sasos = await ConvertSceneObjectsAsync(scene); } await new WaitForUpdate(); await new WaitForBackgroundThread(); { AddUniqueTo(sasos, instantiationQueue); } await new WaitForUpdate(); // Add new objects to observer // notify subscribers of event foreach (var saso in sasos) { if (!sceneObjects.ContainsKey(saso.Id)) { sceneObjects.Add(saso.Id, saso); // If creating GameObjects, delay the sending of the event until the creation is finished if (!CreateGameObjects) { SendSceneObjectAdded(saso, saso.Id); } } else { if (CreateGameObjects) { UpdateInstantiatedSceneObject(sceneObjects[saso.Id], saso); } sceneObjects[saso.Id] = saso; SendSceneObjectUpdated(saso, saso.Id); } } List removedSasoIds = new List(); foreach (var saso in sceneObjects.Values) { if (!sasos.Contains(saso)) { removedSasoIds.Add(saso.Id); } } foreach (var id in removedSasoIds) { SendSceneObjectRemoved(id); if (CreateGameObjects) { UpdateInstantiatedSceneObject(sceneObjects[id], null); } sceneObjects.Remove(id); } if (observerState == ObserverState.Working) { observerState = ObserverState.Idle; } continue; default: await new WaitForUpdate(); continue; } } } /// /// Gets the matrix representing the transform from scene space to world space /// /// The transform matrix private System.Numerics.Matrix4x4? GetSceneToWorldTransform() { var result = System.Numerics.Matrix4x4.Identity; #if WINDOWS_UWP // On HoloLens 2 device if (isOpenXRLoaderActive) #elif MSFT_OPENXR // In editor and using OpenXR if (isOpenXRLoaderActive && isOpenXRRemotingConnected && !ShouldLoadFromFile) #else // All other cases if (false) #endif // WINDOWS_UWP { #if MSFT_OPENXR SpatialGraphNode node = SpatialGraphNode.FromStaticNodeId(sceneOriginId); if (node.TryLocate(FrameTime.OnUpdate, out Pose pose)) { result = Matrix4x4.TRS(pose.position, pose.rotation, Vector3.one).ToSystemNumerics(); } else { return null; } #endif // MSFT_OPENXR } else { #if WINDOWS_UWP SpatialCoordinateSystem sceneOrigin = SpatialGraphInteropPreview.CreateCoordinateSystemForNode(sceneOriginId); SpatialCoordinateSystem worldOrigin = WindowsMixedReality.WindowsMixedRealityUtilities.SpatialCoordinateSystem; var sceneToWorld = sceneOrigin.TryGetTransformTo(worldOrigin); if (sceneToWorld.HasValue) { result = sceneToWorld.Value; // numerics } else { return null; } #endif // WINDOWS_UWP } return result; } /// /// Gets scene asynchronously from file or SceneObserver /// /// The previous scene /// The retrieved scene private async Task GetSceneAsync(Scene previousScene) { Scene scene = null; if (Application.isEditor && ShouldLoadFromFile) { if (sceneBytes == null) { Debug.LogError("sceneBytes is null!"); } // Move onto a background thread for the expensive scene loading stuff if (UsePersistentObjects && previousScene != null) { scene = Scene.Deserialize(sceneBytes, previousScene); } else { // This happens first time through as we have no history yet scene = Scene.Deserialize(sceneBytes); } } else { SceneQuerySettings sceneQuerySettings = new SceneQuerySettings() { EnableSceneObjectQuads = RequestPlaneData, EnableSceneObjectMeshes = RequestMeshData, EnableOnlyObservedSceneObjects = !InferRegions, EnableWorldMesh = SurfaceTypes.IsMaskSet(SpatialAwarenessSurfaceTypes.World), RequestedMeshLevelOfDetail = LevelOfDetailToMeshLOD(WorldMeshLevelOfDetail) }; if (UsePersistentObjects && previousScene != null) { scene = await SceneObserver.ComputeAsync(sceneQuerySettings, QueryRadius, previousScene); } else { scene = await SceneObserver.ComputeAsync(sceneQuerySettings, QueryRadius); } } return scene; } /// /// Converts a to a MRTK/platform agnostic SpatialAwarenessSceneObject. /// /// The to convert /// The converted SpatialAwarenessSceneObject. private SpatialAwarenessSceneObject ConvertSceneObject(SceneObject sceneObject) { int quadCount = sceneObject.Quads.Count; int meshCount = sceneObject.Meshes.Count; List quads = new List(quadCount); List meshes = new List(meshCount); if (RequestPlaneData) { SceneQuad sceneQuad = null; for (int i = 0; i < quadCount; ++i) { sceneQuad = sceneObject.Quads[i]; var quadIdKey = sceneQuad.Id; byte[] occlusionMaskBytes = null; if (RequestOcclusionMask) { occlusionMaskBytes = new byte[OcclusionMaskResolution.x * OcclusionMaskResolution.y]; sceneQuad.GetSurfaceMask((ushort)OcclusionMaskResolution.x, (ushort)OcclusionMaskResolution.y, occlusionMaskBytes); } var extents = new Vector2(sceneQuad.Extents.X, sceneQuad.Extents.Y); int hashedQuadId = quadIdKey.GetHashCode(); var quad = new SpatialAwarenessSceneObject.QuadData { Id = hashedQuadId, Extents = extents, OcclusionMask = occlusionMaskBytes }; if (IdToGuidLookup.ContainsKey(hashedQuadId) && IdToGuidLookup[hashedQuadId] != quadIdKey) { Debug.LogWarning("Possible collision"); } IdToGuidLookup[hashedQuadId] = quadIdKey; quads.Add(quad); // Store a cache so we can retrieve best position on plane later. if (!cachedSceneQuads.ContainsKey(hashedQuadId)) { cachedSceneQuads.Add(hashedQuadId, new Tuple(sceneQuad, sceneObject)); } } } if (RequestMeshData) { for (int i = 0; i < meshCount; ++i) { var meshData = MeshData(sceneObject.Meshes[i]); meshes.Add(meshData); } } // World space conversion System.Numerics.Matrix4x4 worldTransformMatrix = sceneObject.GetLocationAsMatrix() * sceneToWorldTransformMatrix * correctOrientation; System.Numerics.Vector3 worldTranslationSystem; System.Numerics.Quaternion worldRotationSystem; System.Numerics.Matrix4x4.Decompose(worldTransformMatrix, out _, out worldRotationSystem, out worldTranslationSystem); int hashedId = sceneObject.Id.GetHashCode(); var result = SpatialAwarenessSceneObject.Create( hashedId, SpatialAwarenessSurfaceType(sceneObject.Kind), worldTranslationSystem.ToUnityVector3(), worldRotationSystem.ToUnityQuaternion(), quads, meshes); if (IdToGuidLookup.ContainsKey(hashedId) && IdToGuidLookup[hashedId] != sceneObject.Id) { Debug.LogWarning("Possible collision"); } IdToGuidLookup[hashedId] = sceneObject.Id; return result; } /// /// Converts all SurfaceType-matching s in a scene to SpatialAwarenessSceneObjects. /// /// The scene containing s to convert /// Task containing the resulting list of converted SpatialAwarenessSceneObjects. private async Task> ConvertSceneObjectsAsync(Scene scene) { convertedObjects.Clear(); if (scene.SceneObjects.Count == 0) { return convertedObjects; } var filteredSceneObjects = FilterSelectedSurfaceTypes(scene.SceneObjects); int sceneObjectCount = filteredSceneObjects.Count; for (int i = 0; i < sceneObjectCount; ++i) { var saso = ConvertSceneObject(filteredSceneObjects[i]); convertedObjects.Add(saso); } await Task.Yield(); return convertedObjects; } /// /// Filters SceneObject of selected SurfaceTypes /// /// List of SceneObjects to be filtered /// The filtered list private List FilterSelectedSurfaceTypes(IReadOnlyList newObjects) { filteredSelectedSurfaceTypesResult.Clear(); int count = newObjects.Count; for (int i = 0; i < count; ++i) { if (!SurfaceTypes.IsMaskSet(SpatialAwarenessSurfaceType(newObjects[i].Kind))) { continue; } filteredSelectedSurfaceTypesResult.Add(newObjects[i]); } return filteredSelectedSurfaceTypesResult; } /// /// Adds new SpatialAwarenessSceneObjects to the existing queue /// /// List of SpatialAwarenessSceneObjects to be added /// The queue where new SpatialAwarenessSceneObjects will be added private void AddUniqueTo(List newObjects, ConcurrentQueue existingQueue) { int length = newObjects.Count; for (int i = 0; i < length; ++i) { if (!sceneObjects.ContainsKey(newObjects[i].Id)) { existingQueue.Enqueue(newObjects[i]); } } } /// /// Instantiate a SceneObject in the scene /// /// The SpatialAwarenessSceneObject to instantiate private void InstantiateSceneObject(SpatialAwarenessSceneObject saso) { // Until this point the SASO has been a data representation surfaceTypeName = $"{saso.SurfaceType} {saso.Id}"; saso.GameObject = new GameObject(surfaceTypeName) { layer = DefaultPhysicsLayer }; saso.GameObject.transform.SetParent(ObservedObjectParent.transform); saso.GameObject.transform.localPosition = saso.Position; saso.GameObject.transform.localRotation = saso.Rotation; saso.GameObject.transform.localScale = Vector3.one; // Make GameObjects for Quads and Meshes if (RequestPlaneData) { // Add MeshFilter, attach shared quad and scale it // later, we can update scale of existing quads if they change size // (as opposed to modifying the vertices directly, when persisting objects) int quadCount = saso.Quads.Count; for (int i = 0; i < quadCount; ++i) { var quad = saso.Quads[i]; InstantiateQuad(saso, quad); } } if (RequestMeshData) { int meshCount = saso.Meshes.Count; for (int i = 0; i < meshCount; ++i) { var meshAlias = saso.Meshes[i]; InstantiateMesh(saso, meshAlias); } } } /// /// Instantiate a Mesh GameObject in the scene /// /// The SpatialAwarenessSceneObject containing the mesh to instantiate /// The MeshData object to instantiate private void InstantiateMesh(SpatialAwarenessSceneObject saso, SpatialAwarenessSceneObject.MeshData mesh) { var meshGo = new GameObject($"Mesh {mesh.Id}"); mesh.GameObject = meshGo; var meshFilter = meshGo.AddComponent(); meshFilter.mesh = UnityMeshFromMeshData(mesh); var meshRenderer = meshGo.AddComponent(); meshGo.AddComponent(); if (DefaultMaterial) { meshRenderer.sharedMaterial = DefaultMaterial; } if (saso.SurfaceType == SpatialAwarenessSurfaceTypes.World && DefaultWorldMeshMaterial) { meshRenderer.sharedMaterial = DefaultWorldMeshMaterial; } meshGo.transform.SetParent(saso.GameObject.transform, false); } /// /// Instantiate a Quad GameObject in the scene /// /// The SpatialAwarenessSceneObject containing the quad to instantiate /// The Quad object to instantiate private void InstantiateQuad(SpatialAwarenessSceneObject saso, SpatialAwarenessSceneObject.QuadData quad) { var quadGo = new GameObject($"Quad {quad.Id}"); quad.GameObject = quadGo; var meshFilter = quadGo.AddComponent(); meshFilter.mesh = normalizedQuadMesh; var meshRenderer = quadGo.AddComponent(); quadGo.AddComponent(); if (DefaultMaterial) { meshRenderer.sharedMaterial = DefaultMaterial; if (defaultTexture == null) { defaultTexture = DefaultMaterial.mainTexture; } } if (RequestOcclusionMask) { if (quad.OcclusionMask != null) { var occlusionTexture = OcclusionTexture(quad.OcclusionMask); meshRenderer.material.mainTexture = occlusionTexture; } } else { meshRenderer.material.mainTexture = defaultTexture; } quadGo.transform.SetParent(saso.GameObject.transform); quadGo.transform.localPosition = UnityEngine.Vector3.zero; quadGo.transform.localRotation = UnityEngine.Quaternion.identity; quadGo.transform.localScale = new UnityEngine.Vector3(quad.Extents.x, quad.Extents.y, 0); } /// /// Update an instantiated SpatialAwarenessSceneObject in the scene /// /// The existing SpatialAwarenessSceneObject in the scene /// The new SpatialAwarenessSceneObject with updated info private void UpdateInstantiatedSceneObject(SpatialAwarenessSceneObject existingSaso, SpatialAwarenessSceneObject newSaso) { if (newSaso == null) { GameObject.Destroy(existingSaso.GameObject); return; } newSaso.GameObject = existingSaso.GameObject; newSaso.GameObject.transform.localPosition = newSaso.Position; newSaso.GameObject.transform.localRotation = newSaso.Rotation; // Update GameObjects for Quads and Meshes if (RequestPlaneData) { int i = 0; while (i < existingSaso.Quads.Count && i < newSaso.Quads.Count) { var gameObject = newSaso.Quads[i].GameObject = existingSaso.Quads[i].GameObject; gameObject.name = $"Quad {newSaso.Quads[i].Id}"; gameObject.transform.localScale = new Vector3(newSaso.Quads[i].Extents.x, newSaso.Quads[i].Extents.y, 0); if (RequestOcclusionMask && ((existingSaso.Quads[i].OcclusionMask == null || newSaso.Quads[i].OcclusionMask == null) || !existingSaso.Quads[i].OcclusionMask.SequenceEqual(newSaso.Quads[i].OcclusionMask))) { var meshRender = newSaso.Quads[i].GameObject.GetComponent(); meshRender.enabled = true; if (newSaso.Quads[i].OcclusionMask != null) { var occlusionTexture = OcclusionTexture(newSaso.Quads[i].OcclusionMask); meshRender.material.mainTexture = occlusionTexture; } else { meshRender.enabled = false; } } i++; } if (existingSaso.Quads.Count < newSaso.Quads.Count) { for (; i < newSaso.Quads.Count; i++) { InstantiateQuad(newSaso, newSaso.Quads[i]); } } else { for (; i < existingSaso.Quads.Count; i++) { GameObject.Destroy(existingSaso.Quads[i].GameObject); } } } if (RequestMeshData) { int i = 0; while (i < existingSaso.Meshes.Count && i < newSaso.Meshes.Count) { var gameObject = newSaso.Meshes[i].GameObject = existingSaso.Meshes[i].GameObject; gameObject.name = $"Mesh {newSaso.Meshes[i].Id}"; if (DefaultWorldMeshMaterial && existingSaso.SurfaceType != newSaso.SurfaceType) { if (newSaso.SurfaceType == SpatialAwarenessSurfaceTypes.World) { newSaso.Meshes[i].GameObject.GetComponent().sharedMaterial = DefaultWorldMeshMaterial; } else if (existingSaso.SurfaceType == SpatialAwarenessSurfaceTypes.World) { newSaso.Meshes[i].GameObject.GetComponent().sharedMaterial = DefaultMaterial; } } if (!existingSaso.Meshes[i].UVs.SequenceEqual(newSaso.Meshes[i].UVs) || !existingSaso.Meshes[i].Indices.SequenceEqual(newSaso.Meshes[i].Indices) || !existingSaso.Meshes[i].Vertices.SequenceEqual(newSaso.Meshes[i].Vertices)) { var meshFilter = newSaso.Meshes[i].GameObject.GetComponent(); meshFilter.mesh = UnityMeshFromMeshData(newSaso.Meshes[i]); } i++; } if (existingSaso.Meshes.Count < newSaso.Meshes.Count) { for (; i < newSaso.Meshes.Count; i++) { InstantiateMesh(newSaso, newSaso.Meshes[i]); } } else { for (; i < existingSaso.Meshes.Count; i++) { GameObject.Destroy(existingSaso.Meshes[i].GameObject); } } } } /// /// Generates occlusion texture from the occlusion mask /// /// The occlusion mask to use /// The generated texture private Texture2D OcclusionTexture(byte[] textureBytes) { Assert.IsNotNull(textureBytes); // Create a new texture. Texture2D result = new Texture2D(OcclusionMaskResolution.x, OcclusionMaskResolution.y); result.wrapMode = TextureWrapMode.Clamp; // Transfer the invalidation mask onto the texture. Color[] pixels = result.GetPixels(); var numPixels = pixels.Length; for (int i = 0; i < numPixels; ++i) { var value = textureBytes[i]; switch (value) { case (byte)SceneRegionSurfaceKind.NotSurface: pixels[i] = Color.clear; break; case (byte)SceneRegionSurfaceKind.SurfaceObserved: pixels[i] = Color.cyan; break; case (byte)SceneRegionSurfaceKind.SurfaceInferred: pixels[i] = Color.yellow; break; default: Debug.LogWarning($"Got unknown surface kind {value.ToString()}"); pixels[i] = Color.magenta; break; } } result.SetPixels(pixels); result.Apply(true); return result; } /// /// Destroy the instantiated SceneObjects /// private void CleanupInstantiatedSceneObjects() { if (ObservedObjectParent != null) { int kidCount = ObservedObjectParent.transform.childCount; for (int i = 0; i < kidCount; ++i) { UnityEngine.Object.Destroy(ObservedObjectParent.transform.GetChild(i).gameObject); } } } /// public override void ClearObservations() { base.ClearObservations(); cachedSceneQuads.Clear(); CleanupInstantiatedSceneObjects(); instantiationQueue = new ConcurrentQueue(); foreach (var sceneObject in sceneObjects) { SendSceneObjectRemoved(sceneObject.Key); } sceneObjects.Clear(); IdToGuidLookup.Clear(); } #if WINDOWS_UWP /// /// Saves the 's data stream as a file for later use /// /// the private async void SaveToFile(string prefix) { SceneQuerySettings sceneQuerySettings = new SceneQuerySettings() { EnableSceneObjectQuads = true, EnableSceneObjectMeshes = true, EnableOnlyObservedSceneObjects = false, EnableWorldMesh = true, RequestedMeshLevelOfDetail = LevelOfDetailToMeshLOD(WorldMeshLevelOfDetail) }; var serializedScene = await SceneObserver.ComputeSerializedAsync(sceneQuerySettings, QueryRadius); var bytes = new byte[serializedScene.Size]; serializedScene.GetData(bytes); var timestamp = DateTime.Now.ToString("yyyyMMdd_hhmmss"); var filename = $"SceneUnderStanding_{timestamp}.bytes"; if (prefix != "") { filename = $"{prefix}_{timestamp}.bytes"; } StorageFolder folderLocation = ApplicationData.Current.LocalFolder; IStorageFile storageFile = await folderLocation.CreateFileAsync(filename); await FileIO.WriteBytesAsync(storageFile, bytes); } #endif // WINDOWS_UWP /// /// Converts a MRTK/platform agnostic to a . /// /// The to convert. /// The equivalent private SceneMeshLevelOfDetail LevelOfDetailToMeshLOD(SpatialAwarenessMeshLevelOfDetail levelOfDetail) { switch (levelOfDetail) { case SpatialAwarenessMeshLevelOfDetail.Custom: Debug.LogWarning("SceneUnderstanding LOD is set to custom, falling back to Medium"); return SceneMeshLevelOfDetail.Medium; case SpatialAwarenessMeshLevelOfDetail.Coarse: return SceneMeshLevelOfDetail.Coarse; case SpatialAwarenessMeshLevelOfDetail.Medium: return SceneMeshLevelOfDetail.Medium; case SpatialAwarenessMeshLevelOfDetail.Fine: return SceneMeshLevelOfDetail.Fine; case SpatialAwarenessMeshLevelOfDetail.Unlimited: return SceneMeshLevelOfDetail.Unlimited; default: throw new NotImplementedException(); } } /// /// Helper to convert from Right hand to Left hand coordinates using . /// https://docs.microsoft.com/en-us/windows/mixed-reality/unity-xrdevice-advanced#converting-between-coordinate-systems /// /// The Right handed . /// The in left hand form. public static System.Numerics.Matrix4x4 RightToLeftHanded(System.Numerics.Matrix4x4 matrix) { matrix.M13 = -matrix.M13; matrix.M23 = -matrix.M23; matrix.M43 = -matrix.M43; matrix.M31 = -matrix.M31; matrix.M32 = -matrix.M32; matrix.M34 = -matrix.M34; return matrix; } /// /// Generates Unity Mesh from MeshData /// /// The MeshData to get data from /// The resulting Unity Mesh private static Mesh UnityMeshFromMeshData(SpatialAwarenessSceneObject.MeshData meshData) { Mesh unityMesh = new Mesh(); // Unity has a limit of 65,535 vertices in a mesh. // This limit exists because by default Unity uses 16-bit index buffers. // Starting with 2018.1, Unity allows one to use 32-bit index buffers. if (meshData.Vertices.Length > 65535) { unityMesh.indexFormat = UnityEngine.Rendering.IndexFormat.UInt32; } unityMesh.SetVertices(new List(meshData.Vertices)); unityMesh.SetIndices(meshData.Indices, MeshTopology.Triangles, 0); unityMesh.SetUVs(0, new List(meshData.UVs)); unityMesh.RecalculateNormals(); return unityMesh; } /// /// Orients the root game object, such that the Scene Understanding floor lies on the Unity world's X-Z plane. /// The floor type with the largest area is chosen as the reference. /// /// Scene Understanding scene. private System.Numerics.Vector3 ToUpFromBiggestFloor(IReadOnlyList sasos) { float areaForlargestFloorSoFar = 0; SceneObject floorSceneObject = null; SceneQuad floorQuad = null; var result = System.Numerics.Vector3.Zero; // Find the largest floor quad. var count = sasos.Count; for (var i = 0; i < count; ++i) { if (sasos[i].Kind == SceneObjectKind.Floor) { var quads = sasos[i].Quads; Assert.IsNotNull(quads); var qcount = quads.Count; for (int j = 0; j < qcount; j++) { float quadArea = quads[j].Extents.X * quads[j].Extents.Y; if (quadArea > areaForlargestFloorSoFar) { areaForlargestFloorSoFar = quadArea; floorSceneObject = sasos[i]; floorQuad = quads[j]; } } } } if (floorQuad != null) { // Compute the floor quad's normal. float halfWidthMeters = floorQuad.Extents.X * .5f; float halfHeightMeters = floorQuad.Extents.Y * .5f; System.Numerics.Vector3 point1 = new System.Numerics.Vector3(-halfWidthMeters, -halfHeightMeters, 0); System.Numerics.Vector3 point2 = new System.Numerics.Vector3(halfWidthMeters, -halfHeightMeters, 0); System.Numerics.Vector3 point3 = new System.Numerics.Vector3(-halfWidthMeters, halfHeightMeters, 0); System.Numerics.Matrix4x4 objectToSceneOrigin = floorSceneObject.GetLocationAsMatrix(); objectToSceneOrigin = RightToLeftHanded(objectToSceneOrigin); System.Numerics.Vector3 tPoint1 = System.Numerics.Vector3.Transform(point1, objectToSceneOrigin); System.Numerics.Vector3 tPoint2 = System.Numerics.Vector3.Transform(point2, objectToSceneOrigin); System.Numerics.Vector3 tPoint3 = System.Numerics.Vector3.Transform(point3, objectToSceneOrigin); System.Numerics.Vector3 p21 = tPoint2 - tPoint1; System.Numerics.Vector3 p31 = tPoint3 - tPoint1; result = System.Numerics.Vector3.Cross(p21, p31); } return result; } /// /// Generates MeshData from SceneMesh /// /// The SceneMesh to get data from /// The resulting MeshData private SpatialAwarenessSceneObject.MeshData MeshData(SceneMesh sceneMesh) { // Indices var indices = new int[sceneMesh.TriangleIndexCount]; var meshIndices = new uint[sceneMesh.TriangleIndexCount]; sceneMesh.GetTriangleIndices(meshIndices); var ilength = meshIndices.Length; for (int i = 0; i < ilength; ++i) { indices[i] = (int)meshIndices[i]; } // Vertices var vertices = new Vector3[sceneMesh.VertexCount]; var uvs = new Vector2[sceneMesh.VertexCount]; var meshVertices = new System.Numerics.Vector3[sceneMesh.VertexCount]; sceneMesh.GetVertexPositions(meshVertices); var vertexCount = meshVertices.Length; var minx = meshVertices[0].X; var miny = meshVertices[0].Y; var maxx = minx; var maxy = miny; for (int i = 0; i < vertexCount; ++i) { var x = meshVertices[i].X; var y = meshVertices[i].Y; vertices[i] = new Vector3(x, y, -meshVertices[i].Z); minx = Math.Min(minx, x); miny = Math.Min(miny, y); maxx = Math.Max(maxx, x); maxy = Math.Max(maxy, y); } // UVs - planar square projection float smallestDimension = Math.Min(minx, miny); float biggestDimension = Math.Max(maxx, maxy); for (int i = 0; i < vertexCount; ++i) { uvs[i] = new Vector2( Mathf.InverseLerp(smallestDimension, biggestDimension, vertices[i].x), Mathf.InverseLerp(smallestDimension, biggestDimension, vertices[i].y)); } int hashedId = sceneMesh.Id.GetHashCode(); var result = new SpatialAwarenessSceneObject.MeshData { Indices = indices, Vertices = vertices, Id = hashedId, UVs = uvs }; if (IdToGuidLookup.ContainsKey(hashedId) && IdToGuidLookup[hashedId] != sceneMesh.Id) { Debug.LogWarning("Possible collision"); } IdToGuidLookup[hashedId] = sceneMesh.Id; return result; } /// /// Converts a to a MRTK/platform agnostic /// /// The to convert. /// The equivalent private SpatialAwarenessSurfaceTypes SpatialAwarenessSurfaceType(SceneObjectKind label) { switch (label) { case SceneObjectKind.Background: return SpatialAwarenessSurfaceTypes.Background; case SceneObjectKind.Wall: return SpatialAwarenessSurfaceTypes.Wall; case SceneObjectKind.Floor: return SpatialAwarenessSurfaceTypes.Floor; case SceneObjectKind.Ceiling: return SpatialAwarenessSurfaceTypes.Ceiling; case SceneObjectKind.Platform: return SpatialAwarenessSurfaceTypes.Platform; case SceneObjectKind.World: return SpatialAwarenessSurfaceTypes.World; case SceneObjectKind.CompletelyInferred: return SpatialAwarenessSurfaceTypes.Inferred; case SceneObjectKind.Unknown: default: return SpatialAwarenessSurfaceTypes.Unknown; } } #endregion Private #endif // SCENE_UNDERSTANDING_PRESENT && UNITY_WSA } }