// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using Microsoft.MixedReality.Toolkit.Utilities;
using Microsoft.MixedReality.Toolkit.Utilities.Gltf.Schema;
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using UnityEngine;
using UnityEngine.Rendering;
#if WINDOWS_UWP
using Windows.Storage;
using Windows.Storage.Streams;
#endif // WINDOWS_UWP
namespace Microsoft.MixedReality.Toolkit.Utilities.Gltf.Serialization
{
public static class ConstructGltf
{
private static readonly WaitForUpdate Update = new WaitForUpdate();
private static readonly WaitForBackgroundThread BackgroundThread = new WaitForBackgroundThread();
private static readonly int SrcBlendId = Shader.PropertyToID("_SrcBlend");
private static readonly int DstBlendId = Shader.PropertyToID("_DstBlend");
private static readonly int ZWriteId = Shader.PropertyToID("_ZWrite");
private static readonly int ModeId = Shader.PropertyToID("_Mode");
private static readonly int EmissionMapId = Shader.PropertyToID("_EmissionMap");
private static readonly int EmissionColorId = Shader.PropertyToID("_EmissionColor");
private static readonly int MetallicGlossMapId = Shader.PropertyToID("_MetallicGlossMap");
private static readonly int GlossinessId = Shader.PropertyToID("_Glossiness");
private static readonly int MetallicId = Shader.PropertyToID("_Metallic");
private static readonly int BumpMapId = Shader.PropertyToID("_BumpMap");
private static readonly int EmissiveColorId = Shader.PropertyToID("_EmissiveColor");
private static readonly int ChannelMapId = Shader.PropertyToID("_ChannelMap");
private static readonly int SmoothnessId = Shader.PropertyToID("_Smoothness");
private static readonly int NormalMapId = Shader.PropertyToID("_NormalMap");
private static readonly int NormalMapScaleId = Shader.PropertyToID("_NormalMapScale");
private static readonly int CullModeId = Shader.PropertyToID("_CullMode");
///
/// Constructs the glTF Object.
///
/// The new GameObject of the final constructed
public static async void Construct(this GltfObject gltfObject)
{
await gltfObject.ConstructAsync();
}
///
/// Constructs the glTF Object.
///
/// The new GameObject of the final constructed
public static async Task ConstructAsync(this GltfObject gltfObject)
{
if (!gltfObject.asset.version.Contains("2.0"))
{
Debug.LogWarning($"Expected glTF 2.0, but this asset is using {gltfObject.asset.version}");
return null;
}
if (gltfObject.UseBackgroundThread) { await Update; }
var rootObject = new GameObject($"glTF Scene {gltfObject.Name}");
rootObject.SetActive(false);
if (gltfObject.UseBackgroundThread) await BackgroundThread;
for (int i = 0; i < gltfObject.bufferViews?.Length; i++)
{
gltfObject.ConstructBufferView(gltfObject.bufferViews[i]);
}
for (int i = 0; i < gltfObject.textures?.Length; i++)
{
await gltfObject.ConstructTextureAsync(gltfObject.textures[i]);
}
for (int i = 0; i < gltfObject.materials?.Length; i++)
{
await gltfObject.ConstructMaterialAsync(gltfObject.materials[i], i);
}
if (gltfObject.scenes == null)
{
Debug.LogError($"No scenes found for {gltfObject.Name}");
}
if (gltfObject.UseBackgroundThread) await Update;
for (int i = 0; i < gltfObject.scenes?.Length; i++)
{
await gltfObject.ConstructSceneAsync(gltfObject.scenes[i], rootObject);
}
rootObject.SetActive(true);
return gltfObject.GameObjectReference = rootObject;
}
private static void ConstructBufferView(this GltfObject gltfObject, GltfBufferView bufferView)
{
bufferView.Buffer = gltfObject.buffers[bufferView.buffer];
if (bufferView.Buffer.BufferData == null &&
!string.IsNullOrEmpty(gltfObject.Uri) &&
!string.IsNullOrEmpty(bufferView.Buffer.uri))
{
var parentDirectory = Directory.GetParent(gltfObject.Uri).FullName;
bufferView.Buffer.BufferData = File.ReadAllBytes(Path.Combine(parentDirectory, bufferView.Buffer.uri));
}
}
private static async Task ConstructTextureAsync(this GltfObject gltfObject, GltfTexture gltfTexture)
{
if (gltfObject.UseBackgroundThread) await BackgroundThread;
if (gltfTexture.source >= 0)
{
GltfImage gltfImage = gltfObject.images[gltfTexture.source];
byte[] imageData = null;
Texture2D texture = null;
if (!string.IsNullOrEmpty(gltfObject.Uri) && !string.IsNullOrEmpty(gltfImage.uri))
{
var parentDirectory = Directory.GetParent(gltfObject.Uri).FullName;
var path = Path.Combine(parentDirectory, gltfImage.uri);
#if UNITY_EDITOR
if (gltfObject.UseBackgroundThread) await Update;
var projectPath = Path.GetFullPath(path).Replace(Path.GetFullPath(Application.dataPath), "Assets");
texture = UnityEditor.AssetDatabase.LoadAssetAtPath(projectPath);
if (gltfObject.UseBackgroundThread) await BackgroundThread;
#endif
if (texture == null)
{
#if WINDOWS_UWP
if (gltfObject.UseBackgroundThread)
{
try
{
var storageFile = await StorageFile.GetFileFromPathAsync(path);
if (storageFile != null)
{
var buffer = await FileIO.ReadBufferAsync(storageFile);
using (DataReader dataReader = DataReader.FromBuffer(buffer))
{
imageData = new byte[buffer.Length];
dataReader.ReadBytes(imageData);
}
}
}
catch (Exception e)
{
Debug.LogError(e.Message);
}
}
else
{
imageData = UnityEngine.Windows.File.ReadAllBytes(path);
}
#else
using (FileStream stream = File.Open(path, FileMode.Open))
{
imageData = new byte[stream.Length];
if (gltfObject.UseBackgroundThread)
{
await stream.ReadAsync(imageData, 0, (int)stream.Length);
}
else
{
stream.Read(imageData, 0, (int)stream.Length);
}
}
#endif
}
}
else
{
var imageBufferView = gltfObject.bufferViews[gltfImage.bufferView];
imageData = new byte[imageBufferView.byteLength];
Array.Copy(imageBufferView.Buffer.BufferData, imageBufferView.byteOffset, imageData, 0, imageData.Length);
}
if (texture == null)
{
if (gltfObject.UseBackgroundThread) await Update;
// TODO Load texture async
texture = new Texture2D(2, 2);
gltfImage.Texture = texture;
gltfImage.Texture.LoadImage(imageData);
}
else
{
gltfImage.Texture = texture;
}
gltfTexture.Texture = texture;
if (gltfObject.UseBackgroundThread) await BackgroundThread;
}
}
private static async Task ConstructMaterialAsync(this GltfObject gltfObject, GltfMaterial gltfMaterial, int materialId)
{
if (gltfObject.UseBackgroundThread) await Update;
Material material = await CreateMRTKShaderMaterial(gltfObject, gltfMaterial, materialId);
if (material == null)
{
Debug.LogWarning("The Mixed Reality Toolkit/Standard Shader was not found. Falling back to Standard Shader");
material = await CreateStandardShaderMaterial(gltfObject, gltfMaterial, materialId);
}
if (material == null)
{
Debug.LogWarning("The Standard Shader was not found. Failed to create material for glTF object");
}
else
{
gltfMaterial.Material = material;
}
if (gltfObject.UseBackgroundThread) await BackgroundThread;
}
private static async Task CreateMRTKShaderMaterial(GltfObject gltfObject, GltfMaterial gltfMaterial, int materialId)
{
var shader = Shader.Find("Mixed Reality Toolkit/Standard");
if (shader == null) { return null; }
var material = new Material(shader)
{
name = string.IsNullOrEmpty(gltfMaterial.name) ? $"glTF Material {materialId}" : gltfMaterial.name
};
if (gltfMaterial.pbrMetallicRoughness.baseColorTexture?.index >= 0)
{
material.mainTexture = gltfObject.images[gltfMaterial.pbrMetallicRoughness.baseColorTexture.index].Texture;
}
material.color = gltfMaterial.pbrMetallicRoughness.baseColorFactor.GetColorValue();
if (gltfMaterial.alphaMode == "MASK")
{
material.SetInt(SrcBlendId, (int)BlendMode.One);
material.SetInt(DstBlendId, (int)BlendMode.Zero);
material.SetInt(ZWriteId, 1);
material.SetInt(ModeId, 3);
material.SetOverrideTag("RenderType", "Cutout");
material.EnableKeyword("_ALPHATEST_ON");
material.DisableKeyword("_ALPHABLEND_ON");
material.DisableKeyword("_ALPHAPREMULTIPLY_ON");
material.renderQueue = 2450;
}
else if (gltfMaterial.alphaMode == "BLEND")
{
material.SetInt(SrcBlendId, (int)BlendMode.One);
material.SetInt(DstBlendId, (int)BlendMode.OneMinusSrcAlpha);
material.SetInt(ZWriteId, 0);
material.SetInt(ModeId, 3);
material.SetOverrideTag("RenderType", "Transparency");
material.DisableKeyword("_ALPHATEST_ON");
material.DisableKeyword("_ALPHABLEND_ON");
material.EnableKeyword("_ALPHAPREMULTIPLY_ON");
material.renderQueue = 3000;
}
if (gltfMaterial.emissiveTexture?.index >= 0 && material.HasProperty("_EmissionMap"))
{
material.EnableKeyword("_EMISSION");
material.SetColor(EmissiveColorId, gltfMaterial.emissiveFactor.GetColorValue());
}
if (gltfMaterial.pbrMetallicRoughness.metallicRoughnessTexture?.index >= 0)
{
var texture = gltfObject.images[gltfMaterial.pbrMetallicRoughness.metallicRoughnessTexture.index].Texture;
Texture2D occlusionTexture = null;
if (gltfMaterial.occlusionTexture.index >= 0)
{
occlusionTexture = gltfObject.images[gltfMaterial.occlusionTexture.index].Texture;
}
if (texture.isReadable)
{
var pixels = texture.GetPixels();
Color[] occlusionPixels = null;
if (occlusionTexture != null &&
occlusionTexture.isReadable)
{
occlusionPixels = occlusionTexture.GetPixels();
}
if (gltfObject.UseBackgroundThread) await BackgroundThread;
var pixelCache = new Color[pixels.Length];
for (int c = 0; c < pixels.Length; c++)
{
pixelCache[c].r = pixels[c].b; // MRTK standard shader metallic value, glTF metallic value
pixelCache[c].g = occlusionPixels?[c].r ?? 1.0f; // MRTK standard shader occlusion value, glTF occlusion value if available
pixelCache[c].b = 0f; // MRTK standard shader emission value
pixelCache[c].a = (1.0f - pixels[c].g); // MRTK standard shader smoothness value, invert of glTF roughness value
}
if (gltfObject.UseBackgroundThread) await Update;
texture.SetPixels(pixelCache);
texture.Apply();
material.SetTexture(ChannelMapId, texture);
material.EnableKeyword("_CHANNEL_MAP");
}
else
{
material.DisableKeyword("_CHANNEL_MAP");
}
material.SetFloat(SmoothnessId, Mathf.Abs((float)gltfMaterial.pbrMetallicRoughness.roughnessFactor - 1f));
material.SetFloat(MetallicId, (float)gltfMaterial.pbrMetallicRoughness.metallicFactor);
}
if (gltfMaterial.normalTexture?.index >= 0)
{
material.SetTexture(NormalMapId, gltfObject.images[gltfMaterial.normalTexture.index].Texture);
material.SetFloat(NormalMapScaleId, (float)gltfMaterial.normalTexture.scale);
material.EnableKeyword("_NORMAL_MAP");
}
if (gltfMaterial.doubleSided)
{
material.SetFloat(CullModeId, (float)UnityEngine.Rendering.CullMode.Off);
}
material.globalIlluminationFlags = MaterialGlobalIlluminationFlags.RealtimeEmissive;
return material;
}
private static async Task CreateStandardShaderMaterial(GltfObject gltfObject, GltfMaterial gltfMaterial, int materialId)
{
var shader = Shader.Find("Standard");
if (shader == null) { return null; }
var material = new Material(shader)
{
name = string.IsNullOrEmpty(gltfMaterial.name) ? $"glTF Material {materialId}" : gltfMaterial.name
};
if (gltfMaterial.pbrMetallicRoughness.baseColorTexture?.index >= 0)
{
material.mainTexture = gltfObject.images[gltfMaterial.pbrMetallicRoughness.baseColorTexture.index].Texture;
}
if (gltfMaterial.pbrMetallicRoughness?.baseColorFactor != null)
{
material.color = gltfMaterial.pbrMetallicRoughness.baseColorFactor.GetColorValue();
}
if (gltfMaterial.alphaMode == "MASK")
{
material.SetInt(SrcBlendId, (int)BlendMode.One);
material.SetInt(DstBlendId, (int)BlendMode.Zero);
material.SetInt(ZWriteId, 1);
material.SetInt(ModeId, 3);
material.SetOverrideTag("RenderType", "Cutout");
material.EnableKeyword("_ALPHATEST_ON");
material.DisableKeyword("_ALPHABLEND_ON");
material.DisableKeyword("_ALPHAPREMULTIPLY_ON");
material.renderQueue = 2450;
}
else if (gltfMaterial.alphaMode == "BLEND")
{
material.SetInt(SrcBlendId, (int)BlendMode.One);
material.SetInt(DstBlendId, (int)BlendMode.OneMinusSrcAlpha);
material.SetInt(ZWriteId, 0);
material.SetInt(ModeId, 3);
material.SetOverrideTag("RenderType", "Transparency");
material.DisableKeyword("_ALPHATEST_ON");
material.DisableKeyword("_ALPHABLEND_ON");
material.EnableKeyword("_ALPHAPREMULTIPLY_ON");
material.renderQueue = 3000;
}
if (gltfMaterial.emissiveTexture?.index >= 0)
{
material.EnableKeyword("_EmissionMap");
material.EnableKeyword("_EMISSION");
material.SetTexture(EmissionMapId, gltfObject.images[gltfMaterial.emissiveTexture.index].Texture);
material.SetColor(EmissionColorId, gltfMaterial.emissiveFactor.GetColorValue());
}
if (gltfMaterial.pbrMetallicRoughness.metallicRoughnessTexture?.index >= 0)
{
var texture = gltfObject.images[gltfMaterial.pbrMetallicRoughness.metallicRoughnessTexture.index].Texture;
if (texture.isReadable)
{
var pixels = texture.GetPixels();
if (gltfObject.UseBackgroundThread) await BackgroundThread;
var pixelCache = new Color[pixels.Length];
for (int c = 0; c < pixels.Length; c++)
{
// Unity only looks for metal in R channel, and smoothness in A.
pixelCache[c].r = pixels[c].g;
pixelCache[c].g = 0f;
pixelCache[c].b = 0f;
pixelCache[c].a = pixels[c].b;
}
if (gltfObject.UseBackgroundThread) await Update;
texture.SetPixels(pixelCache);
texture.Apply();
material.SetTexture(MetallicGlossMapId, texture);
}
material.SetFloat(GlossinessId, Mathf.Abs((float)gltfMaterial.pbrMetallicRoughness.roughnessFactor - 1f));
material.SetFloat(MetallicId, (float)gltfMaterial.pbrMetallicRoughness.metallicFactor);
material.EnableKeyword("_MetallicGlossMap");
material.EnableKeyword("_METALLICGLOSSMAP");
}
if (gltfMaterial.normalTexture?.index >= 0)
{
material.SetTexture(BumpMapId, gltfObject.images[gltfMaterial.normalTexture.index].Texture);
material.EnableKeyword("_BumpMap");
}
material.globalIlluminationFlags = MaterialGlobalIlluminationFlags.RealtimeEmissive;
return material;
}
private static async Task ConstructSceneAsync(this GltfObject gltfObject, GltfScene gltfScene, GameObject root)
{
for (int i = 0; i < gltfScene.nodes.Length; i++)
{
// Note: glTF objects are currently imported with their original scale from the glTF scene, which may apply an unexpected transform
// to the root node. If this behavior needs to be changed, functionality should be added below to ConstructNodeAsync
await ConstructNodeAsync(gltfObject, gltfObject.nodes[gltfScene.nodes[i]], gltfScene.nodes[i], root.transform);
}
}
private static async Task ConstructNodeAsync(GltfObject gltfObject, GltfNode node, int nodeId, Transform parent)
{
if (gltfObject.UseBackgroundThread) await Update;
var nodeName = string.IsNullOrEmpty(node.name) ? $"glTF Node {nodeId}" : node.name;
var nodeGameObject = new GameObject(nodeName);
gltfObject.NodeGameObjectPairs.Add(nodeId, nodeGameObject);
// If we're creating a really large node, we need it to not be visible in partial stages. So we hide it while we create it
nodeGameObject.SetActive(false);
if (gltfObject.UseBackgroundThread) await BackgroundThread;
node.Matrix = node.GetTrsProperties(out Vector3 position, out Quaternion rotation, out Vector3 scale);
if (node.Matrix == Matrix4x4.identity)
{
if (node.translation != null)
{
position = node.translation.GetVector3Value();
}
if (node.rotation != null)
{
rotation = node.rotation.GetQuaternionValue();
}
if (node.scale != null)
{
scale = node.scale.GetVector3Value(false);
}
}
if (gltfObject.UseBackgroundThread) await Update;
nodeGameObject.transform.localPosition = position;
nodeGameObject.transform.localRotation = rotation;
nodeGameObject.transform.localScale = scale;
if (node.mesh >= 0)
{
await ConstructMeshAsync(gltfObject, nodeGameObject, node.mesh);
}
if (node.children != null)
{
for (int i = 0; i < node.children.Length; i++)
{
await ConstructNodeAsync(gltfObject, gltfObject.nodes[node.children[i]], node.children[i], nodeGameObject.transform);
}
}
nodeGameObject.transform.SetParent(parent, false);
nodeGameObject.SetActive(true);
}
private static async Task ConstructMeshAsync(GltfObject gltfObject, GameObject parent, int meshId)
{
GltfMesh gltfMesh = gltfObject.meshes[meshId];
var renderer = parent.AddComponent();
var filter = parent.AddComponent();
if (gltfMesh.primitives.Length == 1)
{
gltfMesh.Mesh = await ConstructMeshPrimitiveAsync(gltfObject, gltfMesh.primitives[0]);
gltfMesh.Mesh.name = gltfMesh.name;
filter.sharedMesh = gltfMesh.Mesh;
renderer.sharedMaterial = gltfObject.materials[gltfMesh.primitives[0].material].Material;
return;
}
var materials = new List();
var meshCombines = new CombineInstance[gltfMesh.primitives.Length];
for (int i = 0; i < gltfMesh.primitives.Length; i++)
{
meshCombines[i].mesh = await ConstructMeshPrimitiveAsync(gltfObject, gltfMesh.primitives[i]);
var meshMaterial = gltfObject.materials[gltfMesh.primitives[i].material].Material;
if (!materials.Contains(meshMaterial))
{
materials.Add(meshMaterial);
}
}
var newMesh = new Mesh();
newMesh.CombineMeshes(meshCombines);
gltfMesh.Mesh = filter.sharedMesh = newMesh;
gltfMesh.Mesh.name = gltfMesh.name;
renderer.sharedMaterials = materials.ToArray();
}
private static async Task ConstructMeshPrimitiveAsync(GltfObject gltfObject, GltfMeshPrimitive meshPrimitive)
{
if (gltfObject.UseBackgroundThread) await BackgroundThread;
GltfAccessor positionAccessor = null;
GltfAccessor normalsAccessor = null;
GltfAccessor textCoord0Accessor = null;
GltfAccessor textCoord1Accessor = null;
GltfAccessor textCoord2Accessor = null;
GltfAccessor textCoord3Accessor = null;
GltfAccessor colorAccessor = null;
GltfAccessor indicesAccessor = null;
GltfAccessor tangentAccessor = null;
GltfAccessor weight0Accessor = null;
GltfAccessor joint0Accessor = null;
int vertexCount = 0;
positionAccessor = gltfObject.GetAccessor(meshPrimitive.Attributes.POSITION);
if (positionAccessor != null)
{
vertexCount = positionAccessor.count;
}
normalsAccessor = gltfObject.GetAccessor(meshPrimitive.Attributes.NORMAL);
textCoord0Accessor = gltfObject.GetAccessor(meshPrimitive.Attributes.TEXCOORD_0);
textCoord1Accessor = gltfObject.GetAccessor(meshPrimitive.Attributes.TEXCOORD_1);
textCoord2Accessor = gltfObject.GetAccessor(meshPrimitive.Attributes.TEXCOORD_2);
textCoord3Accessor = gltfObject.GetAccessor(meshPrimitive.Attributes.TEXCOORD_3);
colorAccessor = gltfObject.GetAccessor(meshPrimitive.Attributes.COLOR_0);
indicesAccessor = gltfObject.GetAccessor(meshPrimitive.indices);
tangentAccessor = gltfObject.GetAccessor(meshPrimitive.Attributes.TANGENT);
weight0Accessor = gltfObject.GetAccessor(meshPrimitive.Attributes.WEIGHTS_0);
joint0Accessor = gltfObject.GetAccessor(meshPrimitive.Attributes.JOINTS_0);
if (gltfObject.UseBackgroundThread) await Update;
var mesh = new Mesh
{
indexFormat = vertexCount > UInt16.MaxValue ? IndexFormat.UInt32 : IndexFormat.UInt16,
};
if (positionAccessor != null)
{
mesh.vertices = positionAccessor.GetVector3Array();
}
if (normalsAccessor != null)
{
mesh.normals = normalsAccessor.GetVector3Array();
}
if (textCoord0Accessor != null)
{
mesh.uv = textCoord0Accessor.GetVector2Array();
}
if (textCoord1Accessor != null)
{
mesh.uv2 = textCoord1Accessor.GetVector2Array();
}
if (textCoord2Accessor != null)
{
mesh.uv3 = textCoord2Accessor.GetVector2Array();
}
if (textCoord3Accessor != null)
{
mesh.uv4 = textCoord3Accessor.GetVector2Array();
}
if (colorAccessor != null)
{
mesh.colors = colorAccessor.GetColorArray();
}
if (indicesAccessor != null)
{
mesh.triangles = indicesAccessor.GetIntArray();
}
if (tangentAccessor != null)
{
mesh.tangents = tangentAccessor.GetVector4Array();
}
if (weight0Accessor != null && joint0Accessor != null)
{
mesh.boneWeights = CreateBoneWeightArray(joint0Accessor.GetVector4Array(false), weight0Accessor.GetVector4Array(false), vertexCount);
}
mesh.RecalculateBounds();
meshPrimitive.SubMesh = mesh;
return mesh;
}
private static BoneWeight[] CreateBoneWeightArray(Vector4[] joints, Vector4[] weights, int vertexCount)
{
NormalizeBoneWeightArray(weights);
var boneWeights = new BoneWeight[vertexCount];
for (int i = 0; i < vertexCount; i++)
{
boneWeights[i].boneIndex0 = (int)joints[i].x;
boneWeights[i].boneIndex1 = (int)joints[i].y;
boneWeights[i].boneIndex2 = (int)joints[i].z;
boneWeights[i].boneIndex3 = (int)joints[i].w;
boneWeights[i].weight0 = weights[i].x;
boneWeights[i].weight1 = weights[i].y;
boneWeights[i].weight2 = weights[i].z;
boneWeights[i].weight3 = weights[i].w;
}
return boneWeights;
}
private static void NormalizeBoneWeightArray(Vector4[] weights)
{
for (int i = 0; i < weights.Length; i++)
{
var weightSum = weights[i].x + weights[i].y + weights[i].z + weights[i].w;
if (!Mathf.Approximately(weightSum, 0))
{
weights[i] /= weightSum;
}
}
}
}
}