diff --git a/Assets/vfx-graph-gs/Main.unity b/Assets/vfx-graph-gs/Main.unity index 05617f9..7b86ca9 100644 --- a/Assets/vfx-graph-gs/Main.unity +++ b/Assets/vfx-graph-gs/Main.unity @@ -157,7 +157,7 @@ PrefabInstance: objectReference: {fileID: 0} - target: {fileID: -973084862543737302, guid: 25c54d8c074f08447ab1c822a6a052fa, type: 3} propertyPath: m_PropertySheet.m_Float.m_Array.Array.data[0].m_Value - value: 64 + value: 32 objectReference: {fileID: 0} - target: {fileID: -973084862543737302, guid: 25c54d8c074f08447ab1c822a6a052fa, type: 3} propertyPath: m_PropertySheet.m_Uint.m_Array.Array.data[0].m_Overridden diff --git a/Assets/vfx-graph-gs/Scripts/SplatImporter.cs b/Assets/vfx-graph-gs/Scripts/SplatImporter.cs index 813e067..c438094 100644 --- a/Assets/vfx-graph-gs/Scripts/SplatImporter.cs +++ b/Assets/vfx-graph-gs/Scripts/SplatImporter.cs @@ -18,19 +18,12 @@ using Unity.VisualScripting; using System.Collections; using Unity.Collections; using UnityEditor.Experimental.GraphView; - -/******************************************************************************* - TODO: - 1. Move header parsing to ParsePLYFile - 2. Improve exception reporting and handling - 3. Use Burst compiler for parsing binary data - 4. If we can reasonably assume that items are contigious (e.g. position) - use MemoryMarshal - 5. Use a struct for header_info - 6. Write handler for data other than 32 bit float - 7. Pass around Span instead of byte[] for memory safety - 8. Implement opacity clamping in the VFX Graph -*******************************************************************************/ +using UnityEngine.Playables; +using Unity.Android.Gradle.Manifest; +using UnityEngine.Timeline; +using Unity.Jobs; +using System.Xml.Schema; +using Unity.Android.Gradle; namespace SplatVfx.Editor { @@ -39,16 +32,14 @@ namespace SplatVfx.Editor public sealed class SplatImporter : ScriptedImporter { const float SH_C0 = 0.28209479177387814f; + const string DefaultVfxPath = "Assets/vfx-graph-gs/SplatVFX.vfx"; - #region ScriptedImporter implementation - - [BurstCompile] public override void OnImportAsset(AssetImportContext context) { - var ply = ParsePLYFile(context.assetPath); - var header_info = ParseHeader(ply.header); + var (headerString, endHeaderPosition) = GetHeader(context.assetPath); + var headerData = ParseMetadata(headerString); - var data = ImportAsSplatData(header_info, ply.binaryData); + var data = ImportData(headerData, context.assetPath, endHeaderPosition); data.name = Path.GetFileNameWithoutExtension(context.assetPath); context.AddObjectToAsset("data", data); @@ -57,14 +48,7 @@ namespace SplatVfx.Editor context.SetMainObject(prefab); } - #endregion - - #region Prefab construction - - const string DefaultVfxPath = "Assets/vfx-graph-gs/SplatVFX.vfx"; - - [BurstCompile] - GameObject CreatePrefab(SplatData data) + private GameObject CreatePrefab(SplatData data) { var go = new GameObject(); var vfx = go.AddComponent(); @@ -78,12 +62,77 @@ namespace SplatVfx.Editor return go; } - #endregion + const int MAX_CHUNK_SIZE = 1048576 * 256; // 256MiB - #region Reader implementation + private enum TypeEnum + { + None, Float32, Float64, Float16, Int8, UInt8, Char, Short, UInt16 + } - [BurstCompile] - (string header, byte[] binaryData) ParsePLYFile(string filePath) + private enum DataEnum + { + x, y, z, f_dc_0, f_dc_1, f_dc_2, f_rest_0, f_rest_1, f_rest_2, + f_rest_3, f_rest_4, f_rest_5, f_rest_6, f_rest_7, f_rest_8, + f_rest_9, f_rest_10, f_rest_11, f_rest_12, f_rest_13, f_rest_14, + f_rest_15, f_rest_16, f_rest_17, f_rest_18, f_rest_19, f_rest_20, + f_rest_21, f_rest_22, f_rest_23, f_rest_24, f_rest_25, f_rest_26, + f_rest_27, f_rest_28, f_rest_29, f_rest_30, f_rest_31, f_rest_32, + f_rest_33, f_rest_34, f_rest_35, f_rest_36, f_rest_37, f_rest_38, + f_rest_39, f_rest_40, f_rest_41, f_rest_42, f_rest_43, f_rest_44, + opacity, scale_0, scale_1, scale_2, rot_0, rot_1, rot_2, rot_3 + } + + /*********************************************************************** + - TODO: Add support for more data types + ***********************************************************************/ + private static readonly Dictionary TypeToByteLengths = + new() + { + { TypeEnum.Float32, 4 }, + { TypeEnum.Float64, 8 }, + { TypeEnum.Float16, 2 }, // keeping this here to support half precision floats in the future + { TypeEnum.UInt8, 1 }, + { TypeEnum.Int8, 1 } + }; + + private static readonly Dictionary TypeMappings = + new() + { + { "float", TypeEnum.Float32 }, + { "double", TypeEnum.Float64 }, + { "uint8", TypeEnum.UInt8 }, + { "int8", TypeEnum.Int8 } + }; + + /*********************************************************************** + - TODO: Add endianness + - TODO: Add float size + ***********************************************************************/ + private struct HeaderData + { + public uint NumElements; + public uint BinaryDataStartOffset; + public NativeArray ElementByteOffsets; + public NativeArray ElementTypes; + public int BytesPerElement; + + public HeaderData(int count) + { + NumElements = 0; + BinaryDataStartOffset = 0; + ElementByteOffsets = new NativeArray(count, Allocator.Persistent); + ElementTypes = new NativeArray(count, Allocator.Persistent); + BytesPerElement = 0; + } + + public void Dispose() + { + ElementByteOffsets.Dispose(); + ElementTypes.Dispose(); + } + } + + private static (string headerString, int endHeaderPosition) GetHeader(string filePath) { using var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read); using var reader = new StreamReader(fileStream); @@ -100,58 +149,251 @@ namespace SplatVfx.Editor } } - long endHeaderPosition = headerBuilder.Length - headerBuilder.ToString().Split('\n').Length + 1; - fileStream.Seek(endHeaderPosition, SeekOrigin.Begin); + // Old logic that works because AppendLine adds another \n + // TODO: Fix this logic + int endHeaderPosition = headerBuilder.Length - headerBuilder.ToString().Split('\n').Length + 1; + Debug.Log($"Total filesize: {fileStream.Length}"); + Debug.Log($"Byte stream start position: {endHeaderPosition}"); - var binaryReader = new BinaryReader(fileStream); - var binaryData = binaryReader.ReadBytes((int)(fileStream.Length - endHeaderPosition)); - - return (headerBuilder.ToString(), binaryData); + return (headerBuilder.ToString(), endHeaderPosition); } - [BurstCompile] - (string format, int num_element, List<(string, string)> properties) - ParseHeader(string header) + private static HeaderData ParseMetadata(string headerString) { - using var reader = new StringReader(header); + HeaderData result = new(59); string line; - string format = ""; - int num_element = 0; - var properties = new List<(string, string)>(); + using var reader = new StringReader(headerString); while ((line = reader.ReadLine()) != null) { - if (line.StartsWith("end_header")) - { + if (line.Trim() == "end_header") break; - } var strings = line.Split(" "); switch (strings.FirstOrDefault()) { - case "format": - format = strings[1]; - - break; - case "element": - num_element = int.Parse(strings[2]); - + result.NumElements = uint.Parse(strings[2]); break; case "property": - properties.Add((strings[2].Trim(), strings[1].Trim())); + TypeEnum dataType = TypeMappings[strings[1]]; + // If in DataEnum, store byte offset and type + if (DataEnum.TryParse(strings[2], out DataEnum dataName)) + { + int ix = (int)dataName; + result.ElementByteOffsets[ix] = result.BytesPerElement; + result.ElementTypes[ix] = (int)dataType; + } + + result.BytesPerElement += TypeToByteLengths[dataType]; break; } } + Debug.Log($"Bytes per Element: {result.BytesPerElement}"); - return (format, num_element, properties); + return result; } [BurstCompile] + private struct SplatDataNative + { + public NativeArray position; + public NativeArray color; + public NativeArray sh; + public NativeArray opacity; + public NativeArray scale; + public NativeArray rotation; + + public SplatDataNative(int count) + { + position = new NativeArray(count, Allocator.Persistent); + color = new NativeArray(count, Allocator.Persistent); + sh = new NativeArray(count * 45, Allocator.Persistent); + opacity = new NativeArray(count, Allocator.Persistent); + scale = new NativeArray(count, Allocator.Persistent); + rotation = new NativeArray(count, Allocator.Persistent); + } + + public void Dispose() + { + position.Dispose(); + color.Dispose(); + sh.Dispose(); + opacity.Dispose(); + scale.Dispose(); + rotation.Dispose(); + } + } + + private static SplatData ImportData(HeaderData headerData, string filePath, int endHeaderPosition) + { + var splatData = ScriptableObject.CreateInstance(); + + splatData.PositionArray = new Vector3[headerData.NumElements]; + splatData.ScaleArray = new Vector3[headerData.NumElements]; + splatData.RotationArray = new Quaternion[headerData.NumElements]; + splatData.ColorArray = new Color[headerData.NumElements]; + splatData.OpacityArray = new float[headerData.NumElements]; + splatData.SHArray = new float[headerData.NumElements * 45]; + + int elementsPerChunk = Mathf.Min( + MAX_CHUNK_SIZE / headerData.BytesPerElement, + (int)headerData.NumElements + ); + int chunkSize = elementsPerChunk * headerData.BytesPerElement; + + Debug.Log($"Elements per chunk: {elementsPerChunk}"); + Debug.Log($"Chunk size: {chunkSize}"); + + using var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read); + fileStream.Seek(endHeaderPosition, SeekOrigin.Begin); + + int totalBytes = (int)fileStream.Length - endHeaderPosition; + Debug.Log($"Total bytes: {totalBytes}"); + + var splatDataNative = new SplatDataNative(elementsPerChunk); + var chunkBuffer = new NativeArray(chunkSize, Allocator.Persistent); + var chunkBufferSpan = MemoryMarshal.AsBytes(chunkBuffer); + int numBytesRead; + int numElemsRead; + int totalElemsRead = 0; + + while ((numBytesRead = fileStream.Read(chunkBufferSpan)) > 0) + { + numElemsRead = numBytesRead / headerData.BytesPerElement; + var splatElementJob = new SplatElementJob + { + chunkBytes = chunkBuffer, + splatDataNative = splatDataNative, + headerData = headerData, + totalElemsRead = totalElemsRead, + numElemsRead = numElemsRead + }; + + splatElementJob.Schedule().Complete(); + + for (int i = 0; i < numElemsRead; i++) + { + splatData.PositionArray[totalElemsRead + i] = splatDataNative.position[i]; + splatData.ColorArray[totalElemsRead + i].r = splatDataNative.color[i].x; + splatData.ColorArray[totalElemsRead + i].g = splatDataNative.color[i].y; + splatData.ColorArray[totalElemsRead + i].b = splatDataNative.color[i].z; + splatData.OpacityArray[totalElemsRead + i] = splatDataNative.opacity[i]; + splatData.ScaleArray[totalElemsRead + i] = splatDataNative.scale[i]; + splatData.RotationArray[totalElemsRead + i] = new quaternion(splatDataNative.rotation[i]); + } + + // NativeArray.Copy(splatDataNative.position.Reinterpret(), 0, splatData.PositionArray, totalElemsRead, numElemsRead); + + totalElemsRead += numElemsRead; + Debug.Log($"Current Elements Read: {totalElemsRead}"); + } + + Debug.Log($"First position Vector3: {splatData.PositionArray[0]}"); + + headerData.Dispose(); + splatDataNative.Dispose(); + chunkBuffer.Dispose(); + + splatData.ReleaseGpuResources(); + + return splatData; + } + + [BurstCompile] + private struct SplatElementJob : IJob + { + public NativeArray chunkBytes; + public SplatDataNative splatDataNative; + [ReadOnly] public HeaderData headerData; + [ReadOnly] public int totalElemsRead; + [ReadOnly] public int numElemsRead; + + public void Execute() + { + for (int i = 0; i < numElemsRead; i++) + { + int elemIx = i; + int start = elemIx * headerData.BytesPerElement; + + splatDataNative.position[elemIx] = new() + { + x = chunkBytes.ReinterpretLoad(start + headerData.ElementByteOffsets[(int)DataEnum.x]), + y = chunkBytes.ReinterpretLoad(start + headerData.ElementByteOffsets[(int)DataEnum.y]), + z = chunkBytes.ReinterpretLoad(start + headerData.ElementByteOffsets[(int)DataEnum.z]) + }; + + float3 color = new() + { + x = chunkBytes.ReinterpretLoad(start + headerData.ElementByteOffsets[(int)DataEnum.f_dc_0]), + y = chunkBytes.ReinterpretLoad(start + headerData.ElementByteOffsets[(int)DataEnum.f_dc_1]), + z = chunkBytes.ReinterpretLoad(start + headerData.ElementByteOffsets[(int)DataEnum.f_dc_2]) + }; + color = 0.5f + color * SH_C0; + splatDataNative.color[elemIx] = color; + + for (int j = 0; j < 45; j++) + { + splatDataNative.sh[elemIx + j] = chunkBytes[start + headerData.ElementByteOffsets[(int)DataEnum.f_rest_0 + j]]; + } + + float opacity = chunkBytes[start + headerData.ElementByteOffsets[(int)DataEnum.opacity]]; + opacity = 1 / (1 + math.exp(-opacity)); + splatDataNative.opacity[elemIx] = opacity; + + float3 scale = new() + { + x = chunkBytes.ReinterpretLoad(start + headerData.ElementByteOffsets[(int)DataEnum.scale_0]), + y = chunkBytes.ReinterpretLoad(start + headerData.ElementByteOffsets[(int)DataEnum.scale_1]), + z = chunkBytes.ReinterpretLoad(start + headerData.ElementByteOffsets[(int)DataEnum.scale_2]) + }; + scale = math.exp(scale); + splatDataNative.scale[elemIx] = scale; + + splatDataNative.rotation[elemIx] = new() + { + w = chunkBytes.ReinterpretLoad(start + headerData.ElementByteOffsets[(int)DataEnum.rot_0]), + x = chunkBytes.ReinterpretLoad(start + headerData.ElementByteOffsets[(int)DataEnum.rot_1]), + y = chunkBytes.ReinterpretLoad(start + headerData.ElementByteOffsets[(int)DataEnum.rot_2]), + z = chunkBytes.ReinterpretLoad(start + headerData.ElementByteOffsets[(int)DataEnum.rot_3]) + }; + } + } + } + + // private struct ReadChunkJob + + /*********************************************************************** + ImportData function + - Takes a HeaderData struct and file path string as input + - Following the JOB_SZ and the length of a single element's data + -- Computes how many bytes a single job can process + -- Reads them in chunks + -- As each chunk is read + --- Creates NativeArrays representing all the data points + --- Creates a ReadChunk Jobs to parse the data + - Once each Job is complete, begins copying the NativeArrays into a + newly created SplatData scriptable object + ***********************************************************************/ + + /*********************************************************************** + ReadChunk Job + - This MUST be Burst compiled + - Takes in NativeArrays, the chunk's byte data, and the HeaderData + struct + - It may be more efficient to place the NativeArrays inside a Burst + compiled struct + - Iterates through the byte data using the total length of a data point + as stored in HeaderData + - For each data point, reads the data points and stores them into the + appropriate NativeArray, referencing the byte offset table in + HeaderData to find their locations in the byte data + ***********************************************************************/ + SplatData ImportAsSplatData((string format, int num_element, List<(string, string)> properties) header_info, byte[] binaryData) { var splatData = ScriptableObject.CreateInstance(); @@ -322,7 +564,6 @@ namespace SplatVfx.Editor // color = (Vector4)math.float4(src.r, src.g, src.b, src.a) / 255; // } - #endregion } -} // namespace SplatVfx.Editor +} diff --git a/Assets/vfx-graph-gs/SplatVFX.vfx b/Assets/vfx-graph-gs/SplatVFX.vfx index 7990cef..8566f6f 100644 --- a/Assets/vfx-graph-gs/SplatVFX.vfx +++ b/Assets/vfx-graph-gs/SplatVFX.vfx @@ -209,7 +209,7 @@ MonoBehaviour: x: -692 y: -492 width: 1846 - height: 2501 + height: 2502 --- !u!114 &114350483966674976 MonoBehaviour: m_ObjectHideFlags: 1 @@ -13741,7 +13741,7 @@ MonoBehaviour: m_Parent: {fileID: 114350483966674976} m_Children: - {fileID: 8926484042661619332} - m_UIPosition: {x: 705, y: 700} + m_UIPosition: {x: 705, y: 741} m_UICollapsed: 0 m_UISuperCollapsed: 0 m_InputSlots: [] @@ -16773,7 +16773,7 @@ MonoBehaviour: m_Type: m_SerializableType: System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - m_SerializableObject: True + m_SerializableObject: False m_Space: -1 m_Property: name: _vfx_enabled