feat: implement material palette management and core mesh asset handling infrastructure

src/Editor/Ghost.Editor.Core/Assets/MeshAssetHandler.cs

@@ -1,7 +1,6 @@
 using Ghost.Core;
 using Ghost.Engine;
 using Ghost.Graphics.RHI;
-using Misaki.HighPerformance.LowLevel.Buffer;
 using Misaki.HighPerformance.LowLevel.Collections;
 using Misaki.HighPerformance.Mathematics;
 using System.Runtime.InteropServices;
@@ -10,10 +9,10 @@ namespace Ghost.Editor.Core.Assets;
 
 public class MeshNode : IDisposable
 {
-    public required string Name
+    public string Name
     {
         get; set;
-    }
+    } = string.Empty;
 
     public float4x4 LocalTransform
     {
@@ -35,6 +34,11 @@ public class MeshNode : IDisposable
         Dispose(false);
     }
 
+    public MeshNode Clone()
+    {
+        return (MeshNode)MemberwiseClone();
+    }
+
     protected virtual void Dispose(bool disposing)
     {
     }
@@ -241,7 +245,7 @@ internal class FbxAssetSettings : MeshAssetSettings
 {
 }
 
-internal class FBXAssetHandler : IImportableAssetHandler
+internal class FBXAssetHandler : IImportableAssetHandler, IPackableAssetHandler
 {
     public AssetType RuntimeAssetType => AssetType.Mesh;
 
@@ -273,4 +277,9 @@ internal class FBXAssetHandler : IImportableAssetHandler
     {
         throw new NotImplementedException();
     }
+
+    public ValueTask<Result> PackAsync(string assetPath, MemoryStream targetStream, CancellationToken token = default)
+    {
+        throw new NotImplementedException();
+    }
 }

src/Editor/Ghost.Editor.Core/Assets/MeshProcessor.Import.cs

@@ -15,7 +15,7 @@ using System.Text;
 
 namespace Ghost.Editor.Core.Assets;
 
-internal readonly unsafe struct MeshParsingWorkItem : IJob
+internal readonly unsafe struct MeshParsingJob : IJob
 {
     private struct GeometryPart : IDisposable
     {
@@ -32,19 +32,18 @@ internal readonly unsafe struct MeshParsingWorkItem : IJob
         }
     }
 
+    private readonly MeshNode _rootNode;
+
     private readonly string _filePath;
     private readonly AllocationHandle _allocationHandle;
     private readonly MeshAssetSettings _settings;
-    private readonly TaskCompletionSource<Result<MeshNode>> _taskCompletionSource;
 
-    public readonly Task<Result<MeshNode>> Task => _taskCompletionSource.Task;
+    public MeshParsingJob(MeshNode rootNode, string filePath, AllocationHandle allocationHandle, MeshAssetSettings settings)
-
-    public MeshParsingWorkItem(string filePath, AllocationHandle allocationHandle, MeshAssetSettings settings)
     {
+        _rootNode = rootNode;
         _filePath = filePath;
         _allocationHandle = allocationHandle;
         _settings = settings;
-        _taskCompletionSource = new TaskCompletionSource<Result<MeshNode>>();
     }
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -81,15 +80,13 @@ internal readonly unsafe struct MeshParsingWorkItem : IJob
         );
     }
 
-    private MeshNode ParseHierarchy(ufbx_node* node)
+    private void ParseHierarchy(ufbx_node* node, MeshNode self)
     {
         var children = new List<MeshNode>();
-        var meshNode = new MeshNode
+
-        {
+        self.Name = node->name.ToString();
-            Name = node->name.ToString(),
+        self.LocalTransform = ToFloat4x4(node->local_transform.translation, node->local_transform.rotation, node->local_transform.scale);
-            LocalTransform = ToFloat4x4(node->local_transform.translation, node->local_transform.rotation, node->local_transform.scale),
+        self.Children = children;
-            Children = children
-        };
 
         if (node->mesh != null)
         {
@@ -104,10 +101,12 @@ internal readonly unsafe struct MeshParsingWorkItem : IJob
 
         for (var i = 0u; i < node->children.count; i++)
         {
-            children.Add(ParseHierarchy(node->children.data[i]));
+            var childNode = new MeshNode();
-        }
+            ParseHierarchy(node->children.data[i], childNode);
+            childNode.Parent = self;
 
-        return meshNode;
+            children.Add(childNode);
+        }
     }
 
     private GeometryMeshNode? ParseGeometry(ufbx_mesh* pMesh)
@@ -353,14 +352,11 @@ internal readonly unsafe struct MeshParsingWorkItem : IJob
         using var scene = new DisposablePtr<ufbx_scene>(ufbx_scene.LoadFile((sbyte*)str.GetUnsafePtr(), &load_Opts, &error));
         if (scene.Get() == null)
         {
-            _taskCompletionSource.SetResult(Result.Failure(error.description.ToString()));
+            Logger.Error(error.description.ToString());
             return;
         }
 
-        var rootNode = ParseHierarchy(scene.Get()->root_node);
+        ParseHierarchy(scene.Get()->root_node, _rootNode);
-        rootNode.Name = Path.GetFileNameWithoutExtension(_filePath);
-
-        _taskCompletionSource.SetResult(Result.Success(rootNode));
     }
 }
 

src/Editor/Ghost.Editor.Core/Assets/TextureProcessor.Compress.cs

@@ -1,6 +1,7 @@
 using Ghost.Core;
 using Ghost.Engine;
 using Ghost.Nvtt;
+using Misaki.HighPerformance.Jobs;
 using Misaki.HighPerformance.LowLevel;
 using Misaki.HighPerformance.LowLevel.Buffer;
 using Misaki.HighPerformance.LowLevel.Collections;
@@ -13,25 +14,23 @@ namespace Ghost.Editor.Core.Assets;
 
 internal static partial class TextureProcessor
 {
-    private class NvttPipelineTask : IThreadPoolWorkItem
+    private struct NvttPipelineJob : IJob
     {
-        private readonly string _outputPath;
+        private readonly Wrapper<Result<int>> _result;
 
+        private readonly string _outputPath;
         private readonly TextureAssetHandler.TextureInfo _textureInfo;
         private readonly TextureAssetSettings _settings;
         private UnsafeArray<MipLevel> _mipLevels;
 
-        private readonly TaskCompletionSource<Result<int>> _completionSource;
+        public NvttPipelineJob(Wrapper<Result<int>> result, string outputPath, TextureAssetHandler.TextureInfo textureInfo, TextureAssetSettings settings, UnsafeArray<MipLevel> mipLevels)
-
-        public Task<Result<int>> Task => _completionSource.Task;
-
-        public NvttPipelineTask(string outputPath, TextureAssetHandler.TextureInfo textureInfo, TextureAssetSettings settings, UnsafeArray<MipLevel> mipLevels)
         {
+            _result = result;
+
             _outputPath = outputPath;
             _textureInfo = textureInfo;
             _settings = settings;
             _mipLevels = mipLevels;
-            _completionSource = new TaskCompletionSource<Result<int>>();
         }
 
         private unsafe Result<int> RunMipGenCompressionPipeline()
@@ -226,11 +225,12 @@ internal static partial class TextureProcessor
             return Result.Success(maxCubeMips);
         }
 
-        public void Execute()
+        public void Execute(ref readonly JobExecutionContext context)
         {
-            Result<int> finalResult;
             try
             {
+                Result<int> finalResult;
+
                 if (_settings.Basic.TextureShape == TextureShape.TextureCube)
                 {
                     finalResult = RunCubeMapCompressionPipeline();
@@ -239,13 +239,13 @@ internal static partial class TextureProcessor
                 {
                     finalResult = RunMipGenCompressionPipeline();
                 }
+
+                _result.Value = finalResult;
             }
             catch (Exception ex)
             {
-                finalResult = Result.Failure($"Compression threw an exception: {ex.Message}");
+                Logger.Error($"Exception during NVTT compression: {ex}");
             }
-
-            _completionSource.SetResult(finalResult);
         }
     }
 
@@ -360,15 +360,17 @@ internal static partial class TextureProcessor
                 baseCubeData.Dispose();
             }
 
-            var workItem = new NvttPipelineTask(cachePath, textureInfo, settings, mipLevels);
+            var result = new Wrapper<Result<int>>();
-            ThreadPool.UnsafeQueueUserWorkItem(workItem, true);
+            var nvttJob = new NvttPipelineJob(result, cachePath, textureInfo, settings, mipLevels);
-            var result = await workItem.Task.WaitAsync(cancellationToken).ConfigureAwait(false);
+            var nvttJobHandle = scheduler.Schedule(in nvttJob);
-            if (result.IsFailure)
+            await scheduler.WaitAsync(nvttJobHandle, cancellationToken);
+
+            if (result.Value.IsFailure)
             {
-                return Result.Failure(result.Message);
+                return Result.Failure(result.Value.Message);
             }
 
-            return (cachePath, result.Value);
+            return (cachePath, result.Value.Value);
         }
         finally
         {

src/Runtime/Ghost.Core/Common.cs

@@ -1,2 +1,9 @@
 namespace Ghost.Core;
 
+public class Wrapper<T>
+{
+    public T? Value
+    {
+        get; set;
+    }
+}

src/Runtime/Ghost.Core/Ghost.Core.csproj

@@ -21,8 +21,8 @@
 
   <ItemGroup>
     <PackageReference Include="Misaki.HighPerformance" Version="1.0.8" />
-    <PackageReference Include="Misaki.HighPerformance.Jobs" Version="3.1.0" />
+    <PackageReference Include="Misaki.HighPerformance.Jobs" Version="3.1.2" />
-    <PackageReference Include="Misaki.HighPerformance.LowLevel" Version="1.6.15">
+    <PackageReference Include="Misaki.HighPerformance.LowLevel" Version="1.6.16">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

src/Runtime/Ghost.Engine/Components/MeshInstance.cs

@@ -1,6 +1,7 @@
 using Ghost.Core;
 using Ghost.Entities;
 using Ghost.Graphics.Core;
+using Ghost.Graphics.Services;
 
 namespace Ghost.Engine.Components;
 

src/Runtime/Ghost.Entities/Component.cs

@@ -75,7 +75,7 @@ internal static class ComponentRegistry
                 size = sizeof(T),
                 alignment = (int)MemoryUtility.AlignOf<T>(),
                 isEnableable = typeof(IEnableableComponent).IsAssignableFrom(type),
-                isSharedWarper = typeof(ISharedWarper).IsAssignableFrom(type),
+                isSharedWarper = typeof(ISharedWrapper).IsAssignableFrom(type),
                 isCleanup = typeof(ICleanupComponent).IsAssignableFrom(type),
             };
 

src/Runtime/Ghost.Entities/SharedComponent.cs

@@ -7,7 +7,7 @@ using System.Diagnostics;
 namespace Ghost.Entities;
 
 public interface ISharedComponent;
-public interface ISharedWarper
+public interface ISharedWrapper
 {
     int Index
     {
@@ -15,7 +15,7 @@ public interface ISharedWarper
     }
 }
 
-public struct Shared<T> : IComponent, ISharedWarper
+public struct Shared<T> : IComponent, ISharedWrapper
     where T : unmanaged, ISharedComponent
 {
     public int Index

src/Runtime/Ghost.Graphics/Core/RenderList.cs

@@ -1,4 +1,5 @@
 using Ghost.Core;
+using Ghost.Graphics.Services;
 using Misaki.HighPerformance.LowLevel.Buffer;
 using Misaki.HighPerformance.LowLevel.Collections;
 using Misaki.HighPerformance.Mathematics;

src/Runtime/Ghost.Graphics/Services/MaterialPaletteStore.cs

@@ -1,9 +1,10 @@
 using Ghost.Core;
+using Ghost.Graphics.Core;
 using Misaki.HighPerformance.LowLevel.Buffer;
 using Misaki.HighPerformance.LowLevel.Collections;
 using System.Runtime.CompilerServices;
 
-namespace Ghost.Graphics.Core;
+namespace Ghost.Graphics.Services;
 
 public readonly struct MaterialPalette
 {

src/Runtime/Ghost.Graphics/Services/ResourceManager.cs

@@ -33,8 +33,8 @@ public sealed partial class ResourceManager : IDisposable
     private readonly MaterialPaletteStore _materialPalettes;
 
     // TODO: Any better way? System.Threading.Lock is very fast though, it use spin lock before entering kernel.
-    // rw lock slim is an option but it has more overhead on read, and for more than 90% of the time we are reading, it may not be a good option.
+    // rw lock slim is an option but it has more overhead on read. Because more than 90% of the time we are reading, it may not be a good option.
-    // Plus UnsafeSlotMap use jagged array internally, which means we can have concurrent read and write on different slots without any issue, so we only need to lock when writing to those slots.
+    // Plus UnsafeSlotMap use jagged array internally, which means we can have concurrent read and write, but not add and remove, on different slots without any issue, so we only need to lock when writing to those slots.
     private readonly Lock _meshWriteLock;
     private readonly Lock _materialWriteLock;
     private readonly Lock _shaderWriteLock;