feat(render): refactor pipeline & shader system for DX12 WG

Major refactor of render pipeline and shader system:
- Replaced legacy shader properties with source generator and attribute-based HLSL struct generation.
- Introduced ShaderPropertiesRegistry for runtime property layout/code registration.
- Added modular IRenderPipeline, IRenderPipelineSettings, and IRenderPayload interfaces.
- Implemented GhostRenderPipeline and ECS-driven GPUScene management.
- Added experimental DirectX 12 Work Graph support.
- Refactored shader compilation, variant hashing, and caching.
- Updated APIs for consistency and improved codegen for registration.

These changes modernize the rendering infrastructure for advanced features like work graphs and dynamic pipelines.

BREAKING CHANGE: Shader DSL, pipeline, and property APIs have changed. Existing shaders and pipeline integrations must be updated.

src/Runtime/Ghost.Engine/EngineCore.cs

@@ -39,4 +39,4 @@ public sealed partial class EngineCore : IDisposable
         _renderSystem.Dispose();
         _jobScheduler.Dispose();
     }
-}
+}

src/Runtime/Ghost.Engine/Ghost.Engine.csproj

@@ -26,7 +26,7 @@
   <ItemGroup>
     <ProjectReference Include="..\Ghost.Core\Ghost.Core.csproj" />
     <ProjectReference Include="..\Ghost.Entities\Ghost.Entities.csproj" />
-    <!--<ProjectReference Include="..\Ghost.Generator\Ghost.Generator.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />-->
+    <ProjectReference Include="..\Ghost.Generator\Ghost.Generator.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
     <ProjectReference Include="..\Ghost.Graphics\Ghost.Graphics.csproj" />
   </ItemGroup>
 

src/Runtime/Ghost.Engine/RenderPipeline/GPUScene.cs

@@ -0,0 +1,111 @@
+using Ghost.Core;
+using Ghost.Graphics.RHI;
+using System.Diagnostics;
+
+namespace Ghost.Engine.RenderPipeline;
+
+internal unsafe class GPUScene : IDisposable
+{
+    private readonly IResourceAllocator _resourceAllocator;
+    private readonly IResourceDatabase _resourceDatabase;
+
+    private Handle<GPUBuffer> _sceneBuffer;
+    private uint _instanceCount;
+    private uint _capacity;
+
+    private uint _requiredResize;
+    private bool _disposed;
+
+    internal GPUScene(IResourceAllocator resourceAllocator, IResourceDatabase resourceDatabase, uint initialCount)
+    {
+        _resourceAllocator = resourceAllocator;
+        _resourceDatabase = resourceDatabase;
+
+        var bufferDesc = new BufferDesc
+        {
+            Size = initialCount * (ulong)sizeof(InstanceData),
+            Stride = (uint)sizeof(InstanceData),
+            Usage = BufferUsage.Structured | BufferUsage.UnorderedAccess | BufferUsage.ShaderResource,
+            HeapType = HeapType.Default,
+        };
+
+        _sceneBuffer = _resourceAllocator.CreateBuffer(in bufferDesc, "SceneBuffer");
+        Debug.Assert(_sceneBuffer.IsValid, "Failed to create GPUScene buffer.");
+
+        _capacity = initialCount;
+    }
+
+    ~GPUScene()
+    {
+        Dispose();
+    }
+
+    // NOTE: This is not thread safe.
+    public void ResizeIfNeeded(ICommandBuffer cmd)
+    {
+        if (_requiredResize == 0)
+        {
+            return;
+        }
+
+        var newCapacity = _capacity * 2;
+        newCapacity = Math.Max(newCapacity, _capacity + _requiredResize);
+
+        var newBufferDesc = new BufferDesc
+        {
+            Size = (ulong)newCapacity * (ulong)sizeof(InstanceData),
+            Stride = (uint)sizeof(InstanceData),
+            Usage = BufferUsage.Structured | BufferUsage.UnorderedAccess | BufferUsage.ShaderResource,
+            HeapType = HeapType.Default,
+        };
+
+        var newBuffer = _resourceAllocator.CreateBuffer(in newBufferDesc, "SceneBuffer_Resized");
+        Debug.Assert(newBuffer.IsValid);
+
+        // Copy existing data to the new buffer
+        cmd.CopyBuffer(newBuffer, _sceneBuffer, 0, 0, (ulong)_instanceCount * (ulong)sizeof(InstanceData));
+
+        // Replace old buffer with the new one
+        _resourceDatabase.ReleaseResource(_sceneBuffer.AsResource());
+        _sceneBuffer = newBuffer;
+        _capacity = newCapacity;
+
+        _requiredResize = 0;
+    }
+
+    public uint AddInstance()
+    {
+        if (Volatile.Read(ref _instanceCount) >= _capacity)
+        {
+            Interlocked.Increment(ref _requiredResize);
+        }
+
+        var index = Interlocked.Increment(ref _instanceCount);
+        return index;
+    }
+
+    public uint RemoveInstance(uint index)
+    {
+        if (index < 0 || index >= _capacity)
+        {
+            return ~0u;
+        }
+
+        // Return the last index. We will swap the last instance data with the removed index on gpu to keep the buffer compact.
+        var last = Interlocked.Decrement(ref _instanceCount);
+        return last;
+    }
+
+    public void Dispose()
+    {
+        if (_disposed)
+        {
+            return;
+        }
+
+        _resourceDatabase.ReleaseResource(_sceneBuffer.AsResource());
+
+        _disposed = true;
+        GC.SuppressFinalize(this);
+    }
+}

src/Runtime/Ghost.Engine/RenderPipeline/GhostRenderPipeline.cs

@@ -0,0 +1,40 @@
+using Ghost.Graphics;
+using Ghost.Graphics.Core;
+
+namespace Ghost.Engine.RenderPipeline;
+
+internal class GhostRenderPipeline : IRenderPipeline
+{
+    private readonly RenderSystem _renderSystem;
+
+    private readonly GPUScene _gpuScene;
+
+    public GPUScene GPUScene => _gpuScene;
+
+    public GhostRenderPipeline(RenderSystem renderSystem)
+    {
+        _renderSystem = renderSystem;
+        _gpuScene = new GPUScene(renderSystem.GraphicsEngine.ResourceAllocator, renderSystem.GraphicsEngine.ResourceDatabase, 102_400u); // 102.4k objects should be enough for now
+    }
+
+    public void Render(RenderContext ctx, int frameIndex, IRenderPayload payload)
+    {
+        var ghostPayload = (GhostRenderPayload)payload;
+
+        var resourceManager = _renderSystem.ResourceManager;
+        var resourceDatabase = _renderSystem.GraphicsEngine.ResourceDatabase;
+
+        foreach (ref readonly var request in ghostPayload.RenderRequests)
+        {
+            if (!RenderPipelineUtility.GetViewAndProjectionMatrices(_renderSystem, in request, out var view, out var projection, out var screenSize))
+            {
+                continue;
+            }
+        }
+    }
+
+    public void Dispose()
+    {
+        throw new NotImplementedException();
+    }
+}

src/Runtime/Ghost.Engine/RenderPipeline/GhostRenderPipelineSettings.cs

@@ -0,0 +1,93 @@
+using Ghost.Engine.Components;
+using Ghost.Graphics;
+using Ghost.Graphics.Core;
+using Misaki.HighPerformance.LowLevel.Collections;
+using Misaki.HighPerformance.Mathematics;
+using System.Collections.Concurrent;
+
+namespace Ghost.Engine.RenderPipeline;
+
+internal sealed class GhostRenderPayload : IRenderPayload
+{
+    public struct AddInstanceRequest
+    {
+        public MeshInstance meshInstance;
+        public float4x4 localToWorld;
+        public uint instanceId;
+    }
+
+    public struct RemoveInstanceRequest
+    {
+        public uint instanceId;
+        public uint swapWithInstanceId;
+    }
+
+    private readonly GhostRenderPipeline _renderPipeline;
+
+    private UnsafeList<RenderRequest> _renderRequests;
+
+    // TODO: Consider using a more efficient data structure for these queues, such as a lock-free ring buffer or a custom concurrent queue implementation.
+    private readonly ConcurrentQueue<AddInstanceRequest> _addRequest;
+    private readonly ConcurrentQueue<RemoveInstanceRequest> _removeRequest;
+
+    public ReadOnlySpan<RenderRequest> RenderRequests => _renderRequests;
+
+    public ConcurrentQueue<AddInstanceRequest> AddRequest => _addRequest;
+    public ConcurrentQueue<RemoveInstanceRequest> RemoveRequest => _removeRequest;
+
+    public GhostRenderPayload(GhostRenderPipeline renderPipeline)
+    {
+        _renderPipeline = renderPipeline;
+
+        _renderRequests = new UnsafeList<RenderRequest>(4, Misaki.HighPerformance.LowLevel.Buffer.Allocator.Persistent);
+        _addRequest = new ConcurrentQueue<AddInstanceRequest>();
+        _removeRequest = new ConcurrentQueue<RemoveInstanceRequest>();
+    }
+
+    // NOTE: This is not thread safe.
+    public void AddRenderRequest(ref readonly RenderRequest renderRequest)
+    {
+        _renderRequests.Add(renderRequest);
+    }
+
+    public uint AddInstance(float4x4 ltw, ref readonly MeshInstance meshInstance)
+    {
+        var index = _renderPipeline.GPUScene.AddInstance();
+        _addRequest.Enqueue(new AddInstanceRequest { instanceId = index, localToWorld = ltw, meshInstance = meshInstance });
+        return index;
+    }
+
+    public void RemoveInstance(uint instanceId)
+    {
+        var swapWithInstanceId = _renderPipeline.GPUScene.RemoveInstance(instanceId);
+        if (swapWithInstanceId != ~0u)
+        {
+            _removeRequest.Enqueue(new RemoveInstanceRequest { instanceId = instanceId, swapWithInstanceId = swapWithInstanceId });
+        }
+    }
+
+    public void Reset()
+    {
+        _renderRequests.Clear();
+        _addRequest.Clear();
+        _removeRequest.Clear();
+    }
+
+    public void Dispose()
+    {
+        _renderRequests.Dispose();
+    }
+}
+
+internal class GhostRenderPipelineSettings : IRenderPipelineSettings
+{
+    public IRenderPipeline CreatePipeline(RenderSystem renderSystem)
+    {
+        return new GhostRenderPipeline(renderSystem);
+    }
+
+    public IRenderPayload CreatePayload(RenderSystem renderSystem, IRenderPipeline _renderPipeline)
+    {
+        return new GhostRenderPayload((GhostRenderPipeline)_renderPipeline);
+    }
+}

src/Runtime/Ghost.Engine/Systems/AddGPUInstanceSystem.cs

@@ -0,0 +1,52 @@
+using Ghost.Core;
+using Ghost.Engine.Components;
+using Ghost.Engine.RenderPipeline;
+using Ghost.Entities;
+using Ghost.Graphics;
+using Misaki.HighPerformance.Utilities;
+
+namespace Ghost.Engine.Systems;
+
+[RenderPipelineSystem<GhostRenderPipelineSettings>]
+internal class AddGPUInstanceSystem : SystemBase
+{
+    private RenderSystem _renderSystem = null!;
+
+    private Identifier<EntityQuery> _meshInstanceQueryID;
+
+    protected override void OnInitialize(ref readonly SystemAPI systemAPI)
+    {
+        _renderSystem = systemAPI.World.GetService<RenderSystem>();
+
+        _meshInstanceQueryID = QueryBuilder.Create()
+            .WithAll<MeshInstance, LocalToWorld>()
+            .WithAbsent<GPUInstanceRef>()
+            .Build(systemAPI.World, true);
+
+        RequireQueryForUpdate(_meshInstanceQueryID);
+    }
+
+    protected override void OnUpdate(ref readonly SystemAPI systemAPI)
+    {
+        var payload = (GhostRenderPayload)_renderSystem.GetCurrentFramePayload();
+
+        ref var meshInstanceQuery = ref systemAPI.World.ComponentManager.GetEntityQueryReference(_meshInstanceQueryID);
+
+        foreach (var chunk in meshInstanceQuery.GetChunkIterator())
+        {
+            var meshInstances = chunk.GetComponentData<MeshInstance>();
+            var localToWorlds = chunk.GetComponentData<LocalToWorld>();
+            var entities = chunk.GetEntities();
+
+            for (var i = 0; i < chunk.EntityCount; i++)
+            {
+                ref readonly var meshInstance = ref meshInstances.GetElementUnsafe(i);
+                var localToWorld = localToWorlds.GetElementUnsafe(i);
+                var entity = entities.GetElementUnsafe(i);
+
+                var index = payload.AddInstance(localToWorld.matrix, in meshInstance);
+                systemAPI.World.EntityCommandBuffer.AddComponent(entity, new GPUInstanceRef { gpuSceneIndex = index });
+            }
+        }
+    }
+}

src/Runtime/Ghost.Engine/Systems/RemoveGPUInstanceSystem.cs

@@ -0,0 +1,50 @@
+using Ghost.Core;
+using Ghost.Engine.Components;
+using Ghost.Engine.RenderPipeline;
+using Ghost.Entities;
+using Ghost.Graphics;
+using Misaki.HighPerformance.Utilities;
+
+namespace Ghost.Engine.Systems;
+
+[RenderPipelineSystem<GhostRenderPipelineSettings>]
+internal class RemoveGPUInstanceSystem : SystemBase
+{
+    private RenderSystem _renderSystem = null!;
+
+    private Identifier<EntityQuery> _gpuInstanceQueryID;
+
+    protected override void OnInitialize(ref readonly SystemAPI systemAPI)
+    {
+        _renderSystem = systemAPI.World.GetService<RenderSystem>();
+
+        _gpuInstanceQueryID = QueryBuilder.Create()
+            .WithAll<GPUInstanceRef>()
+            .WithAbsent<MeshInstance>()
+            .Build(systemAPI.World, true);
+
+        RequireQueryForUpdate(_gpuInstanceQueryID);
+    }
+
+    protected override void OnUpdate(ref readonly SystemAPI systemAPI)
+    {
+        var payload = (GhostRenderPayload)_renderSystem.GetCurrentFramePayload();
+
+        ref var gpuInstanceQuery = ref systemAPI.World.ComponentManager.GetEntityQueryReference(_gpuInstanceQueryID);
+
+        foreach (var chunk in gpuInstanceQuery.GetChunkIterator())
+        {
+            var gpuInstanceRefs = chunk.GetComponentData<GPUInstanceRef>();
+            var entities = chunk.GetEntities();
+
+            for (var i = 0; i < chunk.EntityCount; i++)
+            {
+                var gpuInstance = gpuInstanceRefs.GetElementUnsafe(i);
+                var entity = entities.GetElementUnsafe(i);
+
+                payload.RemoveInstance(gpuInstance.gpuSceneIndex);
+                systemAPI.World.EntityCommandBuffer.RemoveComponent<GPUInstanceRef>(entity);
+            }
+        }
+    }
+}

src/Runtime/Ghost.Engine/Systems/RenderPipelineSystemAttribute.cs

@@ -1,5 +1,5 @@
 using Ghost.Entities;
-using Ghost.Graphics.RenderPipeline;
+using Ghost.Graphics;
 
 namespace Ghost.Engine.Systems;
 
@@ -17,20 +17,20 @@ public class RenderPipelineSystemAttribute<T> : RenderPipelineSystemAttribute
 
 public static class RenderPipelineSystemRegistry
 {
-    private static readonly Dictionary<Type, List<Func<ISystem>>> s_renderPipelineSystems = new();
+    private static readonly Dictionary<nint, List<Func<ISystem>>> s_renderPipelineSystems = new();
 
-    public static void RegisterRenderPipelineSystem(Type settingsType, Func<ISystem> systemFactory)
+    public static void RegisterRenderPipelineSystem(nint settingsType, Func<ISystem> systemFactory)
     {
         if (!s_renderPipelineSystems.TryGetValue(settingsType, out var systems))
         {
-            systems = new List<Func<ISystem>>();
+            systems = new List<Func<ISystem>>(4);
             s_renderPipelineSystems[settingsType] = systems;
         }
 
         systems.Add(systemFactory);
     }
 
-    internal static IEnumerable<Func<ISystem>> GetRenderPipelineSystems(Type settingsType)
+    internal static IEnumerable<Func<ISystem>> GetRenderPipelineSystems(nint settingsType)
     {
         if (s_renderPipelineSystems.TryGetValue(settingsType, out var systems))
         {