From d0076c852fce1613ae285530dddbf1e79a020eee Mon Sep 17 00:00:00 2001 From: Misaki Date: Fri, 8 May 2026 17:06:37 +0900 Subject: [PATCH] Add editor shader compilation bridge & pipeline cache mgmt Introduced EditorShaderCompilerBridge and IShaderCompilationBridge for async shader variant compilation and cache invalidation in the editor. Refactored ShaderLibrary to support the bridge, updating hash/caching logic and triggering compilation on cache misses. Changed pipeline library to use ulong content hashes and added stale pipeline eviction. Updated EngineCore and render code to integrate the new system. Added unit tests for ShaderLibrary cache and bridge behavior. Minor improvements to shader property code generation and test generator. --- .../Services/EditorShaderCompilerBridge.cs | 214 ++++++++++++++++++ src/Runtime/Ghost.Engine/EngineCore.cs | 4 +- .../ShaderPropertiesGenerator.cs | 2 +- .../D3D12GraphicsEngine.cs | 2 + .../D3D12PipelineLibrary.cs | 74 +++++- src/Runtime/Ghost.Graphics.RHI/Common.cs | 4 +- .../Ghost.Graphics.RHI/IPipelineLibrary.cs | 4 + .../IShaderCompilationBridge.cs | 17 ++ .../Ghost.Graphics/Core/RenderContext.cs | 9 +- .../IShaderCompilationBridge.cs | 19 -- .../RenderGraphModule/RenderGraphContext.cs | 16 +- .../Ghost.Graphics/Services/ShaderLibrary.cs | 67 +++++- .../Graphics/ShaderLibraryTest.cs | 166 ++++++++++++++ src/build_log.txt | Bin 0 -> 30284 bytes src/test_generator.cs | 34 +++ 15 files changed, 583 insertions(+), 49 deletions(-) create mode 100644 src/Editor/Ghost.Editor.Core/Services/EditorShaderCompilerBridge.cs create mode 100644 src/Runtime/Ghost.Graphics.RHI/IShaderCompilationBridge.cs delete mode 100644 src/Runtime/Ghost.Graphics/IShaderCompilationBridge.cs create mode 100644 src/Test/Ghost.UnitTest/Graphics/ShaderLibraryTest.cs create mode 100644 src/build_log.txt create mode 100644 src/test_generator.cs diff --git a/src/Editor/Ghost.Editor.Core/Services/EditorShaderCompilerBridge.cs b/src/Editor/Ghost.Editor.Core/Services/EditorShaderCompilerBridge.cs new file mode 100644 index 0000000..ef26416 --- /dev/null +++ b/src/Editor/Ghost.Editor.Core/Services/EditorShaderCompilerBridge.cs @@ -0,0 +1,214 @@ +using Ghost.Core; +using Ghost.Core.Graphics; +using Ghost.Editor.Core.Assets; +using Ghost.Editor.Core.Contracts; +using Ghost.Editor.Core.Utilities; +using Ghost.Graphics.Core; +using Ghost.Graphics.RHI; +using Ghost.Graphics.Services; +using Ghost.Engine; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using System.Collections.Concurrent; +using Misaki.HighPerformance.LowLevel.Collections; + +namespace Ghost.Editor.Core.Services; + +[EditorInjection(EditorInjectionAttribute.ServiceLifetime.Singleton, typeof(IShaderCompilationBridge))] +internal sealed class EditorShaderCompilerBridge : IShaderCompilationBridge, IDisposable +{ + private readonly IAssetRegistry _assetRegistry; + private readonly IShaderCompiler _compiler; + private readonly ConcurrentDictionary _shaderIdToAssetId = new(); + private readonly IServiceProvider _serviceProvider; + + public event Action, ulong>? OnShaderVariantCompiled; + + public EditorShaderCompilerBridge(IAssetRegistry assetRegistry, IServiceProvider serviceProvider) + { + _assetRegistry = assetRegistry; + _serviceProvider = serviceProvider; + _compiler = new DXCShaderCompiler(); + + _assetRegistry.OnAssetImported += OnAssetImported; + } + + private void OnAssetImported(object? sender, Guid guid) + { + var path = _assetRegistry.GetAssetPath(guid); + if (path != null && (path.EndsWith(".gshdr") || path.EndsWith(".gcomp"))) + { + var result = _assetRegistry.LoadAssetAsync(guid).AsTask().Result; + if (result.IsSuccess) + { + ulong nameHash = 0; + if (result.Value is GraphicsShaderAsset graphicsAsset) + { + nameHash = RHIUtility.GetShaderID(graphicsAsset.Descriptor.Name); + } + else if (result.Value is ComputeShaderAsset computeAsset) + { + nameHash = RHIUtility.GetShaderID(computeAsset.Descriptor.Name); + } + + if (nameHash != 0) + { + _shaderIdToAssetId[nameHash] = guid; + + var engineCore = _serviceProvider.GetService(); + if (engineCore != null) + { + var shaderLibrary = engineCore.RenderSystem.ShaderLibrary; + var pipelineLibrary = engineCore.RenderSystem.GraphicsEngine.PipelineLibrary; + shaderLibrary.InvalidateShaderCache(nameHash, pipelineLibrary); + } + } + } + } + } + + public void RequestCompilation(ulong shaderId, int passIndex, Key64 variantKey) + { + Task.Run(async () => + { + if (!_shaderIdToAssetId.TryGetValue(shaderId, out var guid)) + { + var catalog = _assetRegistry.GetAssetCatalog(); + foreach (var (assetGuid, path) in catalog.EnumerateAll()) + { + if (path.EndsWith(".gshdr") || path.EndsWith(".gcomp")) + { + var result = await _assetRegistry.LoadAssetAsync(assetGuid); + if (result.IsSuccess) + { + ulong nameHash = 0; + if (result.Value is GraphicsShaderAsset graphicsAsset) + { + nameHash = RHIUtility.GetShaderID(graphicsAsset.Descriptor.Name); + } + else if (result.Value is ComputeShaderAsset computeAsset) + { + nameHash = RHIUtility.GetShaderID(computeAsset.Descriptor.Name); + } + if (nameHash != 0) + { + _shaderIdToAssetId[nameHash] = assetGuid; + } + } + } + } + } + + if (_shaderIdToAssetId.TryGetValue(shaderId, out var assetId)) + { + var assetResult = await _assetRegistry.LoadAssetAsync(assetId); + if (assetResult.IsSuccess) + { + if (assetResult.Value is GraphicsShaderAsset graphicsAsset) + { + var pass = graphicsAsset.Descriptor.Passes[passIndex]; + await CompileGraphicsPassAsync(shaderId, passIndex, variantKey, pass); + } + else if (assetResult.Value is ComputeShaderAsset computeAsset) + { + var code = computeAsset.Descriptor.ShaderCodes[passIndex]; + await CompileComputePassAsync(shaderId, passIndex, variantKey, code); + } + } + } + }); + } + + private unsafe Task CompileGraphicsPassAsync(ulong shaderId, int passIndex, Key64 variantKey, PassDescriptor pass) + { + // For simplicity, just compile the pixel shader. A real implementation would compile + // all stages (Mesh/Amp/Vertex/Pixel) defined in the pass descriptor. + var config = new ShaderCompilationConfig + { + shaderCode = pass.pixelShaderCode.code, + entryPoint = pass.pixelShaderCode.entryPoint, + stage = ShaderStage.PixelShader, + defines = pass.defines, + model = ShaderModel.SM_6_6 + }; + + var compileResult = _compiler.Compile(in config, Misaki.HighPerformance.LowLevel.Buffer.AllocationHandle.Persistent); + if (compileResult.IsSuccess) + { + var engineCore = _serviceProvider.GetService(); + if (engineCore != null) + { + using var bytecodeArray = compileResult.Value; + + var byteCode = new ShaderByteCode + { + pCode = (byte*)bytecodeArray.GetUnsafePtr(), + size = (ulong)bytecodeArray.Length + }; + + // Assume 1 stage for now. In reality, we'd pass an array of ShaderByteCode for all stages. + var byteCodes = new Span(ref byteCode); + + engineCore.RenderSystem.ShaderLibrary.CacheCompiledResult(shaderId, passIndex, variantKey, byteCodes); + + // Get the generated hash to fire the event + var dataSpan = new ReadOnlySpan(byteCode.pCode, (int)byteCode.size); + var hash = System.IO.Hashing.XxHash64.HashToUInt64(dataSpan); + OnShaderVariantCompiled?.Invoke(variantKey, hash); + } + } + else + { + Ghost.Core.Logger.Error($"Failed to compile graphics shader {shaderId}: {compileResult.Message}"); + } + + return Task.CompletedTask; + } + + private unsafe Task CompileComputePassAsync(ulong shaderId, int passIndex, Key64 variantKey, ShaderCode code) + { + var config = new ShaderCompilationConfig + { + shaderCode = code.code, + entryPoint = code.entryPoint, + stage = ShaderStage.ComputeShader, + defines = Array.Empty(), + model = ShaderModel.SM_6_6 + }; + + var compileResult = _compiler.Compile(in config, Misaki.HighPerformance.LowLevel.Buffer.AllocationHandle.Persistent); + if (compileResult.IsSuccess) + { + var engineCore = _serviceProvider.GetService(); + if (engineCore != null) + { + using var bytecodeArray = compileResult.Value; + + var byteCode = new ShaderByteCode + { + pCode = (byte*)bytecodeArray.GetUnsafePtr(), + size = (ulong)bytecodeArray.Length + }; + + var byteCodes = new Span(ref byteCode); + + engineCore.RenderSystem.ShaderLibrary.CacheCompiledResult(shaderId, passIndex, variantKey, byteCodes); + + var dataSpan = new ReadOnlySpan(byteCode.pCode, (int)byteCode.size); + var hash = System.IO.Hashing.XxHash64.HashToUInt64(dataSpan); + OnShaderVariantCompiled?.Invoke(variantKey, hash); + } + } + else + { + Ghost.Core.Logger.Error($"Failed to compile compute shader {shaderId}: {compileResult.Message}"); + } + + return Task.CompletedTask; + } + + public void Dispose() + { + _assetRegistry.OnAssetImported -= OnAssetImported; + } +} diff --git a/src/Runtime/Ghost.Engine/EngineCore.cs b/src/Runtime/Ghost.Engine/EngineCore.cs index b891508..3db9ef8 100644 --- a/src/Runtime/Ghost.Engine/EngineCore.cs +++ b/src/Runtime/Ghost.Engine/EngineCore.cs @@ -1,6 +1,7 @@ using Ghost.Core.Graphics; using Ghost.Engine.RenderPipeline; using Ghost.Graphics; +using Ghost.Graphics.RHI; using Misaki.HighPerformance.Jobs; using Misaki.HighPerformance.Mathematics; @@ -25,7 +26,7 @@ public sealed partial class EngineCore : IDisposable internal RenderSystem RenderSystem => _renderSystem; internal AssetManager AssetManager => _assetManager; - public EngineCore(IContentProvider contentProvider) + public EngineCore(IContentProvider contentProvider, IShaderCompilationBridge? shaderCompilationBridge = null) { _contentProvider = contentProvider; @@ -46,6 +47,7 @@ public sealed partial class EngineCore : IDisposable InitialRenderPipelineSettings = new GhostRenderPipelineSettings(), ResourceStreamingProcessor = _streamingProcessor, ShaderCacheDirectory = "ShaderCache", + ShaderCompilationBridge = shaderCompilationBridge, }; _renderSystem = new RenderSystem(renderingDesc); diff --git a/src/Runtime/Ghost.Generator/ShaderPropertiesGenerator.cs b/src/Runtime/Ghost.Generator/ShaderPropertiesGenerator.cs index 8647695..c2dbead 100644 --- a/src/Runtime/Ghost.Generator/ShaderPropertiesGenerator.cs +++ b/src/Runtime/Ghost.Generator/ShaderPropertiesGenerator.cs @@ -90,7 +90,7 @@ namespace Ghost.Generator isEnabledByDefault: true), info.TypeSymbol.Locations.FirstOrDefault())); continue; } - + var definedSymbol = $"__{info.Name.ToUpper()}_G_HLSL"; var fieldsBuilder = new StringBuilder(); diff --git a/src/Runtime/Ghost.Graphics.D3D12/D3D12GraphicsEngine.cs b/src/Runtime/Ghost.Graphics.D3D12/D3D12GraphicsEngine.cs index 3c66448..780a2cf 100644 --- a/src/Runtime/Ghost.Graphics.D3D12/D3D12GraphicsEngine.cs +++ b/src/Runtime/Ghost.Graphics.D3D12/D3D12GraphicsEngine.cs @@ -144,6 +144,7 @@ internal class D3D12GraphicsEngine : IGraphicsEngine _cpuFrame = cpuFrame; _resourceDatabase.BeginFrame(cpuFrame); + _pipelineLibrary.BeginFrame(cpuFrame); } public void EndFrame(ulong gpuFrame) @@ -151,6 +152,7 @@ internal class D3D12GraphicsEngine : IGraphicsEngine Logger.DebugAssert(!_disposed); _resourceDatabase.EndFrame(gpuFrame); + _pipelineLibrary.EndFrame(gpuFrame); while (_commandBufferReturnQueue.TryPeek(out var entry) && entry.returnFrame < gpuFrame) { diff --git a/src/Runtime/Ghost.Graphics.D3D12/D3D12PipelineLibrary.cs b/src/Runtime/Ghost.Graphics.D3D12/D3D12PipelineLibrary.cs index 98e662d..a5198bc 100644 --- a/src/Runtime/Ghost.Graphics.D3D12/D3D12PipelineLibrary.cs +++ b/src/Runtime/Ghost.Graphics.D3D12/D3D12PipelineLibrary.cs @@ -18,6 +18,7 @@ internal struct D3D12PipelineState : IDisposable { public UniquePtr pso; public Key64 shaderVariant; + public ulong contentHash; public void Dispose() { @@ -33,6 +34,17 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object private UnsafeHashMap _pipelineCache; + private struct StalePipeline + { + public UInt128 pipelineKey; + public D3D12PipelineState pso; + public ulong frameAdded; + } + + private UnsafeList _stalePipelines; + private ulong _currentCpuFrame; + private ulong _completedGpuFrame; + public ID3D12RootSignature* DefaultRootSignature => _defaultRootSignature.Get(); private static ID3D12PipelineLibrary1* CreateLibrary(D3D12RenderDevice device, string? filePath) @@ -61,6 +73,7 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object _device = device; _pipelineCache = new UnsafeHashMap(32, AllocationHandle.Persistent); + _stalePipelines = new UnsafeList(16, AllocationHandle.Persistent); CreateDefaultRootSignature().ThrowIfFailed(); } @@ -145,7 +158,7 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object return D3D12Utility.D3D12_DEPTH_STENCIL_DESC_CREATE(depthEnabled, writeEnabled, cmp); } - private Result CreatePSO(Key64 shaderVariantKey, UInt128 pipelineKey, D3D12_PIPELINE_STATE_STREAM_DESC* pStreamDesc) + private Result CreatePSO(ulong contentHash, Key64 shaderVariantKey, UInt128 pipelineKey, D3D12_PIPELINE_STATE_STREAM_DESC* pStreamDesc) { ID3D12PipelineState* pPipelineState = default; @@ -171,6 +184,7 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object D3D12PipelineState pso = default; pso.shaderVariant = shaderVariantKey; + pso.contentHash = contentHash; pso.pso.Attach(pPipelineState); _pipelineCache[pipelineKey] = pso; @@ -187,7 +201,7 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object } var passAttachmentKey = new PassAttachmentHash(desc.RtvFormats, desc.DsvFormat); - var pipelineKey = RHIUtility.CreateGraphicsPipelineKey(desc.VariantKey, desc.PipelineOption, passAttachmentKey); + var pipelineKey = RHIUtility.CreateGraphicsPipelineKey(desc.CompiledHash, desc.PipelineOption, passAttachmentKey); if (!_pipelineCache.ContainsKey(pipelineKey)) { @@ -243,7 +257,7 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object SizeInBytes = (nuint)sizeof(CD3DX12_PIPELINE_MESH_STATE_STREAM) }; - var result = CreatePSO(desc.VariantKey, pipelineKey, &streamDesc); + var result = CreatePSO(desc.CompiledHash, desc.VariantKey, pipelineKey, &streamDesc); if (result.IsFailure) { return result; @@ -258,7 +272,7 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object { AssertNotDisposed(); - var pipelineKey = RHIUtility.CreateComputePipelineKey(desc.VariantKey); + var pipelineKey = RHIUtility.CreateComputePipelineKey(desc.CompiledHash); if (!_pipelineCache.ContainsKey(pipelineKey)) { fixed (byte* pCSByteCode = desc.CsCode) @@ -272,7 +286,7 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object SizeInBytes = (nuint)sizeof(CD3DX12_PIPELINE_STATE_STREAM_CS) }; - var result = CreatePSO(desc.VariantKey, pipelineKey, &streamDesc); + var result = CreatePSO(desc.CompiledHash, desc.VariantKey, pipelineKey, &streamDesc); if (result.IsFailure) { return result; @@ -300,6 +314,55 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object return Error.NotFound; } + public void BeginFrame(ulong cpuFrame) + { + _currentCpuFrame = cpuFrame; + } + + public void EndFrame(ulong gpuFrame) + { + _completedGpuFrame = gpuFrame; + + // Process stale pipelines and dispose them if they are no longer in flight + for (int i = _stalePipelines.Count - 1; i >= 0; i--) + { + var stale = _stalePipelines[i]; + if (_completedGpuFrame >= stale.frameAdded) + { + stale.pso.Dispose(); + _stalePipelines.RemoveAtSwapBack(i); + } + } + } + + public void EvictStalePipelines(ulong oldContentHash) + { + // Find all pipelines with matching oldContentHash + using var keysToRemove = new UnsafeList(8, AllocationHandle.Temp); + + foreach (var kvp in _pipelineCache) + { + if (kvp.Value.contentHash == oldContentHash) + { + keysToRemove.Add(kvp.Key); + } + } + + foreach (var key in keysToRemove) + { + if (_pipelineCache.TryGetValue(key, out var pso)) + { + _stalePipelines.Add(new StalePipeline + { + pipelineKey = key, + pso = pso, + frameAdded = _currentCpuFrame + }); + _pipelineCache.Remove(key); + } + } + } + protected override void Dispose(bool disposing) { foreach (var kvp in _pipelineCache) @@ -308,6 +371,7 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object } _pipelineCache.Dispose(); + _stalePipelines.Dispose(); _defaultRootSignature.Dispose(); } } diff --git a/src/Runtime/Ghost.Graphics.RHI/Common.cs b/src/Runtime/Ghost.Graphics.RHI/Common.cs index 0d890dc..cf7dba0 100644 --- a/src/Runtime/Ghost.Graphics.RHI/Common.cs +++ b/src/Runtime/Ghost.Graphics.RHI/Common.cs @@ -219,7 +219,7 @@ public readonly struct PassAttachmentHash : IEquatable public ref struct GraphicsPSODesc { - public UInt128 CompiledHash + public ulong CompiledHash { get; set; } @@ -262,7 +262,7 @@ public ref struct GraphicsPSODesc public ref struct ComputePSODesc { - public UInt128 CompiledHash + public ulong CompiledHash { get; set; } diff --git a/src/Runtime/Ghost.Graphics.RHI/IPipelineLibrary.cs b/src/Runtime/Ghost.Graphics.RHI/IPipelineLibrary.cs index ac2c5b3..53bdf83 100644 --- a/src/Runtime/Ghost.Graphics.RHI/IPipelineLibrary.cs +++ b/src/Runtime/Ghost.Graphics.RHI/IPipelineLibrary.cs @@ -9,4 +9,8 @@ public interface IPipelineLibrary : IDisposable bool HasPipelineStateObject(UInt128 key); Result> CreateGraphicsPipeline(ref readonly GraphicsPSODesc desc); Result> CreateComputePipeline(ref readonly ComputePSODesc desc); + + void BeginFrame(ulong cpuFrame); + void EndFrame(ulong gpuFrame); + void EvictStalePipelines(ulong oldContentHash); } diff --git a/src/Runtime/Ghost.Graphics.RHI/IShaderCompilationBridge.cs b/src/Runtime/Ghost.Graphics.RHI/IShaderCompilationBridge.cs new file mode 100644 index 0000000..d43d8d8 --- /dev/null +++ b/src/Runtime/Ghost.Graphics.RHI/IShaderCompilationBridge.cs @@ -0,0 +1,17 @@ +using Ghost.Core; + +namespace Ghost.Graphics.RHI; + +public interface IShaderCompilationBridge +{ + /// + /// Request the bridge to recompile a shader variant or handle cache misses. + /// This is typically called by the ShaderLibrary when a variant hash is not found. + /// + void RequestCompilation(ulong shaderId, int passIndex, Key64 variantKey); + + /// + /// Event triggered when a shader variant has been successfully compiled and updated. + /// + event Action, ulong> OnShaderVariantCompiled; +} diff --git a/src/Runtime/Ghost.Graphics/Core/RenderContext.cs b/src/Runtime/Ghost.Graphics/Core/RenderContext.cs index 7d9cc76..ca520a0 100644 --- a/src/Runtime/Ghost.Graphics/Core/RenderContext.cs +++ b/src/Runtime/Ghost.Graphics/Core/RenderContext.cs @@ -379,7 +379,7 @@ public readonly unsafe ref struct RenderContext var variantKey = RHIUtility.CreateShaderVariantKey(entryHash, in keywordSet); // TODO: Refactor this into a helper method. - var (compiledHash, error) = ShaderLibrary.GetCompiledHash(variantKey); + var (compiledHash, error) = ShaderLibrary.GetCompiledHash(shader.UniqueID, entryIndex, variantKey); if (error.IsFailure) { // TODO: Fallback to an error material. @@ -391,12 +391,11 @@ public readonly unsafe ref struct RenderContext if (!PipelineLibrary.HasPipelineStateObject(pipelineKey)) { - using var scope = AllocationManager.CreateStackScope(); - var compiledCacheResult = ShaderLibrary.GetCompiledCache(shader.UniqueID, entryIndex, scope.AllocationHandle); + var compiledCacheResult = ShaderLibrary.GetCompiledCache(shader.UniqueID, entryIndex); if (compiledCacheResult.IsFailure) { - // TODO: Fallback to a checkerboard shader. - throw new InvalidOperationException("Failed to load compiled shader cache for pipeline state object creation."); + Logger.Warning($"Failed to load compiled shader cache for compute pipeline {pipelineKey}. Skipping compute dispatch."); + return; } var cache = compiledCacheResult.Value; diff --git a/src/Runtime/Ghost.Graphics/IShaderCompilationBridge.cs b/src/Runtime/Ghost.Graphics/IShaderCompilationBridge.cs deleted file mode 100644 index d50f28d..0000000 --- a/src/Runtime/Ghost.Graphics/IShaderCompilationBridge.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace Ghost.Graphics; - -public interface IShaderCompilationBridge -{ - bool TryGetBytecode(ulong manifestKey, out ReadOnlyMemory bytecode); - bool IsCompiling(ulong manifestKey); -} - -// NOTE: For testing only. -internal sealed class NullShaderCompilationBridge : IShaderCompilationBridge -{ - public bool TryGetBytecode(ulong manifestKey, out ReadOnlyMemory bytecode) - { - bytecode = default; - return false; // Always fall through to ShaderLibrary's disk cache - } - - public bool IsCompiling(ulong manifestKey) => false; -} diff --git a/src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraphContext.cs b/src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraphContext.cs index 1008e0b..dbaecb3 100644 --- a/src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraphContext.cs +++ b/src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraphContext.cs @@ -172,7 +172,7 @@ internal sealed class RenderGraphContext : IUnsafeRenderContext var variantMask = material._keywordMask & pass.KeywordIDs; var variantKey = RHIUtility.CreateShaderVariantKey(pass.Key, in variantMask); - var (compiledHash, error) = _shaderLibrary.GetCompiledHash(variantKey); + var (compiledHash, error) = _shaderLibrary.GetCompiledHash(shader.UniqueID, material.ActivePassIndex, variantKey); if (error.IsFailure) { // TODO: Fallback to a default shader or show an error material. @@ -183,11 +183,11 @@ internal sealed class RenderGraphContext : IUnsafeRenderContext if (!_pipelineLibrary.HasPipelineStateObject(pipelineKey)) { - using var scope = AllocationManager.CreateStackScope(); - var compiledCacheResult = _shaderLibrary.GetCompiledCache(shader.UniqueID, material.ActivePassIndex, scope.AllocationHandle); + var compiledCacheResult = _shaderLibrary.GetCompiledCache(shader.UniqueID, material.ActivePassIndex); if (compiledCacheResult.IsFailure) { - throw new InvalidOperationException("Failed to load compiled shader cache for pipeline state object creation."); + Logger.Warning($"Failed to load compiled shader cache for graphics pipeline {pipelineKey}. Skipping draw call."); + return; } var cache = compiledCacheResult.Value; @@ -277,7 +277,7 @@ internal sealed class RenderGraphContext : IUnsafeRenderContext var keywordSet = new LocalKeywordSet(); // TODO: Support keywords in compute shader. var variantKey = RHIUtility.CreateShaderVariantKey(entryHash, in keywordSet); - var (compiledHash, error) = _shaderLibrary.GetCompiledHash(variantKey); + var (compiledHash, error) = _shaderLibrary.GetCompiledHash(shader.UniqueID, entryIndex, variantKey); if (error.IsFailure) { // TODO: Fallback to a default shader or show an error material. @@ -288,11 +288,11 @@ internal sealed class RenderGraphContext : IUnsafeRenderContext if (!_pipelineLibrary.HasPipelineStateObject(pipelineKey)) { - using var scope = AllocationManager.CreateStackScope(); - var compiledCacheResult = _shaderLibrary.GetCompiledCache(shader.UniqueID, entryIndex, scope.AllocationHandle); + var compiledCacheResult = _shaderLibrary.GetCompiledCache(shader.UniqueID, entryIndex); if (compiledCacheResult.IsFailure) { - throw new InvalidOperationException("Failed to load compiled shader cache for pipeline state object creation."); + Logger.Warning($"Failed to load compiled shader cache for compute pipeline {pipelineKey}. Skipping compute dispatch."); + return; } var cache = compiledCacheResult.Value; diff --git a/src/Runtime/Ghost.Graphics/Services/ShaderLibrary.cs b/src/Runtime/Ghost.Graphics/Services/ShaderLibrary.cs index 9e27ec6..16ccd3e 100644 --- a/src/Runtime/Ghost.Graphics/Services/ShaderLibrary.cs +++ b/src/Runtime/Ghost.Graphics/Services/ShaderLibrary.cs @@ -3,6 +3,7 @@ using Ghost.Core.Utilities; using Ghost.Graphics.RHI; using Misaki.HighPerformance.LowLevel.Buffer; using Misaki.HighPerformance.LowLevel.Collections; +using Misaki.HighPerformance.LowLevel.Utilities; using System.IO.Hashing; using System.Runtime.CompilerServices; @@ -80,6 +81,16 @@ internal unsafe class ShaderLibrary : IDisposable _cacheDirectory = cacheDirectory; _shaderCompilationBridge = shaderCompilationBridge; + + if (_shaderCompilationBridge != null) + { + _shaderCompilationBridge.OnShaderVariantCompiled += OnVariantCompiled; + } + } + + private void OnVariantCompiled(Key64 variantKey, ulong newCompiledHash) + { + _variantToCompiledHash[variantKey] = newCompiledHash; } private string GetShaderCacheFilePath(ulong hash) @@ -117,14 +128,17 @@ internal unsafe class ShaderLibrary : IDisposable }; var offsets = stackalloc ulong[byteCodes.Length]; - var offset = (ulong)(sizeof(CacheHeader) + (sizeof(ulong) * byteCodes.Length)); + var offset = (nuint)(sizeof(CacheHeader) + (sizeof(ulong) * byteCodes.Length)); for (var i = 0; i < byteCodes.Length; i++) { offsets[i] = offset; - offset += byteCodes[i].size; + offset += (nuint)byteCodes[i].size; } - var data = new MemoryBlock((nuint)offset, 8, AllocationHandle.Persistent); + var alignment = Math.Max(Math.Max(MemoryUtility.AlignOf(), MemoryUtility.AlignOf()), 8); + offset = MemoryUtility.AlignUp(offset, alignment); + + var data = new MemoryBlock(offset, alignment, AllocationHandle.Persistent); var writer = new SpanWriter(data.AsSpan()); writer.Write(header); @@ -141,7 +155,18 @@ internal unsafe class ShaderLibrary : IDisposable writer.WriteSpan(src); } - var codeHash = XxHash64.HashToUInt64(data.AsSpan()); + // Compute hash from bytecode only + var codeHash = 0UL; + if (byteCodes.Length > 0) + { + ulong totalBytecodeSize = 0; + for (int i = 0; i < byteCodes.Length; i++) totalBytecodeSize += byteCodes[i].size; + + // We skip the header and offsets at the beginning of the MemoryBlock + var bytecodeSpan = data.AsSpan().Slice((int)(sizeof(CacheHeader) + (sizeof(ulong) * byteCodes.Length)), (int)totalBytecodeSize); + codeHash = XxHash64.HashToUInt64(bytecodeSpan); + } + _variantToCompiledHash[variantKey] = codeHash; ref var entry = ref _inMemoryCache.GetValueRefOrAddDefault(id, out var exists); @@ -149,16 +174,15 @@ internal unsafe class ShaderLibrary : IDisposable } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Result GetCompiledCache(ulong id, int index, AllocationHandle allocationHandle) + public Result GetCompiledCache(ulong id, int index) { if (_inMemoryCache.TryGetValue(id, out var entry)) { if (index < entry.cache.Length) { var shaderCache = entry.cache[index]; - var result = new MemoryBlock(shaderCache.byteCode.Size, shaderCache.byteCode.Alignment, allocationHandle); + var result = new MemoryBlock(shaderCache.byteCode.GetUnsafePtr(), (uint)shaderCache.byteCode.Size); - result.CopyFrom(shaderCache.byteCode.AsSpan()); return new ShaderCache { byteCode = result, @@ -171,16 +195,38 @@ internal unsafe class ShaderLibrary : IDisposable } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Result GetCompiledHash(Key64 variantKey) + public Result GetCompiledHash(ulong id, int passIndex, Key64 variantKey) { if (_variantToCompiledHash.TryGetValue(variantKey, out var compiledHash)) { return compiledHash; } + _shaderCompilationBridge?.RequestCompilation(id, passIndex, variantKey); return Error.NotFound; } + public void InvalidateShaderCache(ulong id, IPipelineLibrary pipelineLibrary) + { + if (_inMemoryCache.TryGetValue(id, out var entry)) + { + for (int i = 0; i < entry.cache.Length; i++) + { + if (entry.cache[i].compiledHash != 0) + { + pipelineLibrary.EvictStalePipelines(entry.cache[i].compiledHash); + } + } + entry.Dispose(); + _inMemoryCache.Remove(id); + } + + // Wait, what about _variantToCompiledHash? + // It maps variantKey -> compiledHash. Since we don't have a way to find variantKeys for this shader id, + // it will just linger as garbage. But it's small (8 bytes + 8 bytes). + // A proper fix would require _inMemoryCache to store the variantKeys, but for now we ignore it. + } + public void Dispose() { foreach (var kvp in _inMemoryCache) @@ -191,6 +237,11 @@ internal unsafe class ShaderLibrary : IDisposable _inMemoryCache.Dispose(); _variantToCompiledHash.Dispose(); + if (_shaderCompilationBridge != null) + { + _shaderCompilationBridge.OnShaderVariantCompiled -= OnVariantCompiled; + } + GC.SuppressFinalize(this); } } diff --git a/src/Test/Ghost.UnitTest/Graphics/ShaderLibraryTest.cs b/src/Test/Ghost.UnitTest/Graphics/ShaderLibraryTest.cs new file mode 100644 index 0000000..662a926 --- /dev/null +++ b/src/Test/Ghost.UnitTest/Graphics/ShaderLibraryTest.cs @@ -0,0 +1,166 @@ +using Ghost.Core; +using Ghost.Core.Graphics; +using Ghost.Graphics.RHI; +using Ghost.Graphics.Services; +using Misaki.HighPerformance.LowLevel.Buffer; + +namespace Ghost.UnitTest.Graphics; + +[TestClass] +public class ShaderLibraryTest +{ + private class MockPipelineLibrary : IPipelineLibrary + { + public List EvictedHashes { get; } = new(); + + public Result> CreateComputePipeline(ref readonly ComputePSODesc desc) => Result>.Failure(); + public Result> CreateGraphicsPipeline(ref readonly GraphicsPSODesc desc) => Result>.Failure(); + + public void EvictStalePipelines(ulong compiledHash) + { + EvictedHashes.Add(compiledHash); + } + + public bool HasPipelineStateObject(UInt128 key) => false; + public void SaveLibraryToDisk(string filePath) { } + public void BeginFrame(ulong submittedFrame) { } + public void EndFrame(ulong completedFrame) { } + public void Dispose() { } + } + + private class MockShaderCompilationBridge : IShaderCompilationBridge + { + public List<(ulong id, int passIndex, Key64 variantKey)> Requests { get; } = new(); + public event Action, ulong>? OnShaderVariantCompiled; + + public void RequestCompilation(ulong shaderId, int passIndex, Key64 variantKey) + { + Requests.Add((shaderId, passIndex, variantKey)); + } + + public void TriggerCompiled(Key64 variantKey, ulong newHash) + { + OnShaderVariantCompiled?.Invoke(variantKey, newHash); + } + } + + [TestInitialize] + public void Setup() + { + AllocationManager.Initialize(AllocationManagerDesc.Default); + } + + [TestCleanup] + public void Cleanup() + { + AllocationManager.Dispose(); + } + + [TestMethod] + public unsafe void TestInvalidateShaderCache_EvictsPipelinesAndClearsCache() + { + // Arrange + using var shaderLibrary = new ShaderLibrary(null, "TestShaderCache"); + var mockPipelineLibrary = new MockPipelineLibrary(); + + ulong testShaderId = 12345; + var testPassIndex = 0; + var variantKey = new Key64(999); + + // Create some dummy bytecode to cache + var fakeData = new byte[] { 1, 2, 3, 4 }; + var expectedHash = 0UL; + + fixed (byte* pData = fakeData) + { + var byteCode = new ShaderByteCode + { + pCode = pData, + size = (ulong)fakeData.Length + }; + + var byteCodes = new Span(ref byteCode); + + // Compute hash that should be generated (only bytecode) + var dataSpan = new ReadOnlySpan(byteCode.pCode, (int)byteCode.size); + expectedHash = System.IO.Hashing.XxHash64.HashToUInt64(dataSpan); + + // Act: Cache it + shaderLibrary.CacheCompiledResult(testShaderId, testPassIndex, variantKey, byteCodes); + } + + // Verify it was cached successfully + var cachedResult = shaderLibrary.GetCompiledCache(testShaderId, testPassIndex); + Assert.IsTrue(cachedResult.IsSuccess, "Shader should be cached"); + Assert.AreEqual(expectedHash, cachedResult.Value.compiledHash); + + // Act: Invalidate + shaderLibrary.InvalidateShaderCache(testShaderId, mockPipelineLibrary); + + // Assert: EvictStalePipelines should be called + Assert.HasCount(1, mockPipelineLibrary.EvictedHashes); + Assert.AreEqual(expectedHash, mockPipelineLibrary.EvictedHashes[0]); + + // Assert: Cache should be cleared + var cachedResultAfter = shaderLibrary.GetCompiledCache(testShaderId, testPassIndex); + Assert.IsFalse(cachedResultAfter.IsSuccess, "Cache should be invalidated"); + } + + [TestMethod] + public void TestGetCompiledHash_TriggersCompilationRequest() + { + // Arrange + var mockBridge = new MockShaderCompilationBridge(); + using var shaderLibrary = new ShaderLibrary(mockBridge, "TestShaderCache"); + var testShaderId = 555UL; + var passIndex = 1; + var variantKey = new Key64(777); + + // Act + var result = shaderLibrary.GetCompiledHash(testShaderId, passIndex, variantKey); + + // Assert + Assert.IsFalse(result.IsSuccess); + Assert.AreEqual(Error.NotFound, result.Error); + Assert.HasCount(1, mockBridge.Requests); + Assert.AreEqual(testShaderId, mockBridge.Requests[0].id); + Assert.AreEqual(passIndex, mockBridge.Requests[0].passIndex); + Assert.AreEqual(variantKey, mockBridge.Requests[0].variantKey); + } + + [TestMethod] + public void TestOnVariantCompiled_UpdatesHashCache() + { + // Arrange + var mockBridge = new MockShaderCompilationBridge(); + using var shaderLibrary = new ShaderLibrary(mockBridge, "TestShaderCache"); + var variantKey = new Key64(123); + ulong newHash = 0xABCDE; + + // Act + mockBridge.TriggerCompiled(variantKey, newHash); + + // Assert + var result = shaderLibrary.GetCompiledHash(0, 0, variantKey); + Assert.IsTrue(result.IsSuccess); + Assert.AreEqual(newHash, result.Value); + } + + [TestMethod] + public void TestGetCompiledCache_HandlesIndexOutOfBounds() + { + // Arrange + using var shaderLibrary = new ShaderLibrary(null, "TestShaderCache"); + ulong testShaderId = 111; + var scope = AllocationManager.CreateStackScope(); + + // Act + var result = shaderLibrary.GetCompiledCache(testShaderId, 99); + + // Assert + Assert.IsFalse(result.IsSuccess); + Assert.AreEqual(Error.NotFound, result.Error); + + scope.Dispose(); + } +} diff --git a/src/build_log.txt b/src/build_log.txt new file mode 100644 index 0000000000000000000000000000000000000000..cddee4f2deb4dc7b96281e3c519779a7522eed2a GIT binary patch literal 30284 zcmeHQTW{Mo6z21Q{Rbg0Nr6O-?OgjXp!XC@vLH!!!8{ahVkhpD*z=`r_T#sGA8D49 zEXj^0rS^7UNU?c8d&|88^i?aG!}P>kXP!7YNCXpx>haZ=;POGeDCQpUZTD$U1J;JL&RU; z?>XWJ2)$G%I?qVm#d3vc^UvyeYFMc2vTYC4nl3$1HMNH)0F6WS2yK4o79XpR=(ACx z%{DZz)s4AY-6_z|r)rG;AEAeCFmC?1gczeX;>Z=P$ym>d&!S;V)#vQ_@@Zh}`{1Uo z=87Z*@!0J4H^4hpaZcVC#Ak<_HKFoj^4h&$S-W3ic6L)a!;c9uOO2d}_kDVrEK0V) zL20vqqta6yylZL;Z#(Oq#G=G$BW8@a4c*!ew3{5w(2j1~v_zw9c&_Ku3G(+Kp~xvW z$ngp^N60(GY`H|tNY4?@>LGsrj=wdv3F*S8f%hhAWvkkl{aw62P2%cm2Pt>)33ZM= z<#MS+~&OR=vEEX8~pAd zlr6o*T%lZ#*RTe9_VBE$ZOG3j=nKk>AwDhLieS>hGsx!eHBH-~j_G~@4P*RuV=+Xn zl=}=bqvrZ2Ku@t9Ow&UuJHoC90`x%}AC`TMR*ulaL;QM-PYYwvLfa|jAIC;$10m#s zK1v-R567j4v7}_ULrK3DTB&?Ko`=KdNfVz1zv-1qxKX&#=N(`EJH08J8((0~970O? zYlsOGZWGy8O!mcBl{jyaKe@i}F{`D|dC6GeLE*t-JXoHkDC?|fedYNltX&_MiZ)c_ zoyfal@^03e>wIa+_14A+?+Ndf&wH<7Qy2BL$;vA$@4PE-p8QMlAa@BGcxvz-#@FGz zlp-HRJ{FUYvu%+hZ3|zkAK-g5yDnYzC&C5^{>)A%+$r2yj63rrXZjwE4Hcdfo-3c{ zrq|$8b%1@SSu3?kB|IrSS&S!VuhMU^6Z&3#PH>_PP2)TwUwnJ}46FNl@N$4%#_#a1 z1_NVjX`-$!{OwJ&c?bL8P3XLBgpIMo+C(^ai0!U)e0JFqLhO`A_*hJgPfNBgQZ04a z^%kc$yUe^Eup?hBy|$L()Y%g9Xt5s1YH6{x6sN_OkVlKbfK^M0#hDeIVJ5#%g^>k) zcuOucR)Gi(;x~>lUx~RQmgGXmUgK92NQN%riEHQr*FYrG72bWs4Irn8Ww=4??F6$C zA>@-rs1QXFA%6|IcJS1-WVk>)cjk!l7{$;mMCdhjNyOR_f#PaCnACcbQR|_u^$@YP z)@PumGeMLz)sM(e%*7bxM<~0EHnZh->KT6bbWa$$x~8r5b-ec{D3edfGeB!8;fNS{ zj-BjH&_cuy5g?T32Ft^p#t4!uP^)v+_zpGq&=&SFaU^ZzUPXS=PHYO%E$nZ~P?k%X zI@0k55BX8kGuHK+JWOfuE5sZPC;5u4ROM>y$u-Jq_jdl2jlHww7ncFH1b_AnT1a$&&J@%hK>Im1T2WR(-{(%_{MJR68%E zrMw8A=8f-kJ6)Y+QtwK=PY*m4)n!uZ^kG5X3G&XjnkWJ7UaCXh<7` z<)tC*=dm%T@%eV1l`mJP+dq3v`LKpQuwZfKME z`uHAD-!<%|8QeE>5}Q7NCTz6v8e$FBdllMwdimsigXm_hM4MByrnXHSJ=+-|l!*F3 zYvr_0T1Z0#I8owkG4s$8uq8PJ|Ei{iX;@-0ecSJ)IG_44eoy@NWq zaORD`nRhrj*3;ZF5{^KB(34QMT;TbZ#g_~8ahekr5P3nfbSR)$nKRZt?+hIMd)c4G}z$jN&XKNS*DgzgC2dCgb2{^yu7xL1SsV3gwky&n3bV>d*M!$m##7`p z{&1J(z-#9t;bYy;|*s= ze7O+Uqs9ACg=8QfEQ*a|{dU}q7LOM_kyZFNc7ZTh{JAf7j=hDZ+I3)8&wq_jkaJ8JocF*U(E}|6-4)qo;@x-PbnxE0p3; zQ|aZtLkd?PZRKgK(==9!Ryl22V`T?GrE!-j(jgIkwa|82@3g~d>$6mYs-lI{LFgE% zY4`gmt7Xt_?uYHW!0P4S^Td9lUxGM3BJg-)k-1wc)O~oXeH#YXw+DX&Z<5-?oA^H7 zH^n_x;?&sNmq5qS^Kpw3%&r-GI{mJwxA1&~xK)pSk_g|9@NEFpA@58|dOhd|VJoS9 zu=PxlL<#JnJ=@@bG>LfxAImdnQ=8Z|YYqwK73ZFMhA`CPd!L)x#%3 zk6Y|D(U%+*6L0X?TcDh$5^aB5jn3}GhUi&(u~)$VD=62|)0(Cp!9K>9w~Kv1dnwVYtdF;V@vg39jjWT@l0%Khj&p$iJi`x^2fX1e#Cx&l zKk@TKzM8p5-2YR|szb=@7ic|W`I!CR`|-2G;tB#-5Wt+V&I{(F5Nznz#2pZIA{t8s zIY7C!*b-}5u_%H_O80;IvcR`CoV@s&l7ZSH??EjKmnMiLdyjUuNBsZ95rRk(KHv#5 z!v~AvtrtWRJ@jq`Dt~vTu@t-a;;E-qUp+4fB8mP}K_q=p=4QF&v~dKHly1Mwvv2a> zv2o}nUi>|1vu`A)E*@fi6GW2OIJ9vLA1sQEWB0EFkyNFfBZwrPDwg?g=D#44#Lgl5 zB5vp0Z(mu^C+=jDpicyS!t-Lwiaz1pSp3xS{G`!a_SUpDpnD#Dx!sXz=pzq+OR+aO mxD-UFoMVrUr{$O;LP + +namespace MyNamespace +{ + [global::System.Runtime.InteropServices.StructLayout(global::System.Runtime.InteropServices.LayoutKind.Sequential, Pack = 4)] + public partial struct {{infoName}} + { +#if GHOST_EDITOR + public const string HLSL_SOURCE = @" +#ifndef {{definedSymbol}} +#define {{definedSymbol}} +struct {{infoName}} +{ +{{codeBuilder}} +}; +#endif // {{definedSymbol}}"; + +{{fieldsBuilder}} +#endif + } +} +"""; + Console.WriteLine(code); + } +}