Misaki/GhostEngine
2
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add editor shader compilation bridge & pipeline cache mgmt

Browse Source 
Introduced EditorShaderCompilerBridge and IShaderCompilationBridge for async shader variant compilation and cache invalidation in the editor. Refactored ShaderLibrary to support the bridge, updating hash/caching logic and triggering compilation on cache misses. Changed pipeline library to use ulong content hashes and added stale pipeline eviction. Updated EngineCore and render code to integrate the new system. Added unit tests for ShaderLibrary cache and bridge behavior. Minor improvements to shader property code generation and test generator.
This commit is contained in:
Misaki
2026-05-08 17:06:37 +09:00
parent ba8694ed0c
commit d0076c852f
15 changed files with 583 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/Runtime/Ghost.Engine/EngineCore.cs
View File

@@ -1,6 +1,7 @@
using Ghost.Core.Graphics;
using Ghost.Engine.RenderPipeline;
using Ghost.Graphics;
using Ghost.Graphics.RHI;
using Misaki.HighPerformance.Jobs;
using Misaki.HighPerformance.Mathematics;
@@ -25,7 +26,7 @@ public sealed partial class EngineCore : IDisposable
internal RenderSystem RenderSystem => _renderSystem;
internal AssetManager AssetManager => _assetManager;
public EngineCore(IContentProvider contentProvider)
public EngineCore(IContentProvider contentProvider, IShaderCompilationBridge? shaderCompilationBridge = null)
{
_contentProvider = contentProvider;
@@ -46,6 +47,7 @@ public sealed partial class EngineCore : IDisposable
InitialRenderPipelineSettings = new GhostRenderPipelineSettings(),
ResourceStreamingProcessor = _streamingProcessor,
ShaderCacheDirectory = "ShaderCache",
ShaderCompilationBridge = shaderCompilationBridge,
};
_renderSystem = new RenderSystem(renderingDesc);

2
src/Runtime/Ghost.Generator/ShaderPropertiesGenerator.cs
View File

@@ -90,7 +90,7 @@ namespace Ghost.Generator
isEnabledByDefault: true), info.TypeSymbol.Locations.FirstOrDefault()));
continue;
}
var definedSymbol = $"__{info.Name.ToUpper()}_G_HLSL";
var fieldsBuilder = new StringBuilder();

2
src/Runtime/Ghost.Graphics.D3D12/D3D12GraphicsEngine.cs
View File

@@ -144,6 +144,7 @@ internal class D3D12GraphicsEngine : IGraphicsEngine
_cpuFrame = cpuFrame;
_resourceDatabase.BeginFrame(cpuFrame);
_pipelineLibrary.BeginFrame(cpuFrame);
}
public void EndFrame(ulong gpuFrame)
@@ -151,6 +152,7 @@ internal class D3D12GraphicsEngine : IGraphicsEngine
Logger.DebugAssert(!_disposed);
_resourceDatabase.EndFrame(gpuFrame);
_pipelineLibrary.EndFrame(gpuFrame);
while (_commandBufferReturnQueue.TryPeek(out var entry) && entry.returnFrame < gpuFrame)
{

74
src/Runtime/Ghost.Graphics.D3D12/D3D12PipelineLibrary.cs
View File

@@ -18,6 +18,7 @@ internal struct D3D12PipelineState : IDisposable
{
public UniquePtr<ID3D12PipelineState> pso;
public Key64<ShaderVariant> shaderVariant;
public ulong contentHash;
public void Dispose()
{
@@ -33,6 +34,17 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object<ID3D12PipelineLibrary1>
private UnsafeHashMap<UInt128, D3D12PipelineState> _pipelineCache;
private struct StalePipeline
{
public UInt128 pipelineKey;
public D3D12PipelineState pso;
public ulong frameAdded;
}
private UnsafeList<StalePipeline> _stalePipelines;
private ulong _currentCpuFrame;
private ulong _completedGpuFrame;
public ID3D12RootSignature* DefaultRootSignature => _defaultRootSignature.Get();
private static ID3D12PipelineLibrary1* CreateLibrary(D3D12RenderDevice device, string? filePath)
@@ -61,6 +73,7 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object<ID3D12PipelineLibrary1>
_device = device;
_pipelineCache = new UnsafeHashMap<UInt128, D3D12PipelineState>(32, AllocationHandle.Persistent);
_stalePipelines = new UnsafeList<StalePipeline>(16, AllocationHandle.Persistent);
CreateDefaultRootSignature().ThrowIfFailed();
}
@@ -145,7 +158,7 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object<ID3D12PipelineLibrary1>
return D3D12Utility.D3D12_DEPTH_STENCIL_DESC_CREATE(depthEnabled, writeEnabled, cmp);
}
private Result CreatePSO(Key64<ShaderVariant> shaderVariantKey, UInt128 pipelineKey, D3D12_PIPELINE_STATE_STREAM_DESC* pStreamDesc)
private Result CreatePSO(ulong contentHash, Key64<ShaderVariant> shaderVariantKey, UInt128 pipelineKey, D3D12_PIPELINE_STATE_STREAM_DESC* pStreamDesc)
{
ID3D12PipelineState* pPipelineState = default;
@@ -171,6 +184,7 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object<ID3D12PipelineLibrary1>
D3D12PipelineState pso = default;
pso.shaderVariant = shaderVariantKey;
pso.contentHash = contentHash;
pso.pso.Attach(pPipelineState);
_pipelineCache[pipelineKey] = pso;
@@ -187,7 +201,7 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object<ID3D12PipelineLibrary1>
}
var passAttachmentKey = new PassAttachmentHash(desc.RtvFormats, desc.DsvFormat);
var pipelineKey = RHIUtility.CreateGraphicsPipelineKey(desc.VariantKey, desc.PipelineOption, passAttachmentKey);
var pipelineKey = RHIUtility.CreateGraphicsPipelineKey(desc.CompiledHash, desc.PipelineOption, passAttachmentKey);
if (!_pipelineCache.ContainsKey(pipelineKey))
{
@@ -243,7 +257,7 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object<ID3D12PipelineLibrary1>
SizeInBytes = (nuint)sizeof(CD3DX12_PIPELINE_MESH_STATE_STREAM)
};
var result = CreatePSO(desc.VariantKey, pipelineKey, &streamDesc);
var result = CreatePSO(desc.CompiledHash, desc.VariantKey, pipelineKey, &streamDesc);
if (result.IsFailure)
{
return result;
@@ -258,7 +272,7 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object<ID3D12PipelineLibrary1>
{
AssertNotDisposed();
var pipelineKey = RHIUtility.CreateComputePipelineKey(desc.VariantKey);
var pipelineKey = RHIUtility.CreateComputePipelineKey(desc.CompiledHash);
if (!_pipelineCache.ContainsKey(pipelineKey))
{
fixed (byte* pCSByteCode = desc.CsCode)
@@ -272,7 +286,7 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object<ID3D12PipelineLibrary1>
SizeInBytes = (nuint)sizeof(CD3DX12_PIPELINE_STATE_STREAM_CS)
};
var result = CreatePSO(desc.VariantKey, pipelineKey, &streamDesc);
var result = CreatePSO(desc.CompiledHash, desc.VariantKey, pipelineKey, &streamDesc);
if (result.IsFailure)
{
return result;
@@ -300,6 +314,55 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object<ID3D12PipelineLibrary1>
return Error.NotFound;
}
public void BeginFrame(ulong cpuFrame)
{
_currentCpuFrame = cpuFrame;
}
public void EndFrame(ulong gpuFrame)
{
_completedGpuFrame = gpuFrame;
// Process stale pipelines and dispose them if they are no longer in flight
for (int i = _stalePipelines.Count - 1; i >= 0; i--)
{
var stale = _stalePipelines[i];
if (_completedGpuFrame >= stale.frameAdded)
{
stale.pso.Dispose();
_stalePipelines.RemoveAtSwapBack(i);
}
}
}
public void EvictStalePipelines(ulong oldContentHash)
{
// Find all pipelines with matching oldContentHash
using var keysToRemove = new UnsafeList<UInt128>(8, AllocationHandle.Temp);
foreach (var kvp in _pipelineCache)
{
if (kvp.Value.contentHash == oldContentHash)
{
keysToRemove.Add(kvp.Key);
}
}
foreach (var key in keysToRemove)
{
if (_pipelineCache.TryGetValue(key, out var pso))
{
_stalePipelines.Add(new StalePipeline
{
pipelineKey = key,
pso = pso,
frameAdded = _currentCpuFrame
});
_pipelineCache.Remove(key);
}
}
}
protected override void Dispose(bool disposing)
{
foreach (var kvp in _pipelineCache)
@@ -308,6 +371,7 @@ internal unsafe class D3D12PipelineLibrary : D3D12Object<ID3D12PipelineLibrary1>
}
_pipelineCache.Dispose();
_stalePipelines.Dispose();
_defaultRootSignature.Dispose();
}
}

4
src/Runtime/Ghost.Graphics.RHI/Common.cs
View File

@@ -219,7 +219,7 @@ public readonly struct PassAttachmentHash : IEquatable<PassAttachmentHash>
public ref struct GraphicsPSODesc
{
public UInt128 CompiledHash
public ulong CompiledHash
{
get; set;
}
@@ -262,7 +262,7 @@ public ref struct GraphicsPSODesc
public ref struct ComputePSODesc
{
public UInt128 CompiledHash
public ulong CompiledHash
{
get; set;
}

4
src/Runtime/Ghost.Graphics.RHI/IPipelineLibrary.cs
View File

@@ -9,4 +9,8 @@ public interface IPipelineLibrary : IDisposable
bool HasPipelineStateObject(UInt128 key);
Result<Key128<PipelineState>> CreateGraphicsPipeline(ref readonly GraphicsPSODesc desc);
Result<Key128<PipelineState>> CreateComputePipeline(ref readonly ComputePSODesc desc);
void BeginFrame(ulong cpuFrame);
void EndFrame(ulong gpuFrame);
void EvictStalePipelines(ulong oldContentHash);
}

17
src/Runtime/Ghost.Graphics.RHI/IShaderCompilationBridge.cs Normal file
View File

@@ -0,0 +1,17 @@
using Ghost.Core;
namespace Ghost.Graphics.RHI;
public interface IShaderCompilationBridge
{
/// <summary>
/// Request the bridge to recompile a shader variant or handle cache misses.
/// This is typically called by the ShaderLibrary when a variant hash is not found.
/// </summary>
void RequestCompilation(ulong shaderId, int passIndex, Key64<ShaderVariant> variantKey);
/// <summary>
/// Event triggered when a shader variant has been successfully compiled and updated.
/// </summary>
event Action<Key64<ShaderVariant>, ulong> OnShaderVariantCompiled;
}

9
src/Runtime/Ghost.Graphics/Core/RenderContext.cs
View File

@@ -379,7 +379,7 @@ public readonly unsafe ref struct RenderContext
var variantKey = RHIUtility.CreateShaderVariantKey(entryHash, in keywordSet);
// TODO: Refactor this into a helper method.
var (compiledHash, error) = ShaderLibrary.GetCompiledHash(variantKey);
var (compiledHash, error) = ShaderLibrary.GetCompiledHash(shader.UniqueID, entryIndex, variantKey);
if (error.IsFailure)
{
// TODO: Fallback to an error material.
@@ -391,12 +391,11 @@ public readonly unsafe ref struct RenderContext
if (!PipelineLibrary.HasPipelineStateObject(pipelineKey))
{
using var scope = AllocationManager.CreateStackScope();
var compiledCacheResult = ShaderLibrary.GetCompiledCache(shader.UniqueID, entryIndex, scope.AllocationHandle);
var compiledCacheResult = ShaderLibrary.GetCompiledCache(shader.UniqueID, entryIndex);
if (compiledCacheResult.IsFailure)
{
// TODO: Fallback to a checkerboard shader.
throw new InvalidOperationException("Failed to load compiled shader cache for pipeline state object creation.");
Logger.Warning($"Failed to load compiled shader cache for compute pipeline {pipelineKey}. Skipping compute dispatch.");
return;
}
var cache = compiledCacheResult.Value;

19
src/Runtime/Ghost.Graphics/IShaderCompilationBridge.cs
View File

@@ -1,19 +0,0 @@
namespace Ghost.Graphics;
public interface IShaderCompilationBridge
{
bool TryGetBytecode(ulong manifestKey, out ReadOnlyMemory<byte> bytecode);
bool IsCompiling(ulong manifestKey);
}
// NOTE: For testing only.
internal sealed class NullShaderCompilationBridge : IShaderCompilationBridge
{
public bool TryGetBytecode(ulong manifestKey, out ReadOnlyMemory<byte> bytecode)
{
bytecode = default;
return false; // Always fall through to ShaderLibrary's disk cache
}
public bool IsCompiling(ulong manifestKey) => false;
}

16
src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraphContext.cs
View File

@@ -172,7 +172,7 @@ internal sealed class RenderGraphContext : IUnsafeRenderContext
var variantMask = material._keywordMask & pass.KeywordIDs;
var variantKey = RHIUtility.CreateShaderVariantKey(pass.Key, in variantMask);
var (compiledHash, error) = _shaderLibrary.GetCompiledHash(variantKey);
var (compiledHash, error) = _shaderLibrary.GetCompiledHash(shader.UniqueID, material.ActivePassIndex, variantKey);
if (error.IsFailure)
{
// TODO: Fallback to a default shader or show an error material.
@@ -183,11 +183,11 @@ internal sealed class RenderGraphContext : IUnsafeRenderContext
if (!_pipelineLibrary.HasPipelineStateObject(pipelineKey))
{
using var scope = AllocationManager.CreateStackScope();
var compiledCacheResult = _shaderLibrary.GetCompiledCache(shader.UniqueID, material.ActivePassIndex, scope.AllocationHandle);
var compiledCacheResult = _shaderLibrary.GetCompiledCache(shader.UniqueID, material.ActivePassIndex);
if (compiledCacheResult.IsFailure)
{
throw new InvalidOperationException("Failed to load compiled shader cache for pipeline state object creation.");
Logger.Warning($"Failed to load compiled shader cache for graphics pipeline {pipelineKey}. Skipping draw call.");
return;
}
var cache = compiledCacheResult.Value;
@@ -277,7 +277,7 @@ internal sealed class RenderGraphContext : IUnsafeRenderContext
var keywordSet = new LocalKeywordSet(); // TODO: Support keywords in compute shader.
var variantKey = RHIUtility.CreateShaderVariantKey(entryHash, in keywordSet);
var (compiledHash, error) = _shaderLibrary.GetCompiledHash(variantKey);
var (compiledHash, error) = _shaderLibrary.GetCompiledHash(shader.UniqueID, entryIndex, variantKey);
if (error.IsFailure)
{
// TODO: Fallback to a default shader or show an error material.
@@ -288,11 +288,11 @@ internal sealed class RenderGraphContext : IUnsafeRenderContext
if (!_pipelineLibrary.HasPipelineStateObject(pipelineKey))
{
using var scope = AllocationManager.CreateStackScope();
var compiledCacheResult = _shaderLibrary.GetCompiledCache(shader.UniqueID, entryIndex, scope.AllocationHandle);
var compiledCacheResult = _shaderLibrary.GetCompiledCache(shader.UniqueID, entryIndex);
if (compiledCacheResult.IsFailure)
{
throw new InvalidOperationException("Failed to load compiled shader cache for pipeline state object creation.");
Logger.Warning($"Failed to load compiled shader cache for compute pipeline {pipelineKey}. Skipping compute dispatch.");
return;
}
var cache = compiledCacheResult.Value;

67
src/Runtime/Ghost.Graphics/Services/ShaderLibrary.cs
View File

@@ -3,6 +3,7 @@ using Ghost.Core.Utilities;
using Ghost.Graphics.RHI;
using Misaki.HighPerformance.LowLevel.Buffer;
using Misaki.HighPerformance.LowLevel.Collections;
using Misaki.HighPerformance.LowLevel.Utilities;
using System.IO.Hashing;
using System.Runtime.CompilerServices;
@@ -80,6 +81,16 @@ internal unsafe class ShaderLibrary : IDisposable
_cacheDirectory = cacheDirectory;
_shaderCompilationBridge = shaderCompilationBridge;
if (_shaderCompilationBridge != null)
{
_shaderCompilationBridge.OnShaderVariantCompiled += OnVariantCompiled;
}
}
private void OnVariantCompiled(Key64<ShaderVariant> variantKey, ulong newCompiledHash)
{
_variantToCompiledHash[variantKey] = newCompiledHash;
}
private string GetShaderCacheFilePath(ulong hash)
@@ -117,14 +128,17 @@ internal unsafe class ShaderLibrary : IDisposable
};
var offsets = stackalloc ulong[byteCodes.Length];
var offset = (ulong)(sizeof(CacheHeader) + (sizeof(ulong) * byteCodes.Length));
var offset = (nuint)(sizeof(CacheHeader) + (sizeof(ulong) * byteCodes.Length));
for (var i = 0; i < byteCodes.Length; i++)
{
offsets[i] = offset;
offset += byteCodes[i].size;
offset += (nuint)byteCodes[i].size;
}
var data = new MemoryBlock((nuint)offset, 8, AllocationHandle.Persistent);
var alignment = Math.Max(Math.Max(MemoryUtility.AlignOf<CacheHeader>(), MemoryUtility.AlignOf<ulong>()), 8);
offset = MemoryUtility.AlignUp(offset, alignment);
var data = new MemoryBlock(offset, alignment, AllocationHandle.Persistent);
var writer = new SpanWriter(data.AsSpan<byte>());
writer.Write(header);
@@ -141,7 +155,18 @@ internal unsafe class ShaderLibrary : IDisposable
writer.WriteSpan(src);
}
var codeHash = XxHash64.HashToUInt64(data.AsSpan<byte>());
// Compute hash from bytecode only
var codeHash = 0UL;
if (byteCodes.Length > 0)
{
ulong totalBytecodeSize = 0;
for (int i = 0; i < byteCodes.Length; i++) totalBytecodeSize += byteCodes[i].size;
// We skip the header and offsets at the beginning of the MemoryBlock
var bytecodeSpan = data.AsSpan<byte>().Slice((int)(sizeof(CacheHeader) + (sizeof(ulong) * byteCodes.Length)), (int)totalBytecodeSize);
codeHash = XxHash64.HashToUInt64(bytecodeSpan);
}
_variantToCompiledHash[variantKey] = codeHash;
ref var entry = ref _inMemoryCache.GetValueRefOrAddDefault(id, out var exists);
@@ -149,16 +174,15 @@ internal unsafe class ShaderLibrary : IDisposable
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Result<ShaderCache, Error> GetCompiledCache(ulong id, int index, AllocationHandle allocationHandle)
public Result<ShaderCache, Error> GetCompiledCache(ulong id, int index)
{
if (_inMemoryCache.TryGetValue(id, out var entry))
{
if (index < entry.cache.Length)
{
var shaderCache = entry.cache[index];
var result = new MemoryBlock(shaderCache.byteCode.Size, shaderCache.byteCode.Alignment, allocationHandle);
var result = new MemoryBlock(shaderCache.byteCode.GetUnsafePtr(), (uint)shaderCache.byteCode.Size);
result.CopyFrom(shaderCache.byteCode.AsSpan<byte>());
return new ShaderCache
{
byteCode = result,
@@ -171,16 +195,38 @@ internal unsafe class ShaderLibrary : IDisposable
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Result<ulong, Error> GetCompiledHash(Key64<ShaderVariant> variantKey)
public Result<ulong, Error> GetCompiledHash(ulong id, int passIndex, Key64<ShaderVariant> variantKey)
{
if (_variantToCompiledHash.TryGetValue(variantKey, out var compiledHash))
{
return compiledHash;
}
_shaderCompilationBridge?.RequestCompilation(id, passIndex, variantKey);
return Error.NotFound;
}
public void InvalidateShaderCache(ulong id, IPipelineLibrary pipelineLibrary)
{
if (_inMemoryCache.TryGetValue(id, out var entry))
{
for (int i = 0; i < entry.cache.Length; i++)
{
if (entry.cache[i].compiledHash != 0)
{
pipelineLibrary.EvictStalePipelines(entry.cache[i].compiledHash);
}
}
entry.Dispose();
_inMemoryCache.Remove(id);
}
// Wait, what about _variantToCompiledHash?
// It maps variantKey -> compiledHash. Since we don't have a way to find variantKeys for this shader id,
// it will just linger as garbage. But it's small (8 bytes + 8 bytes).
// A proper fix would require _inMemoryCache to store the variantKeys, but for now we ignore it.
}
public void Dispose()
{
foreach (var kvp in _inMemoryCache)
@@ -191,6 +237,11 @@ internal unsafe class ShaderLibrary : IDisposable
_inMemoryCache.Dispose();
_variantToCompiledHash.Dispose();
if (_shaderCompilationBridge != null)
{
_shaderCompilationBridge.OnShaderVariantCompiled -= OnVariantCompiled;
}
GC.SuppressFinalize(this);
}
}