Misaki/GhostEngine
2
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(shader): rewrite editor shader compilation bridge with keyword resolution

Browse Source 
- Add AssetCatalog.EnumerateByTypes for filtered SQL queries
- Thread LocalKeywordSet through IShaderCompilationBridge API so the
  bridge can resolve keyword bitmask to string defines at compile time
- Eager/lazy popluation of shader-id-to-asset-id map eliminating
  full catalog scan per compilation miss
- Build keyword mapping from PassDescriptor groups to reconstruct
  localIndex -> keywordName in the bridge
- Use CompileShaderPass extension for multi-stage AS/MS/PS compilation
  with correct ShaderModel from descriptor
- Remove double-load of shader asset in compilation flow
- Update test mock to match new interface signature
This commit is contained in:
Misaki
2026-05-08 19:36:24 +09:00
parent d0076c852f
commit 1cc65e8218
10 changed files with 306 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/Editor/Ghost.Editor.Core/Services/AssetCatalog.cs
View File

@@ -89,6 +89,7 @@ public sealed partial class AssetCatalog
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_assets_path ON assets(source_path);
CREATE INDEX IF NOT EXISTS idx_assets_parent ON assets(parent_guid);
CREATE INDEX IF NOT EXISTS idx_assets_type_id ON assets(asset_type_id);
CREATE TABLE IF NOT EXISTS dependencies (
from_guid BLOB(16) NOT NULL REFERENCES assets(guid) ON DELETE CASCADE,
@@ -281,6 +282,33 @@ public sealed partial class AssetCatalog
}
}
public IEnumerable<Guid> EnumerateByTypes(params Guid[] assetTypeIds)
{
if (assetTypeIds.Length == 0)
{
yield break;
}
using var connection = OpenConnection();
using var cmd = connection.CreateCommand();
var parameterNames = new List<string>(assetTypeIds.Length);
for (int i = 0; i < assetTypeIds.Length; i++)
{
string paramName = $"@typeId{i}";
parameterNames.Add(paramName);
cmd.Parameters.AddWithValue(paramName, assetTypeIds[i].ToByteArray());
}
cmd.CommandText = $"SELECT guid FROM assets WHERE asset_type_id IN ({string.Join(", ", parameterNames)})";
using var reader = cmd.ExecuteReader();
while (reader.Read())
{
yield return new Guid((byte[])reader[0]);
}
}
public List<SubAssetInfo> GetSubAssets(Guid parentGuid)
{
using var connection = OpenConnection();

333
src/Editor/Ghost.Editor.Core/Services/EditorShaderCompilerBridge.cs
View File

@@ -3,24 +3,26 @@ using Ghost.Core.Graphics;
using Ghost.Editor.Core.Assets;
using Ghost.Editor.Core.Contracts;
using Ghost.Editor.Core.Utilities;
using Ghost.Engine;
using Ghost.Graphics.Core;
using Ghost.Graphics.RHI;
using Ghost.Graphics.Services;
using Ghost.Engine;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Misaki.HighPerformance.LowLevel.Buffer;
using System.Collections.Concurrent;
using Misaki.HighPerformance.LowLevel.Collections;
using System.Runtime.CompilerServices;
namespace Ghost.Editor.Core.Services;
[EditorInjection(EditorInjectionAttribute.ServiceLifetime.Singleton, typeof(IShaderCompilationBridge))]
internal sealed class EditorShaderCompilerBridge : IShaderCompilationBridge, IDisposable
internal sealed class EditorShaderCompilerBridge : IShaderCompilationBridge
{
private readonly IAssetRegistry _assetRegistry;
private readonly IShaderCompiler _compiler;
private readonly ConcurrentDictionary<ulong, Guid> _shaderIdToAssetId = new();
private readonly IServiceProvider _serviceProvider;
private readonly IShaderCompiler _compiler;
private readonly ConcurrentDictionary<ulong, Guid> _shaderIdToAssetId = new();
private readonly ConcurrentDictionary<Guid, Dictionary<int, string>[]> _assetKeywordMappings = new();
private Task? _shaderDictionaryPopulated;
public event Action<Key64<ShaderVariant>, ulong>? OnShaderVariantCompiled;
@@ -41,19 +43,11 @@ internal sealed class EditorShaderCompilerBridge : IShaderCompilationBridge, IDi
var result = _assetRegistry.LoadAssetAsync(guid).AsTask().Result;
if (result.IsSuccess)
{
ulong nameHash = 0;
if (result.Value is GraphicsShaderAsset graphicsAsset)
{
nameHash = RHIUtility.GetShaderID(graphicsAsset.Descriptor.Name);
}
else if (result.Value is ComputeShaderAsset computeAsset)
{
nameHash = RHIUtility.GetShaderID(computeAsset.Descriptor.Name);
}
var nameHash = ExtractNameHash(result.Value);
if (nameHash != 0)
{
_shaderIdToAssetId[nameHash] = guid;
BuildKeywordMappings(result.Value, guid);
var engineCore = _serviceProvider.GetService<EngineCore>();
if (engineCore != null)
@@ -67,121 +61,282 @@ internal sealed class EditorShaderCompilerBridge : IShaderCompilationBridge, IDi
}
}
public void RequestCompilation(ulong shaderId, int passIndex, Key64<ShaderVariant> variantKey)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static ulong ExtractNameHash(IAsset asset)
{
if (asset is GraphicsShaderAsset graphicsAsset)
{
return RHIUtility.GetShaderID(graphicsAsset.Descriptor.Name);
}
if (asset is ComputeShaderAsset computeAsset)
{
return RHIUtility.GetShaderID(computeAsset.Descriptor.Name);
}
return 0;
}
private Task EnsureShaderDictionaryPopulatedAsync()
{
var existing = Volatile.Read(ref _shaderDictionaryPopulated);
if (existing != null)
{
return existing;
}
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var original = Interlocked.CompareExchange(ref _shaderDictionaryPopulated, tcs.Task, null);
if (original != null)
{
return original;
}
Task.Run(async () =>
{
if (!_shaderIdToAssetId.TryGetValue(shaderId, out var guid))
try
{
var catalog = _assetRegistry.GetAssetCatalog();
foreach (var (assetGuid, path) in catalog.EnumerateAll())
{
if (path.EndsWith(".gshdr") || path.EndsWith(".gcomp"))
var assetGuids = catalog.EnumerateByTypes(typeof(GraphicsShaderAsset).GUID, typeof(ComputeShaderAsset).GUID);
foreach (var assetGuid in assetGuids)
{
var result = await _assetRegistry.LoadAssetAsync(assetGuid);
if (result.IsSuccess)
{
ulong nameHash = 0;
if (result.Value is GraphicsShaderAsset graphicsAsset)
{
nameHash = RHIUtility.GetShaderID(graphicsAsset.Descriptor.Name);
}
else if (result.Value is ComputeShaderAsset computeAsset)
{
nameHash = RHIUtility.GetShaderID(computeAsset.Descriptor.Name);
}
var nameHash = ExtractNameHash(result.Value);
if (nameHash != 0)
{
_shaderIdToAssetId[nameHash] = assetGuid;
}
}
BuildKeywordMappings(result.Value, assetGuid);
}
}
}
if (_shaderIdToAssetId.TryGetValue(shaderId, out var assetId))
tcs.SetResult();
}
catch (Exception ex)
{
tcs.SetException(ex);
}
});
return tcs.Task;
}
private void BuildKeywordMappings(IAsset asset, Guid assetId)
{
if (asset is GraphicsShaderAsset graphicsAsset)
{
var passes = graphicsAsset.Descriptor.Passes;
var mappings = new Dictionary<int, string>[passes.Length];
for (var i = 0; i < passes.Length; i++)
{
mappings[i] = BuildKeywordMappingFromGroups(passes[i].keywords);
}
_assetKeywordMappings[assetId] = mappings;
}
else if (asset is ComputeShaderAsset computeAsset)
{
var entryCount = computeAsset.Descriptor.ShaderCodes.Length;
var mappings = new Dictionary<int, string>[entryCount];
var sharedMapping = BuildKeywordMappingFromGroups(computeAsset.Descriptor.Keywords);
for (var i = 0; i < entryCount; i++)
{
mappings[i] = sharedMapping;
}
_assetKeywordMappings[assetId] = mappings;
}
}
private static Dictionary<int, string> BuildKeywordMappingFromGroups(KeywordsGroup[] groups)
{
var mapping = new Dictionary<int, string>();
var localIndex = 0;
foreach (var group in groups)
{
if (group.keywords == null)
{
continue;
}
if (group.space != KeywordSpace.Local)
{
continue;
}
foreach (var kw in group.keywords)
{
mapping[localIndex++] = kw;
}
}
return mapping;
}
private static string[] BuildVariantDefines(LocalKeywordSet keywordMask, Dictionary<int, string>? keywordMapping)
{
if (keywordMapping == null || keywordMapping.Count == 0)
{
return Array.Empty<string>();
}
var defines = new List<string>(keywordMapping.Count);
foreach (var (localIndex, keywordName) in keywordMapping)
{
if (keywordMask.IsKeywordEnabled(localIndex))
{
defines.Add(keywordName);
}
}
return defines.ToArray();
}
private static ReadOnlySpan<string> CombineDefines(ReadOnlySpan<string> staticDefines, ReadOnlySpan<string> variantDefines)
{
if (variantDefines.Length == 0)
{
return staticDefines;
}
if (staticDefines.Length == 0)
{
return variantDefines;
}
var combined = new string[staticDefines.Length + variantDefines.Length];
staticDefines.CopyTo(combined);
variantDefines.CopyTo(combined.AsSpan(staticDefines.Length));
return combined;
}
public void RequestCompilation(ulong shaderId, int passIndex, Key64<ShaderVariant> variantKey, LocalKeywordSet keywordMask)
{
Task.Run(async () =>
{
await EnsureShaderDictionaryPopulatedAsync();
if (!_shaderIdToAssetId.TryGetValue(shaderId, out var assetId))
{
return;
}
var assetResult = await _assetRegistry.LoadAssetAsync(assetId);
if (assetResult.IsSuccess)
if (assetResult.IsFailure)
{
return;
}
Dictionary<int, string>? keywordMapping = null;
if (_assetKeywordMappings.TryGetValue(assetId, out var mappings) && passIndex < mappings.Length)
{
keywordMapping = mappings[passIndex];
}
if (assetResult.Value is GraphicsShaderAsset graphicsAsset)
{
var pass = graphicsAsset.Descriptor.Passes[passIndex];
await CompileGraphicsPassAsync(shaderId, passIndex, variantKey, pass);
await CompileGraphicsPassAsync(shaderId, passIndex, variantKey, keywordMask, pass, graphicsAsset.Descriptor.ShaderModel, keywordMapping);
}
else if (assetResult.Value is ComputeShaderAsset computeAsset)
{
var code = computeAsset.Descriptor.ShaderCodes[passIndex];
await CompileComputePassAsync(shaderId, passIndex, variantKey, code);
}
}
await CompileComputePassAsync(shaderId, passIndex, variantKey, keywordMask, computeAsset.Descriptor, passIndex, keywordMapping);
}
});
}
private unsafe Task CompileGraphicsPassAsync(ulong shaderId, int passIndex, Key64<ShaderVariant> variantKey, PassDescriptor pass)
private unsafe Task CompileGraphicsPassAsync(ulong shaderId, int passIndex, Key64<ShaderVariant> variantKey, LocalKeywordSet keywordMask, PassDescriptor descriptor, ShaderModel shaderModel, Dictionary<int, string>? keywordMapping)
{
// For simplicity, just compile the pixel shader. A real implementation would compile
// all stages (Mesh/Amp/Vertex/Pixel) defined in the pass descriptor.
var config = new ShaderCompilationConfig
var variantDefines = BuildVariantDefines(keywordMask, keywordMapping);
var additionalConfig = new ShaderCompilationConfig
{
shaderCode = pass.pixelShaderCode.code,
entryPoint = pass.pixelShaderCode.entryPoint,
stage = ShaderStage.PixelShader,
defines = pass.defines,
model = ShaderModel.SM_6_6
defines = variantDefines,
model = shaderModel,
optimizeLevel = CompilerOptimizeLevel.O3,
options = CompilerOption.None
};
var compileResult = _compiler.Compile(in config, Misaki.HighPerformance.LowLevel.Buffer.AllocationHandle.Persistent);
if (compileResult.IsSuccess)
{
var engineCore = _serviceProvider.GetService<EngineCore>();
if (engineCore != null)
{
using var bytecodeArray = compileResult.Value;
var byteCode = new ShaderByteCode
{
pCode = (byte*)bytecodeArray.GetUnsafePtr(),
size = (ulong)bytecodeArray.Length
};
// Assume 1 stage for now. In reality, we'd pass an array of ShaderByteCode for all stages.
var byteCodes = new Span<ShaderByteCode>(ref byteCode);
engineCore.RenderSystem.ShaderLibrary.CacheCompiledResult(shaderId, passIndex, variantKey, byteCodes);
// Get the generated hash to fire the event
var dataSpan = new ReadOnlySpan<byte>(byteCode.pCode, (int)byteCode.size);
var hash = System.IO.Hashing.XxHash64.HashToUInt64(dataSpan);
OnShaderVariantCompiled?.Invoke(variantKey, hash);
}
}
else
var compileResult = _compiler.CompileShaderPass(ref descriptor, ref additionalConfig, AllocationHandle.Persistent);
if (compileResult.IsFailure)
{
Ghost.Core.Logger.Error($"Failed to compile graphics shader {shaderId}: {compileResult.Message}");
return Task.CompletedTask;
}
var engineCore = _serviceProvider.GetService<EngineCore>();
if (engineCore == null)
{
return Task.CompletedTask;
}
using var compiled = compileResult.Value;
var stageCount = 0;
if (compiled.asResult.IsCreated) stageCount++;
if (compiled.msResult.IsCreated) stageCount++;
if (compiled.psResult.IsCreated) stageCount++;
var byteCodes = stackalloc ShaderByteCode[stageCount];
var idx = 0;
if (compiled.asResult.IsCreated)
{
byteCodes[idx++] = new ShaderByteCode { pCode = (byte*)compiled.asResult.GetUnsafePtr(), size = (ulong)compiled.asResult.Length };
}
if (compiled.msResult.IsCreated)
{
byteCodes[idx++] = new ShaderByteCode { pCode = (byte*)compiled.msResult.GetUnsafePtr(), size = (ulong)compiled.msResult.Length };
}
if (compiled.psResult.IsCreated)
{
byteCodes[idx++] = new ShaderByteCode { pCode = (byte*)compiled.psResult.GetUnsafePtr(), size = (ulong)compiled.psResult.Length };
}
var shaderLibrary = engineCore.RenderSystem.ShaderLibrary;
shaderLibrary.CacheCompiledResult(shaderId, passIndex, variantKey, new ReadOnlySpan<ShaderByteCode>(byteCodes, stageCount));
var (compiledHash, _) = shaderLibrary.GetCompiledHash(shaderId, passIndex, variantKey);
OnShaderVariantCompiled?.Invoke(variantKey, compiledHash);
return Task.CompletedTask;
}
private unsafe Task CompileComputePassAsync(ulong shaderId, int passIndex, Key64<ShaderVariant> variantKey, ShaderCode code)
private unsafe Task CompileComputePassAsync(ulong shaderId, int passIndex, Key64<ShaderVariant> variantKey, LocalKeywordSet keywordMask, ComputeShaderDescriptor descriptor, int entryIndex, Dictionary<int, string>? keywordMapping)
{
var variantDefines = BuildVariantDefines(keywordMask, keywordMapping);
var fullDefines = CombineDefines(descriptor.Defines, variantDefines);
var code = descriptor.ShaderCodes[entryIndex];
var config = new ShaderCompilationConfig
{
shaderCode = code.code,
entryPoint = code.entryPoint,
stage = ShaderStage.ComputeShader,
defines = Array.Empty<string>(),
model = ShaderModel.SM_6_6
defines = fullDefines,
model = descriptor.ShaderModel,
optimizeLevel = CompilerOptimizeLevel.O3,
options = CompilerOption.None
};
var compileResult = _compiler.Compile(in config, Misaki.HighPerformance.LowLevel.Buffer.AllocationHandle.Persistent);
if (compileResult.IsSuccess)
var compileResult = _compiler.Compile(ref config, AllocationHandle.Persistent);
if (compileResult.IsFailure)
{
Ghost.Core.Logger.Error($"Failed to compile compute shader {shaderId}: {compileResult.Message}");
return Task.CompletedTask;
}
var engineCore = _serviceProvider.GetService<EngineCore>();
if (engineCore != null)
if (engineCore == null)
{
return Task.CompletedTask;
}
using var bytecodeArray = compileResult.Value;
var byteCode = new ShaderByteCode
@@ -190,19 +345,11 @@ internal sealed class EditorShaderCompilerBridge : IShaderCompilationBridge, IDi
size = (ulong)bytecodeArray.Length
};
var byteCodes = new Span<ShaderByteCode>(ref byteCode);
var shaderLibrary = engineCore.RenderSystem.ShaderLibrary;
shaderLibrary.CacheCompiledResult(shaderId, passIndex, variantKey, new ReadOnlySpan<ShaderByteCode>(ref byteCode));
engineCore.RenderSystem.ShaderLibrary.CacheCompiledResult(shaderId, passIndex, variantKey, byteCodes);
var dataSpan = new ReadOnlySpan<byte>(byteCode.pCode, (int)byteCode.size);
var hash = System.IO.Hashing.XxHash64.HashToUInt64(dataSpan);
OnShaderVariantCompiled?.Invoke(variantKey, hash);
}
}
else
{
Ghost.Core.Logger.Error($"Failed to compile compute shader {shaderId}: {compileResult.Message}");
}
var (compiledHash, _) = shaderLibrary.GetCompiledHash(shaderId, passIndex, variantKey);
OnShaderVariantCompiled?.Invoke(variantKey, compiledHash);
return Task.CompletedTask;
}

2
src/Editor/Ghost.Editor/App.xaml.cs
View File

@@ -7,6 +7,7 @@ using Ghost.Editor.ViewModels.Controls;
using Ghost.Editor.ViewModels.Windows;
using Ghost.Editor.Views.Windows;
using Ghost.Engine;
using Ghost.Graphics.RHI;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.UI.Dispatching;
@@ -66,6 +67,7 @@ public partial class App : Application
services.AddSingleton<IPreviewService, PreviewService>();
services.AddSingleton<IAssetRegistry, AssetRegistry>();
services.AddSingleton<IContentProvider, EditorContentProvider>();
services.AddSingleton<IShaderCompilationBridge, EditorShaderCompilerBridge>();
services.AddSingleton<EngineCore>();

2
src/Runtime/Ghost.Graphics.RHI/Common.cs
View File

@@ -179,7 +179,7 @@ public readonly struct ShaderPass
get; init;
}
public LocalKeywordSet KeywordIDs
public LocalKeywordSet DefinedKeywords
{
get; init;
}

4
src/Runtime/Ghost.Graphics.RHI/IShaderCompilationBridge.cs
View File

@@ -2,13 +2,13 @@ using Ghost.Core;
namespace Ghost.Graphics.RHI;
public interface IShaderCompilationBridge
public interface IShaderCompilationBridge : IDisposable
{
/// <summary>
/// Request the bridge to recompile a shader variant or handle cache misses.
/// This is typically called by the ShaderLibrary when a variant hash is not found.
/// </summary>
void RequestCompilation(ulong shaderId, int passIndex, Key64<ShaderVariant> variantKey);
void RequestCompilation(ulong shaderId, int passIndex, Key64<ShaderVariant> variantKey, LocalKeywordSet keywordMask);
/// <summary>
/// Event triggered when a shader variant has been successfully compiled and updated.

2
src/Runtime/Ghost.Graphics/Core/RenderContext.cs
View File

@@ -379,7 +379,7 @@ public readonly unsafe ref struct RenderContext
var variantKey = RHIUtility.CreateShaderVariantKey(entryHash, in keywordSet);
// TODO: Refactor this into a helper method.
var (compiledHash, error) = ShaderLibrary.GetCompiledHash(shader.UniqueID, entryIndex, variantKey);
var (compiledHash, error) = ShaderLibrary.GetCompiledHash(shader.UniqueID, entryIndex, variantKey, keywordSet);
if (error.IsFailure)
{
// TODO: Fallback to an error material.

2
src/Runtime/Ghost.Graphics/Core/Shader.cs
View File

@@ -119,7 +119,7 @@ public partial struct Shader : IResourceReleasable
{
Key = RHIUtility.GetPassID(_nameHash, i),
DefaultState = pass.localPipeline,
KeywordIDs = keywords,
DefinedKeywords = keywords,
};
_passIDToLocal[GetPassID(pass.name)] = (ushort)i;

6
src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraphContext.cs
View File

@@ -169,10 +169,10 @@ internal sealed class RenderGraphContext : IUnsafeRenderContext
var materialPipeline = material.GetPassPipelineOverride(material.ActivePassIndex);
// Mask out the keywords that are not used in this pass.
var variantMask = material._keywordMask & pass.KeywordIDs;
var variantMask = material._keywordMask & pass.DefinedKeywords;
var variantKey = RHIUtility.CreateShaderVariantKey(pass.Key, in variantMask);
var (compiledHash, error) = _shaderLibrary.GetCompiledHash(shader.UniqueID, material.ActivePassIndex, variantKey);
var (compiledHash, error) = _shaderLibrary.GetCompiledHash(shader.UniqueID, material.ActivePassIndex, variantKey, variantMask);
if (error.IsFailure)
{
// TODO: Fallback to a default shader or show an error material.
@@ -277,7 +277,7 @@ internal sealed class RenderGraphContext : IUnsafeRenderContext
var keywordSet = new LocalKeywordSet(); // TODO: Support keywords in compute shader.
var variantKey = RHIUtility.CreateShaderVariantKey(entryHash, in keywordSet);
var (compiledHash, error) = _shaderLibrary.GetCompiledHash(shader.UniqueID, entryIndex, variantKey);
var (compiledHash, error) = _shaderLibrary.GetCompiledHash(shader.UniqueID, entryIndex, variantKey, keywordSet);
if (error.IsFailure)
{
// TODO: Fallback to a default shader or show an error material.

4
src/Runtime/Ghost.Graphics/Services/ShaderLibrary.cs
View File

@@ -195,14 +195,14 @@ internal unsafe class ShaderLibrary : IDisposable
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Result<ulong, Error> GetCompiledHash(ulong id, int passIndex, Key64<ShaderVariant> variantKey)
public Result<ulong, Error> GetCompiledHash(ulong id, int passIndex, Key64<ShaderVariant> variantKey, LocalKeywordSet keywordMask = default)
{
if (_variantToCompiledHash.TryGetValue(variantKey, out var compiledHash))
{
return compiledHash;
}
_shaderCompilationBridge?.RequestCompilation(id, passIndex, variantKey);
_shaderCompilationBridge?.RequestCompilation(id, passIndex, variantKey, keywordMask);
return Error.NotFound;
}

6
src/Test/Ghost.UnitTest/Graphics/ShaderLibraryTest.cs
View File

@@ -30,12 +30,12 @@ public class ShaderLibraryTest
private class MockShaderCompilationBridge : IShaderCompilationBridge
{
public List<(ulong id, int passIndex, Key64<ShaderVariant> variantKey)> Requests { get; } = new();
public List<(ulong id, int passIndex, Key64<ShaderVariant> variantKey, LocalKeywordSet keywordMask)> Requests { get; } = new();
public event Action<Key64<ShaderVariant>, ulong>? OnShaderVariantCompiled;
public void RequestCompilation(ulong shaderId, int passIndex, Key64<ShaderVariant> variantKey)
public void RequestCompilation(ulong shaderId, int passIndex, Key64<ShaderVariant> variantKey, LocalKeywordSet keywordMask)
{
Requests.Add((shaderId, passIndex, variantKey));
Requests.Add((shaderId, passIndex, variantKey, keywordMask));
}
public void TriggerCompiled(Key64<ShaderVariant> variantKey, ulong newHash)