From c66fda5332c9096987a986af1982d0293ea788b5 Mon Sep 17 00:00:00 2001 From: Misaki Date: Sat, 11 Apr 2026 23:10:39 +0900 Subject: [PATCH] feat(shader): refactor and enhance shader pipeline Refactored the shader compilation pipeline to introduce modularity, improve performance, and enhance maintainability. Key changes include: - Added `ShaderCompilationConfig`, `CompilerOptimizeLevel`, and `ShaderStage` enums. - Replaced `SM` property with `ShaderModel` in shader models. - Introduced `ShaderLibrary` for in-memory and disk-based shader caching. - Refactored `DSLShaderCompiler` and `AntlrShaderCompiler` for better hashing and error handling. - Centralized shader compilation logic in `ShaderCompilerUtility`. - Removed legacy shader compilation logic from `IShaderCompiler`. - Updated `RenderGraph`, `ResourceManager`, and `Material` to integrate with the new caching system. - Improved memory management with `NativeMemoryManager`. BREAKING CHANGE: Removed legacy shader compilation methods and replaced them with a new caching and compilation system. --- .../ShaderCompiler/DSLShaderCompiler.cs | 19 +- .../ShaderParser/AntlrShaderCompiler.cs | 156 +++++++------- .../ShaderParser/ComputeShaderVisitor.cs | 2 +- .../ShaderParser/Model/ShaderModel.cs | 4 +- .../Ghost.DSL/ShaderParser/ShaderVisitor.cs | 2 +- .../Contracts/IShaderCompiler.cs | 66 ++++++ .../Services/DXCShaderCompiler.cs | 199 +----------------- .../Utilities/ShaderCompilerUtility.cs | 167 +++++++++++++++ .../Ghost.Core/Graphics/ShaderDescriptor.cs | 19 +- src/Runtime/Ghost.Core/Result.cs | 59 ++++++ .../Utilities/NativeMemoryManager.cs | 46 ++++ .../RenderPipeline/GhostRenderPipeline.cs | 1 + .../D3D12GraphicsEngine.cs | 2 +- .../D3D12WorkGraphPipeline.cs | 15 -- .../Ghost.Graphics.RHI/IShaderCompiler.cs | 152 ------------- src/Runtime/Ghost.Graphics/Core/Material.cs | 7 +- .../Ghost.Graphics/Core/RenderContext.cs | 1 + src/Runtime/Ghost.Graphics/Core/Shader.cs | 18 +- .../IShaderCompilationBridge.cs | 19 ++ .../RenderGraphModule/RenderGraph.cs | 28 +-- .../RenderGraphModule/RenderGraphContext.cs | 9 +- .../RenderGraphModule/RenderGraphExecutor.cs | 3 +- src/Runtime/Ghost.Graphics/RenderSystem.cs | 18 +- .../Services/ResourceManager.Pool.cs | 2 +- .../Services/ResourceManager.cs | 10 +- .../Services/ResourceUploadBatch.cs | 3 +- .../Ghost.Graphics/Services/ShaderLibrary.cs | 98 ++++++++- .../Services/SwapChainManager.cs | 2 +- .../Utilities/RenderingUtility.cs | 1 + .../RenderPipeline/TestRenderPipeline.cs | 2 +- 30 files changed, 630 insertions(+), 500 deletions(-) create mode 100644 src/Editor/Ghost.Editor.Core/Contracts/IShaderCompiler.cs create mode 100644 src/Editor/Ghost.Editor.Core/Utilities/ShaderCompilerUtility.cs create mode 100644 src/Runtime/Ghost.Core/Utilities/NativeMemoryManager.cs delete mode 100644 src/Runtime/Ghost.Graphics.D3D12/D3D12WorkGraphPipeline.cs delete mode 100644 src/Runtime/Ghost.Graphics.RHI/IShaderCompiler.cs create mode 100644 src/Runtime/Ghost.Graphics/IShaderCompilationBridge.cs diff --git a/src/Editor/Ghost.DSL/ShaderCompiler/DSLShaderCompiler.cs b/src/Editor/Ghost.DSL/ShaderCompiler/DSLShaderCompiler.cs index e4b3ac2..d54dc2f 100644 --- a/src/Editor/Ghost.DSL/ShaderCompiler/DSLShaderCompiler.cs +++ b/src/Editor/Ghost.DSL/ShaderCompiler/DSLShaderCompiler.cs @@ -1,8 +1,10 @@ using Ghost.Core; using Ghost.Core.Graphics; +using Ghost.Core.Utilities; using Ghost.DSL.ShaderParser; using Misaki.HighPerformance.Utilities; using System.IO.Hashing; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; @@ -22,9 +24,15 @@ public struct DSLShaderError internal static class DSLShaderCompiler { - private static ulong GetPassUniqueId(DSLShaderSemantics shader, PassSemantic pass) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong GetUniqueId(string code) { - return XxHash64.HashToUInt64(MemoryMarshal.AsBytes($"{shader.name}_{pass.name}".AsSpan())); + if (string.IsNullOrEmpty(code)) + { + return 0; + } + + return XxHash64.HashToUInt64(MemoryMarshal.AsBytes(code.AsSpan())); } private static PipelineState MeragePipeline(PipelineSemantic? semantic, PipelineState parent) @@ -132,9 +140,13 @@ internal static class DSLShaderCompiler var pixelShaderCode = new ShaderCode { code = result.Value, entryPoint = pass.pixelShader.entry }; + var asHash = Hash.Combine64(GetUniqueId(amplificationShaderCode.code), GetUniqueId(amplificationShaderCode.entryPoint)); + var msHash = Hash.Combine64(GetUniqueId(meshShaderCode.code), GetUniqueId(meshShaderCode.entryPoint)); + var psHash = Hash.Combine64(GetUniqueId(pixelShaderCode.code), GetUniqueId(pixelShaderCode.entryPoint)); + passes[i] = new PassDescriptor { - identifier = GetPassUniqueId(semantics, pass), + identifier = Hash.Combine64(GetUniqueId(semantics.name + pass.name), asHash, msHash, psHash), name = pass.name, amplificationShaderCode = amplificationShaderCode, @@ -289,7 +301,6 @@ internal static class DSLShaderCompiler return new ComputeShaderDescriptor { - identifier = XxHash64.HashToUInt64(MemoryMarshal.AsBytes(semantics.name.AsSpan())), name = semantics.name, propertyBufferSize = propertyInfo.size, shaderModel = semantics.shaderModel, diff --git a/src/Editor/Ghost.DSL/ShaderParser/AntlrShaderCompiler.cs b/src/Editor/Ghost.DSL/ShaderParser/AntlrShaderCompiler.cs index 7711040..9ab73a6 100644 --- a/src/Editor/Ghost.DSL/ShaderParser/AntlrShaderCompiler.cs +++ b/src/Editor/Ghost.DSL/ShaderParser/AntlrShaderCompiler.cs @@ -99,6 +99,40 @@ public class AntlrShaderCompiler } } + private static bool TryGetShaderModel(string model, List errors, out ShaderModel shaderModel) + { + if (string.IsNullOrEmpty(model)) + { + shaderModel = ShaderModel.SM_6_6; // Default to lowest supported shader model for compute shaders + } + else + { + switch (model) + { + case "6_6": + shaderModel = ShaderModel.SM_6_6; + break; + case "6_7": + shaderModel = ShaderModel.SM_6_7; + break; + case "6_8": + shaderModel = ShaderModel.SM_6_8; + break; + default: + shaderModel = default; + errors.Add(new DSLShaderError + { + message = $"Unknown shader model '{model}'.", + line = 0, + column = 0 + }); + return false; + } + } + + return true; + } + public static DSLComputeShaderSemantics? ConvertToComputeSemantics(ComputeShaderModel model, out List errors) { errors = new List(); @@ -122,29 +156,9 @@ public class AntlrShaderCompiler hlsl = model.Hlsl?.Code }; - if (string.IsNullOrEmpty(model.SM)) + if (TryGetShaderModel(model.ShaderModel, errors, out var shaderModel)) { - semantics.shaderModel = ShaderModel.SM_6_8; // Default to highest supported shader model - } - else - { - semantics.shaderModel = model.SM.ToLower() switch - { - "6_6" => ShaderModel.SM_6_6, - "6_7" => ShaderModel.SM_6_7, - "6_8" => ShaderModel.SM_6_8, - _ => ShaderModel.Invalid - }; - - if (semantics.shaderModel == ShaderModel.Invalid) - { - errors.Add(new DSLShaderError - { - message = $"Unknown shader model '{model.SM}'.", - line = 0, - column = 0 - }); - } + semantics.shaderModel = shaderModel; } if (model.Keywords != null) @@ -207,65 +221,6 @@ public class AntlrShaderCompiler return semantics; } - public static DSLShaderSemantics? ConvertToSemantics(GraphicsShaderModel model, out List errors) - { - errors = new List(); - - if (string.IsNullOrWhiteSpace(model.Name)) - { - errors.Add(new DSLShaderError - { - message = "Shader name cannot be empty.", - line = 0, - column = 0 - }); - return null; - } - - var semantics = new DSLShaderSemantics - { - name = model.Name, - pipeline = ConvertPipeline(model.Pipeline, errors) - }; - - if (string.IsNullOrEmpty(model.SM)) - { - semantics.shaderModel = ShaderModel.SM_6_8; // Default to highest supported shader model - } - else - { - semantics.shaderModel = model.SM.ToLower() switch - { - "6_6" => ShaderModel.SM_6_6, - "6_7" => ShaderModel.SM_6_7, - "6_8" => ShaderModel.SM_6_8, - _ => ShaderModel.Invalid - }; - - if (semantics.shaderModel == ShaderModel.Invalid) - { - errors.Add(new DSLShaderError - { - message = $"Unknown shader model '{model.SM}'.", - line = 0, - column = 0 - }); - } - } - - foreach (var pass in model.Passes) - { - var passSemantic = ConvertPass(pass, errors); - if (passSemantic != null) - { - semantics.passes ??= new List(); - semantics.passes.Add(passSemantic); - } - } - - return semantics; - } - private static PipelineSemantic? ConvertPipeline(PipelineBlockModel? pipeline, List errors) { if (pipeline == null || pipeline.Statements.Count == 0) @@ -394,6 +349,45 @@ public class AntlrShaderCompiler return semantic; } + public static DSLShaderSemantics? ConvertToSemantics(GraphicsShaderModel model, out List errors) + { + errors = new List(); + + if (string.IsNullOrWhiteSpace(model.Name)) + { + errors.Add(new DSLShaderError + { + message = "Shader name cannot be empty.", + line = 0, + column = 0 + }); + return null; + } + + var semantics = new DSLShaderSemantics + { + name = model.Name, + pipeline = ConvertPipeline(model.Pipeline, errors) + }; + + if (TryGetShaderModel(model.ShaderModel, errors, out var shaderModel)) + { + semantics.shaderModel = shaderModel; + } + + foreach (var pass in model.Passes) + { + var passSemantic = ConvertPass(pass, errors); + if (passSemantic != null) + { + semantics.passes ??= new List(); + semantics.passes.Add(passSemantic); + } + } + + return semantics; + } + private class ErrorListener : BaseErrorListener, IAntlrErrorListener, IAntlrErrorListener { private readonly List _errors; diff --git a/src/Editor/Ghost.DSL/ShaderParser/ComputeShaderVisitor.cs b/src/Editor/Ghost.DSL/ShaderParser/ComputeShaderVisitor.cs index dc7144a..6509919 100644 --- a/src/Editor/Ghost.DSL/ShaderParser/ComputeShaderVisitor.cs +++ b/src/Editor/Ghost.DSL/ShaderParser/ComputeShaderVisitor.cs @@ -38,7 +38,7 @@ internal class ComputeShaderVisitor : GhostComputeShaderParserBaseVisitor Passes { get; set; } = new(); public List FunctionCalls { get; set; } = new(); @@ -12,7 +12,7 @@ public class GraphicsShaderModel public class ComputeShaderModel { public string Name { get; set; } = string.Empty; - public string SM { get; set; } = string.Empty; + public string ShaderModel { get; set; } = string.Empty; public DefinesBlockModel? Defines { get; set; } public IncludesBlockModel? Includes { get; set; } public KeywordsBlockModel? Keywords { get; set; } diff --git a/src/Editor/Ghost.DSL/ShaderParser/ShaderVisitor.cs b/src/Editor/Ghost.DSL/ShaderParser/ShaderVisitor.cs index a695a76..6dbd4e1 100644 --- a/src/Editor/Ghost.DSL/ShaderParser/ShaderVisitor.cs +++ b/src/Editor/Ghost.DSL/ShaderParser/ShaderVisitor.cs @@ -27,7 +27,7 @@ public class ShaderVisitor : GhostShaderParserBaseVisitor var shaderBody = context.shaderBody(); if (shaderBody != null) { - shader.SM = shaderBody.shaderModel()?.GetText() ?? string.Empty; + shader.ShaderModel = shaderBody.shaderModel()?.GetText() ?? string.Empty; foreach (var pipelineBlock in shaderBody.pipelineBlock()) { diff --git a/src/Editor/Ghost.Editor.Core/Contracts/IShaderCompiler.cs b/src/Editor/Ghost.Editor.Core/Contracts/IShaderCompiler.cs new file mode 100644 index 0000000..1171b3a --- /dev/null +++ b/src/Editor/Ghost.Editor.Core/Contracts/IShaderCompiler.cs @@ -0,0 +1,66 @@ +using Ghost.Core; +using Ghost.Core.Graphics; +using Ghost.Core.Utilities; +using Misaki.HighPerformance.LowLevel.Buffer; +using Misaki.HighPerformance.LowLevel.Collections; + +namespace Ghost.Editor.Core.Contracts; + +public unsafe struct ComputeCompileResult +{ + public fixed ulong resultHash[8]; + public readonly int count; + + public ulong HashCode + { + get + { + var a = Hash.Combine64(resultHash[0], resultHash[1], resultHash[2], resultHash[3]); + var b = Hash.Combine64(resultHash[4], resultHash[5], resultHash[6], resultHash[7]); + return Hash.Combine64(a, b); + } + } +} + +public ref struct ShaderCompilationConfig +{ + public ReadOnlySpan defines; + public string shaderCode; + public string entryPoint; + public ShaderStage stage; + public ShaderModel model; + public CompilerOptimizeLevel optimizeLevel; + public CompilerOption options; +} + +public enum CompilerOptimizeLevel +{ + O0, + O1, + O2, + O3 +} + +[Flags] +public enum CompilerOption +{ + None = 0, + KeepDebugInfo = 1 << 0, + KeepReflections = 1 << 1, + WarnAsError = 1 << 2, + SpirvCrossCompile = 1 << 3 +} + +public enum ShaderStage +{ + TaskShader, + MeshShader, + PixelShader, + ComputeShader, + Library // For ray tracing shaders or work graph shaders that don't fit into the traditional shader stages +} + +public interface IShaderCompiler : IDisposable +{ + Result> Compile(ref readonly ShaderCompilationConfig config, AllocationHandle handle); +} \ No newline at end of file diff --git a/src/Editor/Ghost.Editor.Core/Services/DXCShaderCompiler.cs b/src/Editor/Ghost.Editor.Core/Services/DXCShaderCompiler.cs index 6d4683e..397c8f9 100644 --- a/src/Editor/Ghost.Editor.Core/Services/DXCShaderCompiler.cs +++ b/src/Editor/Ghost.Editor.Core/Services/DXCShaderCompiler.cs @@ -1,18 +1,15 @@ using Ghost.Core; using Ghost.Core.Graphics; +using Ghost.DXC; +using Ghost.Editor.Core.Contracts; using Ghost.Graphics.D3D12.Utilities; -using Ghost.Graphics.RHI; using Misaki.HighPerformance.LowLevel; using Misaki.HighPerformance.LowLevel.Buffer; using Misaki.HighPerformance.LowLevel.Collections; -using Misaki.HighPerformance.Utilities; -using System.IO.Hashing; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; -using Ghost.DXC; - using static Ghost.DXC.UUID; -using System.Runtime.CompilerServices; namespace Ghost.Graphics.Core; @@ -26,14 +23,17 @@ internal sealed partial class DXCShaderCompiler (ShaderStage.PixelShader, ShaderModel.SM_6_6) => "ps_6_6", (ShaderStage.MeshShader, ShaderModel.SM_6_6) => "ms_6_6", (ShaderStage.ComputeShader, ShaderModel.SM_6_6) => "cs_6_6", + (ShaderStage.Library, ShaderModel.SM_6_6) => "lib_6_6", (ShaderStage.TaskShader, ShaderModel.SM_6_7) => "as_6_7", (ShaderStage.PixelShader, ShaderModel.SM_6_7) => "ps_6_7", (ShaderStage.MeshShader, ShaderModel.SM_6_7) => "ms_6_7", (ShaderStage.ComputeShader, ShaderModel.SM_6_7) => "cs_6_7", + (ShaderStage.Library, ShaderModel.SM_6_7) => "lib_6_7", (ShaderStage.TaskShader, ShaderModel.SM_6_8) => "as_6_8", (ShaderStage.PixelShader, ShaderModel.SM_6_8) => "ps_6_8", (ShaderStage.MeshShader, ShaderModel.SM_6_8) => "ms_6_8", (ShaderStage.ComputeShader, ShaderModel.SM_6_8) => "cs_6_8", + (ShaderStage.Library, ShaderModel.SM_6_8) => "lib_6_8", _ => throw new ArgumentOutOfRangeException(nameof(stage), "Unsupported shader stage or compiler version") }; } @@ -105,49 +105,6 @@ internal sealed partial class DXCShaderCompiler return argsArray; } - - private static Result BuildFinalShaderCode(string shaderPath, ReadOnlySpan includes, string? injectedCode) - { - string shaderCode; - if (shaderPath == "hlsl_block") - { - if (string.IsNullOrEmpty(injectedCode)) - { - return Error.InvalidArgument; - } - - shaderCode = string.Empty; - } - else - { - if (!File.Exists(shaderPath)) - { - return Error.NotFound; - } - - shaderCode = File.ReadAllText(shaderPath); - } - - var sb = new StringBuilder(); - foreach (var includePath in includes) - { - sb.AppendLine($"#include \"{includePath}\""); - } - - if (!string.IsNullOrEmpty(injectedCode)) - { - sb.AppendLine($"#line 0 \"injected_code\""); - sb.AppendLine(injectedCode); - } - - if (!string.IsNullOrEmpty(shaderCode)) - { - sb.AppendLine($"#line 0 \"{shaderPath}\""); - sb.AppendLine(shaderCode); - } - - return sb.ToString(); - } } internal sealed unsafe partial class DXCShaderCompiler : IShaderCompiler @@ -155,9 +112,6 @@ internal sealed unsafe partial class DXCShaderCompiler : IShaderCompiler private UniquePtr _compiler; private UniquePtr _utils; - // NOTE: This is just a temporary cache for compiled shader code. We will implement a proper disk cache later. - private readonly Dictionary, ShaderCompileResult> _compiledResults; - private bool _disposed; public DXCShaderCompiler() @@ -179,8 +133,6 @@ internal sealed unsafe partial class DXCShaderCompiler : IShaderCompiler _compiler.Attach(pCompiler); _utils.Attach(pUtils); - - _compiledResults = new Dictionary, ShaderCompileResult>(); } ~DXCShaderCompiler() @@ -188,7 +140,7 @@ internal sealed unsafe partial class DXCShaderCompiler : IShaderCompiler Dispose(); } - public Result> Compile(ref readonly ShaderCompilationConfig config) + public Result> Compile(ref readonly ShaderCompilationConfig config, AllocationHandle allocationHandle) { ObjectDisposedException.ThrowIf(_disposed, this); @@ -221,7 +173,7 @@ internal sealed unsafe partial class DXCShaderCompiler : IShaderCompiler IDxcResult* result = default; IDxcBlob* bytecodeBlob = default; - + try { // Compile shader @@ -268,18 +220,11 @@ internal sealed unsafe partial class DXCShaderCompiler : IShaderCompiler } var bytecodeSize = bytecodeBlob->GetBufferSize(); - var bytecode = new UnsafeArray((int)bytecodeSize, Allocator.Persistent); + var bytecode = new UnsafeArray((int)bytecodeSize, allocationHandle); NativeMemory.Copy(bytecodeBlob->GetBufferPointer(), bytecode.GetUnsafePtr(), (nuint)bytecodeSize); - var compileResult = new ShaderCompileResult - { - bytecode = bytecode, - hashCode = XxHash64.HashToUInt64(bytecode) - }; - - _compiledResults[compileResult.hashCode] = compileResult; - return new Key64(compileResult.hashCode); + return bytecode; } finally { @@ -313,125 +258,6 @@ internal sealed unsafe partial class DXCShaderCompiler : IShaderCompiler } } - public Result CompilePass(ref readonly PassDescriptor descriptor, ref readonly ShaderCompilationConfig additionalConfig, ref readonly LocalKeywordSet keywords) - { - ObjectDisposedException.ThrowIf(_disposed, this); - - string[] fullDefines; - var totalDefineCount = descriptor.defines.Length + additionalConfig.defines.Length; - if (totalDefineCount == 0) - { - fullDefines = Array.Empty(); - } - else - { - fullDefines = new string[totalDefineCount]; - descriptor.defines.CopyTo(fullDefines); - additionalConfig.defines.CopyTo(fullDefines.AsSpan(descriptor.defines.Length)); - } - - Key64 tsResult = default; - var asCode = descriptor.amplificationShaderCode; - if (asCode.IsCreated) - { - var config = new ShaderCompilationConfig - { - defines = fullDefines, - shaderCode = asCode.code, - entryPoint = asCode.entryPoint, - stage = ShaderStage.TaskShader, - model = additionalConfig.model, - optimizeLevel = additionalConfig.optimizeLevel, - options = additionalConfig.options, - }; - - var result = Compile(ref config); - if (result.IsFailure) - { - return Result.Failure(result.Message); - } - - tsResult = result.Value; - } - - Key64 msResult; - var msCode = descriptor.meshShaderCode; - if (msCode.IsCreated) - { - var config = new ShaderCompilationConfig - { - defines = fullDefines, - shaderCode = msCode.code, - entryPoint = msCode.entryPoint, - stage = ShaderStage.MeshShader, - model = additionalConfig.model, - optimizeLevel = additionalConfig.optimizeLevel, - options = additionalConfig.options, - }; - - var result = Compile(ref config); - if (result.IsFailure) - { - return Result.Failure(result.Message); - } - - msResult = result.Value; - } - else - { - return Result.Failure("Mesh shader expected."); - } - - Key64 psResult; - var psCode = descriptor.pixelShaderCode; - if (psCode.IsCreated) - { - var config = new ShaderCompilationConfig - { - defines = fullDefines, - shaderCode = psCode.code, - entryPoint = psCode.entryPoint, - stage = ShaderStage.PixelShader, - model = additionalConfig.model, - optimizeLevel = additionalConfig.optimizeLevel, - options = additionalConfig.options, - }; - - var result = Compile(ref config); - if (result.IsFailure) - { - return Result.Failure(result.Message); - } - - psResult = result.Value; - } - else - { - return Result.Failure("Pixel shader expected."); - } - - var compiled = new GraphicsCompiledResult - { - tsResultHash = tsResult, - msResultHash = msResult, - psResultHash = psResult, - }; - - return compiled; - } - - public Result GetCompiledCache(Key64 key) - { - ObjectDisposedException.ThrowIf(_disposed, this); - - if (_compiledResults.TryGetValue(key, out var compiledResult)) - { - return compiledResult; - } - - return Error.NotFound; - } - public void Dispose() { if (_disposed) @@ -439,11 +265,6 @@ internal sealed unsafe partial class DXCShaderCompiler : IShaderCompiler return; } - foreach (var kvp in _compiledResults) - { - kvp.Value.Dispose(); - } - _compiler.Get()->Release(); _utils.Get()->Release(); diff --git a/src/Editor/Ghost.Editor.Core/Utilities/ShaderCompilerUtility.cs b/src/Editor/Ghost.Editor.Core/Utilities/ShaderCompilerUtility.cs new file mode 100644 index 0000000..5f12f9a --- /dev/null +++ b/src/Editor/Ghost.Editor.Core/Utilities/ShaderCompilerUtility.cs @@ -0,0 +1,167 @@ +using Ghost.Core; +using Ghost.Core.Graphics; +using Ghost.Editor.Core.Contracts; +using Misaki.HighPerformance.LowLevel.Buffer; +using Misaki.HighPerformance.LowLevel.Collections; + +namespace Ghost.Editor.Core.Utilities; + +internal struct GraphicsCompiledResult : IDisposable +{ + public UnsafeArray asResult; + public UnsafeArray msResult; + public UnsafeArray psResult; + + public void Dispose() + { + asResult.Dispose(); + msResult.Dispose(); + psResult.Dispose(); + } +} + +internal static class ShaderCompilerUtility +{ + private static ReadOnlySpan CombineDefines(ReadOnlySpan a, ReadOnlySpan b) + { + ReadOnlySpan combined; + if (b.Length == 0) + { + combined = a; + } + else if (a.Length == 0) + { + combined = b; + } + else + { + var combinedDefines = new string[a.Length + b.Length]; + a.CopyTo(combinedDefines); + b.CopyTo(combinedDefines.AsSpan(a.Length)); + combined = combinedDefines; + } + + return combined; + } + + public static Result CompileShaderPass(this IShaderCompiler shaderCompiler, ref readonly PassDescriptor descriptor, ref readonly ShaderCompilationConfig additionalConfig, AllocationHandle allocationHandle) + { + var fullDefines = CombineDefines(descriptor.defines, additionalConfig.defines); + + var config = new ShaderCompilationConfig + { + defines = fullDefines, + model = additionalConfig.model, + optimizeLevel = additionalConfig.optimizeLevel, + options = additionalConfig.options + }; + + UnsafeArray asResult = default; + if (descriptor.amplificationShaderCode.IsCreated) + { + config.shaderCode = descriptor.amplificationShaderCode.code; + config.entryPoint = descriptor.amplificationShaderCode.entryPoint; + config.stage = ShaderStage.TaskShader; + + var result = shaderCompiler.Compile(ref config, allocationHandle); + if (result.IsFailure) + { + return Result.Failure(result.Message); + } + + asResult = result.Value; + } + + UnsafeArray msResult; + if (descriptor.meshShaderCode.IsCreated) + { + config.shaderCode = descriptor.meshShaderCode.code; + config.entryPoint = descriptor.meshShaderCode.entryPoint; + config.stage = ShaderStage.MeshShader; + + var result = shaderCompiler.Compile(ref config, allocationHandle); + if (result.IsFailure) + { + asResult.Dispose(); + return Result.Failure(result.Message); + } + + msResult = result.Value; + } + else + { + asResult.Dispose(); + return Result.Failure("Mesh shader expected."); + } + + UnsafeArray psResult; + if (descriptor.pixelShaderCode.IsCreated) + { + config.shaderCode = descriptor.pixelShaderCode.code; + config.entryPoint = descriptor.pixelShaderCode.entryPoint; + config.stage = ShaderStage.PixelShader; + + var result = shaderCompiler.Compile(ref config, allocationHandle); + if (result.IsFailure) + { + asResult.Dispose(); + msResult.Dispose(); + return Result.Failure(result.Message); + } + + psResult = result.Value; + } + else + { + asResult.Dispose(); + msResult.Dispose(); + return Result.Failure("Pixel shader expected."); + } + + var compiled = new GraphicsCompiledResult + { + asResult = asResult, + msResult = msResult, + psResult = psResult, + }; + + return compiled; + } + + public static Result>> CompileComputeShader(this IShaderCompiler shaderCompiler, ComputeShaderDescriptor descriptor, ref readonly ShaderCompilationConfig additionalConfig, AllocationHandle allocationHandle) + { + var fullDefines = CombineDefines(descriptor.defines, additionalConfig.defines); + + var config = new ShaderCompilationConfig + { + defines = fullDefines, + model = additionalConfig.model, + optimizeLevel = additionalConfig.optimizeLevel, + options = additionalConfig.options, + stage = ShaderStage.ComputeShader, + }; + + var compiled = new UnsafeArray>(descriptor.shaderCodes.Length, allocationHandle); + for (int i = 0; i < descriptor.shaderCodes.Length; i++) + { + config.shaderCode = descriptor.shaderCodes[i].code; + config.entryPoint = descriptor.shaderCodes[i].entryPoint; + + var result = shaderCompiler.Compile(ref config, allocationHandle); + if (result.IsFailure) + { + for (int j = 0; j < i; j++) + { + compiled[j].Dispose(); + } + + compiled.Dispose(); + return Result.Failure(result.Message); + } + + compiled[i] = result.Value; + } + + return compiled; + } +} \ No newline at end of file diff --git a/src/Runtime/Ghost.Core/Graphics/ShaderDescriptor.cs b/src/Runtime/Ghost.Core/Graphics/ShaderDescriptor.cs index a142075..a3481c4 100644 --- a/src/Runtime/Ghost.Core/Graphics/ShaderDescriptor.cs +++ b/src/Runtime/Ghost.Core/Graphics/ShaderDescriptor.cs @@ -1,8 +1,12 @@ +using Ghost.Core.Utilities; +using System.IO.Hashing; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + namespace Ghost.Core.Graphics; public enum ShaderModel { - Invalid, SM_6_6, SM_6_7, SM_6_8 @@ -20,6 +24,18 @@ public struct ShaderCode public string entryPoint; public readonly bool IsCreated => !string.IsNullOrEmpty(code) && !string.IsNullOrEmpty(entryPoint); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly ulong GetHashCode64() + { + return Hash.Combine64(XxHash64.HashToUInt64(MemoryMarshal.AsBytes(code.AsSpan())), XxHash64.HashToUInt64(MemoryMarshal.AsBytes(entryPoint.AsSpan()))); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override readonly int GetHashCode() + { + return HashCode.Combine(code, entryPoint); + } } public struct KeywordsGroup @@ -53,7 +69,6 @@ public class GraphicsShaderDescriptor public class ComputeShaderDescriptor { - public required ulong identifier; public required string name = string.Empty; public required uint propertyBufferSize; public required ShaderModel shaderModel; diff --git a/src/Runtime/Ghost.Core/Result.cs b/src/Runtime/Ghost.Core/Result.cs index 2800162..fcdfaa3 100644 --- a/src/Runtime/Ghost.Core/Result.cs +++ b/src/Runtime/Ghost.Core/Result.cs @@ -1,5 +1,6 @@ using Misaki.HighPerformance.LowLevel; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace Ghost.Core; @@ -260,6 +261,64 @@ public readonly ref struct RefResult public static implicit operator bool(RefResult result) => result.IsSuccess; } +[AsyncMethodBuilder(typeof(AsyncValueTaskMethodBuilder))] +public readonly struct ResultTask +{ + private readonly ValueTask _task; + + public ResultTask(ValueTask task) + { + _task = task; + } + + public ValueTaskAwaiter GetAwaiter() => _task.GetAwaiter(); + + public ValueTask AsValueTask() => _task; + public Task AsTask() => _task.AsTask(); + + public static implicit operator ResultTask(ValueTask task) => new ResultTask(task); + public static implicit operator ValueTask(ResultTask resultTask) => resultTask._task; +} + +[AsyncMethodBuilder(typeof(AsyncValueTaskMethodBuilder))] +public readonly struct ResultTask +{ + private readonly ValueTask> _task; + + public ResultTask(ValueTask> task) + { + _task = task; + } + + public ValueTaskAwaiter> GetAwaiter() => _task.GetAwaiter(); + + public ValueTask> AsValueTask() => _task; + public Task> AsTask() => _task.AsTask(); + + public static implicit operator ResultTask(ValueTask> task) => new ResultTask(task); + public static implicit operator ValueTask>(ResultTask resultTask) => resultTask._task; +} + +[AsyncMethodBuilder(typeof(AsyncValueTaskMethodBuilder))] +public readonly struct ResultTask + where E : struct, Enum +{ + private readonly ValueTask> _task; + + public ResultTask(ValueTask> task) + { + _task = task; + } + + public ValueTaskAwaiter> GetAwaiter() => _task.GetAwaiter(); + + public ValueTask> AsValueTask() => _task; + public Task> AsTask() => _task.AsTask(); + + public static implicit operator ResultTask(ValueTask> task) => new ResultTask(task); + public static implicit operator ValueTask>(ResultTask resultTask) => resultTask._task; +} + public static class ResultExtensions { extension(Error error) diff --git a/src/Runtime/Ghost.Core/Utilities/NativeMemoryManager.cs b/src/Runtime/Ghost.Core/Utilities/NativeMemoryManager.cs new file mode 100644 index 0000000..d95f2dc --- /dev/null +++ b/src/Runtime/Ghost.Core/Utilities/NativeMemoryManager.cs @@ -0,0 +1,46 @@ +using Misaki.HighPerformance.LowLevel.Collections.Contracts; +using System.Buffers; + +namespace Ghost.Core.Utilities; + +public unsafe class NativeMemoryManager : MemoryManager + where T : unmanaged +{ + private readonly T* _pointer; + private readonly int _length; + + public NativeMemoryManager(T* pointer, int length) + { + _pointer = pointer; + _length = length; + } + + public static NativeMemoryManager FromUnsafeCollection(ref readonly C collection) + where C : unmanaged, IUnsafeCollection + { + if (!collection.IsCreated) + { + throw new InvalidOperationException("The collection is not created."); + } + + return new NativeMemoryManager((T*)collection.GetUnsafePtr(), collection.Count); + } + + public override Span GetSpan() + { + return new Span(_pointer, _length); + } + + public override MemoryHandle Pin(int elementIndex = 0) + { + return new MemoryHandle(_pointer + elementIndex); + } + + public override void Unpin() + { + } + + protected override void Dispose(bool disposing) + { + } +} diff --git a/src/Runtime/Ghost.Engine/RenderPipeline/GhostRenderPipeline.cs b/src/Runtime/Ghost.Engine/RenderPipeline/GhostRenderPipeline.cs index 4de1432..2d48e7f 100644 --- a/src/Runtime/Ghost.Engine/RenderPipeline/GhostRenderPipeline.cs +++ b/src/Runtime/Ghost.Engine/RenderPipeline/GhostRenderPipeline.cs @@ -3,6 +3,7 @@ using Ghost.Graphics; using Ghost.Graphics.Core; using Ghost.Graphics.RenderGraphModule; using Ghost.Graphics.RHI; +using Ghost.Graphics.Services; using Misaki.HighPerformance.LowLevel.Utilities; using Misaki.HighPerformance.Mathematics; using System.Diagnostics; diff --git a/src/Runtime/Ghost.Graphics.D3D12/D3D12GraphicsEngine.cs b/src/Runtime/Ghost.Graphics.D3D12/D3D12GraphicsEngine.cs index 498ef6a..f0c9bbb 100644 --- a/src/Runtime/Ghost.Graphics.D3D12/D3D12GraphicsEngine.cs +++ b/src/Runtime/Ghost.Graphics.D3D12/D3D12GraphicsEngine.cs @@ -66,7 +66,7 @@ internal class D3D12GraphicsEngine : IGraphicsEngine _descriptorAllocator = new D3D12DescriptorAllocator(_device); _resourceDatabase = new D3D12ResourceDatabase(_descriptorAllocator); - _pipelineLibrary = new D3D12PipelineLibrary(_device, _resourceDatabase); + _pipelineLibrary = new D3D12PipelineLibrary(_device); _resourceAllocator = new D3D12ResourceAllocator(_device, _descriptorAllocator, _resourceDatabase, _pipelineLibrary); _commandBufferPool = new List(4); diff --git a/src/Runtime/Ghost.Graphics.D3D12/D3D12WorkGraphPipeline.cs b/src/Runtime/Ghost.Graphics.D3D12/D3D12WorkGraphPipeline.cs deleted file mode 100644 index 211a6a8..0000000 --- a/src/Runtime/Ghost.Graphics.D3D12/D3D12WorkGraphPipeline.cs +++ /dev/null @@ -1,15 +0,0 @@ -using Ghost.Core; -using Ghost.Graphics.RHI; -using Misaki.HighPerformance.LowLevel; -using TerraFX.Interop.DirectX; - -namespace Ghost.Graphics.D3D12; - -internal class D3D12WorkGraphPipeline : IWorkGraphPipeline -{ - private UniquePtr _stateObject; - private D3D12_PROGRAM_IDENTIFIER _programIdentifier; - private D3D12_WORK_GRAPH_MEMORY_REQUIREMENTS _memoryRequirements; - - private Handle _backingBuffer; -} diff --git a/src/Runtime/Ghost.Graphics.RHI/IShaderCompiler.cs b/src/Runtime/Ghost.Graphics.RHI/IShaderCompiler.cs deleted file mode 100644 index d969d84..0000000 --- a/src/Runtime/Ghost.Graphics.RHI/IShaderCompiler.cs +++ /dev/null @@ -1,152 +0,0 @@ -using Ghost.Core; -using Ghost.Core.Graphics; -using Ghost.Core.Utilities; -using Misaki.HighPerformance.LowLevel.Collections; - -namespace Ghost.Graphics.RHI; - -public struct ShaderCompileResult : IDisposable -{ - public UnsafeArray bytecode; - public ulong hashCode; - - public readonly bool IsCreated => bytecode.IsCreated; - - public void Dispose() - { - bytecode.Dispose(); - } -} - -public struct GraphicsCompiledResult -{ - public ulong tsResultHash; - public ulong msResultHash; - public ulong psResultHash; - - public readonly ulong HashCode => Hash.Combine64(tsResultHash, msResultHash, psResultHash); -} - -public unsafe struct ComputeCompileResult -{ - public fixed ulong resultHash[8]; - public readonly int count; - - public ulong HashCode - { - get - { - var a = Hash.Combine64(resultHash[0], resultHash[1], resultHash[2], resultHash[3]); - var b = Hash.Combine64(resultHash[4], resultHash[5], resultHash[6], resultHash[7]); - return Hash.Combine64(a, b); - } - } -} - -public ref struct ShaderCompilationConfig -{ - public ReadOnlySpan defines; - public string shaderCode; - public string entryPoint; - public ShaderStage stage; - public ShaderModel model; - public CompilerOptimizeLevel optimizeLevel; - public CompilerOption options; -} - -public enum CompilerOptimizeLevel -{ - O0, - O1, - O2, - O3 -} - -[Flags] -public enum CompilerOption -{ - None = 0, - KeepDebugInfo = 1 << 0, - KeepReflections = 1 << 1, - WarnAsError = 1 << 2, - SpirvCrossCompile = 1 << 3 -} - -public enum ShaderStage -{ - TaskShader, - MeshShader, - PixelShader, - ComputeShader, - Library // For ray tracing shaders or work graph shaders that don't fit into the traditional shader stages -} - -public enum ShaderInputType -{ - ConstantBuffer, - Texture, - Sampler, - UAV, - StructuredBuffer, - ByteAddressBuffer, - RWStructuredBuffer, - RWByteAddressBuffer -} - -public struct ResourceBindingInfo -{ - public string Name - { - get; set; - } - - public ShaderInputType Type - { - get; set; - } - - public uint BindPoint - { - get; set; - } - - public uint BindCount - { - get; set; - } - - public uint Space - { - get; set; - } - - public uint Size - { - get; set; - } - - public IReadOnlyList? Properties - { - get; set; - } -} - -public readonly struct ShaderReflectionData -{ - public List ResourcesBindings - { - get; - } - - public ShaderReflectionData() - { - ResourcesBindings = new List(); - } -} - -public interface IShaderCompiler : IDisposable -{ - Result> Compile(ref readonly ShaderCompilationConfig config); - Result CompilePass(ref readonly PassDescriptor descriptor, ref readonly ShaderCompilationConfig additionalConfig, ref readonly LocalKeywordSet keywords); - Result GetCompiledCache(Key64 key); -} diff --git a/src/Runtime/Ghost.Graphics/Core/Material.cs b/src/Runtime/Ghost.Graphics/Core/Material.cs index 5f29b94..151cf20 100644 --- a/src/Runtime/Ghost.Graphics/Core/Material.cs +++ b/src/Runtime/Ghost.Graphics/Core/Material.cs @@ -1,6 +1,7 @@ using Ghost.Core; using Ghost.Core.Graphics; using Ghost.Graphics.RHI; +using Ghost.Graphics.Services; using Misaki.HighPerformance.LowLevel.Buffer; using Misaki.HighPerformance.LowLevel.Collections; using System.Runtime.CompilerServices; @@ -87,7 +88,7 @@ public struct Material : IResourceReleasable return r.Error; } - ref readonly var shader = ref r.Value; + ref var shader = ref r.Value; if (_passPipelineOverride.Count < shader.PassCount) { if (!_passPipelineOverride.IsCreated) @@ -212,7 +213,7 @@ public struct Material : IResourceReleasable return r.Error; } - ref readonly var shader = ref r.Value; + ref var shader = ref r.Value; var localIndex = shader.GetLocalKeywordIndex(keywordId); if (localIndex == -1) { @@ -233,7 +234,7 @@ public struct Material : IResourceReleasable return false; } - ref readonly var shader = ref r.Value; + ref var shader = ref r.Value; var localIndex = shader.GetLocalKeywordIndex(keywordId); if (localIndex == -1) { diff --git a/src/Runtime/Ghost.Graphics/Core/RenderContext.cs b/src/Runtime/Ghost.Graphics/Core/RenderContext.cs index 0587545..ca7e336 100644 --- a/src/Runtime/Ghost.Graphics/Core/RenderContext.cs +++ b/src/Runtime/Ghost.Graphics/Core/RenderContext.cs @@ -1,5 +1,6 @@ using Ghost.Core; using Ghost.Graphics.RHI; +using Ghost.Graphics.Services; using Misaki.HighPerformance.LowLevel.Buffer; using Misaki.HighPerformance.LowLevel.Collections; using Misaki.HighPerformance.LowLevel.Utilities; diff --git a/src/Runtime/Ghost.Graphics/Core/Shader.cs b/src/Runtime/Ghost.Graphics/Core/Shader.cs index 3d9862b..1f62800 100644 --- a/src/Runtime/Ghost.Graphics/Core/Shader.cs +++ b/src/Runtime/Ghost.Graphics/Core/Shader.cs @@ -1,6 +1,5 @@ using Ghost.Core; using Ghost.Core.Graphics; -using Ghost.Core.Utilities; using Ghost.Graphics.RHI; using Misaki.HighPerformance.LowLevel.Buffer; using Misaki.HighPerformance.LowLevel.Collections; @@ -87,10 +86,8 @@ public partial struct Shader : IResourceReleasable public readonly int PassCount => _shaderPasses.Count; public readonly uint PropertyBufferSize => _propertyBufferSize; - internal Shader(GraphicsShaderDescriptor descriptor, ReadOnlySpan compiledResults) + internal Shader(GraphicsShaderDescriptor descriptor) { - Debug.Assert(descriptor.passes.Length == compiledResults.Length); - _propertyBufferSize = descriptor.propertyBufferSize; _shaderPasses = new UnsafeArray(descriptor.passes.Length, Allocator.Persistent); _passIDToLocal = new UnsafeHashMap(descriptor.passes.Length, Allocator.Persistent); @@ -100,8 +97,7 @@ public partial struct Shader : IResourceReleasable { ref readonly var pass = ref descriptor.passes[i]; - var passKey = RHIUtility.CreateShaderPassKey(pass.identifier, compiledResults[i].HashCode); - var keywords = default(LocalKeywordSet); + var keywords = new LocalKeywordSet(); if (pass.keywords.Length > 0) { @@ -133,7 +129,7 @@ public partial struct Shader : IResourceReleasable _shaderPasses[i] = new ShaderPass { - Key = passKey, + Key = pass.identifier, DefaultState = pass.localPipeline, KeywordIDs = keywords, }; @@ -213,10 +209,8 @@ public unsafe partial struct ComputeShader : IResourceReleasable public readonly uint PropertyBufferSize => _propertyBufferSize; - internal ComputeShader(ComputeShaderDescriptor descriptor, ReadOnlySpan compiledResults) + internal ComputeShader(ComputeShaderDescriptor descriptor) { - Debug.Assert(descriptor.shaderCodes.Length == compiledResults.Length); - _propertyBufferSize = descriptor.propertyBufferSize; _entryPointCount = descriptor.shaderCodes.Length; @@ -224,7 +218,7 @@ public unsafe partial struct ComputeShader : IResourceReleasable for (var i = 0; i < descriptor.shaderCodes.Length; i++) { - _entryHash[i] = Hash.Combine64(descriptor.identifier, compiledResults[i].hashCode); + _entryHash[i] = descriptor.shaderCodes[i].GetHashCode64(); } var localKeywordIndex = 0; @@ -271,4 +265,4 @@ public unsafe partial struct ComputeShader : IResourceReleasable { _keywordIDToLocal.Dispose(); } -} \ No newline at end of file +} diff --git a/src/Runtime/Ghost.Graphics/IShaderCompilationBridge.cs b/src/Runtime/Ghost.Graphics/IShaderCompilationBridge.cs new file mode 100644 index 0000000..d50f28d --- /dev/null +++ b/src/Runtime/Ghost.Graphics/IShaderCompilationBridge.cs @@ -0,0 +1,19 @@ +namespace Ghost.Graphics; + +public interface IShaderCompilationBridge +{ + bool TryGetBytecode(ulong manifestKey, out ReadOnlyMemory bytecode); + bool IsCompiling(ulong manifestKey); +} + +// NOTE: For testing only. +internal sealed class NullShaderCompilationBridge : IShaderCompilationBridge +{ + public bool TryGetBytecode(ulong manifestKey, out ReadOnlyMemory bytecode) + { + bytecode = default; + return false; // Always fall through to ShaderLibrary's disk cache + } + + public bool IsCompiling(ulong manifestKey) => false; +} diff --git a/src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraph.cs b/src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraph.cs index 94cc712..13fd791 100644 --- a/src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraph.cs +++ b/src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraph.cs @@ -1,6 +1,7 @@ using Ghost.Core; using Ghost.Graphics.RHI; using System.Diagnostics; +using Ghost.Graphics.Services; namespace Ghost.Graphics.RenderGraphModule; @@ -9,8 +10,6 @@ namespace Ghost.Graphics.RenderGraphModule; /// public sealed class RenderGraph : IDisposable { - private readonly ResourceManager _resourceManager; - private readonly IResourceAllocator _resourceAllocator; private readonly IResourceDatabase _resourceDatabase; private readonly RenderGraphObjectPool _objectPool; @@ -38,11 +37,9 @@ public sealed class RenderGraph : IDisposable public RenderGraphBlackboard Blackboard => _blackboard; - public RenderGraph(ResourceManager resourceManager, IResourceAllocator resourceAllocator, IResourceDatabase resourceDatabase, IPipelineLibrary pipelineLibrary, IShaderCompiler shaderCompiler) + public RenderGraph(RenderSystem renderSystem) { - _resourceManager = resourceManager; - _resourceAllocator = resourceAllocator; - _resourceDatabase = resourceDatabase; + _resourceDatabase = renderSystem.GraphicsEngine.ResourceDatabase; _objectPool = new RenderGraphObjectPool(); _resources = new RenderGraphResourceRegistry(_objectPool); @@ -52,30 +49,25 @@ public sealed class RenderGraph : IDisposable _nativePasses = new List(32); _builder = new RenderGraphBuilder(); - _aliasingManager = new ResourceAliasingManager(_resourceAllocator, _objectPool); + _aliasingManager = new ResourceAliasingManager(renderSystem.GraphicsEngine.ResourceAllocator, _objectPool); _compilationCache = new RenderGraphCompilationCache(); _context = new RenderGraphContext( - _resourceManager, - _resourceDatabase, - pipelineLibrary, - shaderCompiler, + renderSystem.ResourceManager, + renderSystem.ShaderLibrary, + renderSystem.GraphicsEngine.ResourceDatabase, + renderSystem.GraphicsEngine.PipelineLibrary, _resources ); _nativePassBuilder = new RenderGraphNativePassBuilder(_objectPool, _resources); - _compiler = new RenderGraphCompiler(_resourceDatabase, _resourceAllocator, _resources, _aliasingManager, _nativePassBuilder, _compilationCache); - _executor = new RenderGraphExecutor(_resourceManager, _resourceDatabase, _resources, _context); + _compiler = new RenderGraphCompiler(renderSystem.GraphicsEngine.ResourceDatabase, renderSystem.GraphicsEngine.ResourceAllocator, _resources, _aliasingManager, _nativePassBuilder, _compilationCache); + _executor = new RenderGraphExecutor(renderSystem.ResourceManager, renderSystem.GraphicsEngine.ResourceDatabase, _resources, _context); _blackboard = new RenderGraphBlackboard(); } - public RenderGraph(ResourceManager resourceManager, IGraphicsEngine graphicsEngine) - : this(resourceManager, graphicsEngine.ResourceAllocator, graphicsEngine.ResourceDatabase, graphicsEngine.PipelineLibrary, graphicsEngine.ShaderCompiler) - { - } - /// /// Resets the render graph for a new frame. /// diff --git a/src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraphContext.cs b/src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraphContext.cs index b0358d9..9e750d1 100644 --- a/src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraphContext.cs +++ b/src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraphContext.cs @@ -1,6 +1,7 @@ using Ghost.Core; using Ghost.Graphics.Core; using Ghost.Graphics.RHI; +using Ghost.Graphics.Services; using Misaki.HighPerformance.Mathematics; namespace Ghost.Graphics.RenderGraphModule; @@ -47,9 +48,9 @@ public interface IUnsafeRenderContext : IRasterRenderContext, IComputeRenderCont internal sealed class RenderGraphContext : IUnsafeRenderContext { private readonly ResourceManager _resourceManager; + private readonly ShaderLibrary _shaderLibrary; private readonly IResourceDatabase _resourceDatabase; private readonly IPipelineLibrary _pipelineLibrary; - private readonly IShaderCompiler _shaderCompiler; private readonly RenderGraphResourceRegistry _resources; private ICommandBuffer _commandBuffer; @@ -77,12 +78,12 @@ internal sealed class RenderGraphContext : IUnsafeRenderContext get; set; } - internal RenderGraphContext(ResourceManager resourceManager, IResourceDatabase resourceDatabase, IPipelineLibrary pipelineLibrary, IShaderCompiler shaderCompiler, RenderGraphResourceRegistry resources) + internal RenderGraphContext(ResourceManager resourceManager, ShaderLibrary shaderLibrary, IResourceDatabase resourceDatabase, IPipelineLibrary pipelineLibrary, RenderGraphResourceRegistry resources) { _resourceManager = resourceManager; + _shaderLibrary = shaderLibrary; _resourceDatabase = resourceDatabase; _pipelineLibrary = pipelineLibrary; - _shaderCompiler = shaderCompiler; _resources = resources; _commandBuffer = null!; @@ -171,7 +172,7 @@ internal sealed class RenderGraphContext : IUnsafeRenderContext if (!_pipelineLibrary.HasPipelineStateObject(pipelineKey)) { - var compiledCacheResult = _shaderCompiler.GetCompiledCache(shaderVariantKey); + var compiledCacheResult = _shaderLibrary.GetCache(shaderVariantKey); if (compiledCacheResult.IsFailure) { throw new InvalidOperationException("Failed to load compiled shader cache for pipeline state object creation."); diff --git a/src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraphExecutor.cs b/src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraphExecutor.cs index 439ddba..a8bb421 100644 --- a/src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraphExecutor.cs +++ b/src/Runtime/Ghost.Graphics/RenderGraphModule/RenderGraphExecutor.cs @@ -1,8 +1,7 @@ using Ghost.Core; using Ghost.Graphics.RHI; -using Misaki.HighPerformance.Mathematics; +using Ghost.Graphics.Services; using System.Diagnostics; -using TerraFX.Interop.Windows; namespace Ghost.Graphics.RenderGraphModule; diff --git a/src/Runtime/Ghost.Graphics/RenderSystem.cs b/src/Runtime/Ghost.Graphics/RenderSystem.cs index 8461996..45fbab3 100644 --- a/src/Runtime/Ghost.Graphics/RenderSystem.cs +++ b/src/Runtime/Ghost.Graphics/RenderSystem.cs @@ -30,6 +30,16 @@ internal readonly struct RenderSystemDesc { get; init; } + + public required string ShaderCacheDirectory + { + get; init; + } + + public IShaderCompilationBridge? ShaderCompilationBridge + { + get; init; + } } /// @@ -79,6 +89,7 @@ public class RenderSystem : IDisposable private readonly IGraphicsEngine _graphicsEngine; private readonly ResourceManager _resourceManager; private readonly SwapChainManager _swapChainManager; + private readonly ShaderLibrary _shaderLibrary; private readonly FrameResource[] _frameResources; private readonly Thread _renderThread; @@ -95,10 +106,12 @@ public class RenderSystem : IDisposable private bool _isRunning; private bool _disposed; - internal SwapChainManager SwapChainManager => _swapChainManager; + internal ShaderLibrary ShaderLibrary => _shaderLibrary; public IGraphicsEngine GraphicsEngine => _graphicsEngine; public ResourceManager ResourceManager => _resourceManager; + public SwapChainManager SwapChainManager => _swapChainManager; + public bool IsRunning => _isRunning; public ulong CPUFenceValue => _cpuFenceValue; @@ -120,7 +133,7 @@ public class RenderSystem : IDisposable } _renderPipeline?.Dispose(); - for (int i = 0; i < _frameResources.Length; i++) + for (var i = 0; i < _frameResources.Length; i++) { _frameResources[i].RenderPayload?.Dispose(); } @@ -165,6 +178,7 @@ public class RenderSystem : IDisposable _resourceManager = new ResourceManager(_graphicsEngine.Device, _graphicsEngine.ResourceAllocator, _graphicsEngine.ResourceDatabase); _swapChainManager = new SwapChainManager(_graphicsEngine); + _shaderLibrary = new ShaderLibrary(desc.ShaderCompilationBridge, desc.ShaderCacheDirectory); // Create frame resources for synchronization _frameResources = new FrameResource[desc.FrameBufferCount]; diff --git a/src/Runtime/Ghost.Graphics/Services/ResourceManager.Pool.cs b/src/Runtime/Ghost.Graphics/Services/ResourceManager.Pool.cs index fc9e584..15669ec 100644 --- a/src/Runtime/Ghost.Graphics/Services/ResourceManager.Pool.cs +++ b/src/Runtime/Ghost.Graphics/Services/ResourceManager.Pool.cs @@ -4,7 +4,7 @@ using Misaki.HighPerformance.LowLevel.Buffer; using Misaki.HighPerformance.LowLevel.Collections; using System.Diagnostics; -namespace Ghost.Graphics; +namespace Ghost.Graphics.Services; public partial class ResourceManager { diff --git a/src/Runtime/Ghost.Graphics/Services/ResourceManager.cs b/src/Runtime/Ghost.Graphics/Services/ResourceManager.cs index f04385a..5185e65 100644 --- a/src/Runtime/Ghost.Graphics/Services/ResourceManager.cs +++ b/src/Runtime/Ghost.Graphics/Services/ResourceManager.cs @@ -6,7 +6,7 @@ using Misaki.HighPerformance.LowLevel.Buffer; using Misaki.HighPerformance.LowLevel.Collections; using System.Diagnostics; -namespace Ghost.Graphics; +namespace Ghost.Graphics.Services; public sealed partial class ResourceManager : IDisposable { @@ -182,7 +182,7 @@ public sealed partial class ResourceManager : IDisposable /// /// An representing the newly created shader. /// The viewGroup containing the shader's properties and passes. - public Handle CreateGraphicsShader(GraphicsShaderDescriptor descriptor, ReadOnlySpan compiledResults) + public Handle CreateGraphicsShader(GraphicsShaderDescriptor descriptor) { Debug.Assert(!_disposed); @@ -194,7 +194,7 @@ public sealed partial class ResourceManager : IDisposable try { - var shader = new Shader(descriptor, compiledResults); + var shader = new Shader(descriptor); var id = _shaders.Add(shader, out var generation); return new Handle(id, generation); @@ -205,7 +205,7 @@ public sealed partial class ResourceManager : IDisposable } } - public Handle CreateComputeShader(ComputeShaderDescriptor descriptor, ReadOnlySpan compiledResults) + public Handle CreateComputeShader(ComputeShaderDescriptor descriptor) { Debug.Assert(!_disposed); var spinner = new SpinWait(); @@ -216,7 +216,7 @@ public sealed partial class ResourceManager : IDisposable try { - var computeShader = new ComputeShader(descriptor, compiledResults); + var computeShader = new ComputeShader(descriptor); var id = _computeShaders.Add(computeShader, out var generation); return new Handle(id, generation); } diff --git a/src/Runtime/Ghost.Graphics/Services/ResourceUploadBatch.cs b/src/Runtime/Ghost.Graphics/Services/ResourceUploadBatch.cs index a1fd1fc..0c76634 100644 --- a/src/Runtime/Ghost.Graphics/Services/ResourceUploadBatch.cs +++ b/src/Runtime/Ghost.Graphics/Services/ResourceUploadBatch.cs @@ -1,5 +1,6 @@ using Ghost.Core; using Ghost.Graphics.RHI; +using Ghost.Graphics.Services; namespace Ghost.Graphics.Services; @@ -44,4 +45,4 @@ public class ResourceUploadBatch { return _device.CopyQueue.WaitAsync(); } -} \ No newline at end of file +} diff --git a/src/Runtime/Ghost.Graphics/Services/ShaderLibrary.cs b/src/Runtime/Ghost.Graphics/Services/ShaderLibrary.cs index ea9d0ab..88638fb 100644 --- a/src/Runtime/Ghost.Graphics/Services/ShaderLibrary.cs +++ b/src/Runtime/Ghost.Graphics/Services/ShaderLibrary.cs @@ -1,11 +1,105 @@ +using Ghost.Core; +using Misaki.HighPerformance.LowLevel.Buffer; +using Misaki.HighPerformance.LowLevel.Collections; +using System.Runtime.CompilerServices; + namespace Ghost.Graphics.Services; -public class ShaderLibrary : IDisposable +internal class ShaderLibrary : IDisposable { + private struct CacheEntry: IDisposable + { + public UnsafeArray> byteCode; + public void Insert(int index, ReadOnlySpan data) + { + if (index >= byteCode.Length) + { + var newByteCode = new UnsafeArray>(index + 1, Allocator.Persistent); + for (int i = 0; i < byteCode.Length; i++) + { + newByteCode[i] = byteCode[i]; + } + + byteCode.Dispose(); + byteCode = newByteCode; + } + + var byteData = new UnsafeArray(data.Length, Allocator.Persistent); + byteData.CopyFrom(data); + byteCode[index] = byteData; + } + + public readonly void Dispose() + { + for (int i = 0; i < byteCode.Length; i++) + { + byteCode[i].Dispose(); + } + } + } + + private UnsafeHashMap _inMemoryCache; + + private readonly string _cacheDirectory; + private readonly IShaderCompilationBridge? _shaderCompilationBridge; + + internal ShaderLibrary(IShaderCompilationBridge? shaderCompilationBridge, string cacheDirectory) + { + _inMemoryCache = new UnsafeHashMap(16, Allocator.Persistent); + + _cacheDirectory = cacheDirectory; + _shaderCompilationBridge = shaderCompilationBridge; + } + + private string GetShaderCacheFilePath(ulong hash) + { + var hashString = hash.ToString("X16"); // Convert to hexadecimal string + var folderName = hashString[..2]; // Use the first two characters as the folder name + var folderPath = Path.Combine(_cacheDirectory, folderName); + + if (!Directory.Exists(folderPath)) + { + Directory.CreateDirectory(folderPath); + } + + return Path.Combine(folderPath, $"shader_cache_{hashString}.bin"); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void CacheCompiledResult(ulong id, int index, ReadOnlySpan byteCode) + { + var data = new UnsafeArray(byteCode.Length, Allocator.Persistent); + data.CopyFrom(byteCode); + + ref var entry = ref _inMemoryCache.GetValueRefOrAddDefault(id, out var exists); + entry.Insert(index, byteCode); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Result, Error> GetCache(ulong id, int index, AllocationHandle allocationHandle) + { + if (_inMemoryCache.TryGetValue(id, out var entry)) + { + if (index < entry.byteCode.Length) + { + var byteCode = entry.byteCode[index]; + var result = new UnsafeArray(byteCode.Length, allocationHandle); + result.CopyFrom(byteCode); + return result; + } + } + + return Error.NotFound; + } public void Dispose() { - throw new NotImplementedException(); + foreach (var kvp in _inMemoryCache) + { + kvp.Value.Dispose(); + } + + GC.SuppressFinalize(this); } } diff --git a/src/Runtime/Ghost.Graphics/Services/SwapChainManager.cs b/src/Runtime/Ghost.Graphics/Services/SwapChainManager.cs index d1becce..e67696f 100644 --- a/src/Runtime/Ghost.Graphics/Services/SwapChainManager.cs +++ b/src/Runtime/Ghost.Graphics/Services/SwapChainManager.cs @@ -50,7 +50,7 @@ internal sealed class SwapChainRecord } } -internal class SwapChainManager : IDisposable +public class SwapChainManager : IDisposable { public const int MAX_SWAP_CHAINS = 8; private readonly IGraphicsEngine _graphicsEngine; diff --git a/src/Runtime/Ghost.Graphics/Utilities/RenderingUtility.cs b/src/Runtime/Ghost.Graphics/Utilities/RenderingUtility.cs index ce8b874..bb8bab2 100644 --- a/src/Runtime/Ghost.Graphics/Utilities/RenderingUtility.cs +++ b/src/Runtime/Ghost.Graphics/Utilities/RenderingUtility.cs @@ -2,6 +2,7 @@ using Ghost.Core; using Ghost.Graphics.RHI; using Misaki.HighPerformance.LowLevel.Utilities; using System.Diagnostics; +using Ghost.Graphics.Services; namespace Ghost.Graphics.Utilities; diff --git a/src/Test/Ghost.Graphics.Test/RenderPipeline/TestRenderPipeline.cs b/src/Test/Ghost.Graphics.Test/RenderPipeline/TestRenderPipeline.cs index a4bdf02..e0f1e94 100644 --- a/src/Test/Ghost.Graphics.Test/RenderPipeline/TestRenderPipeline.cs +++ b/src/Test/Ghost.Graphics.Test/RenderPipeline/TestRenderPipeline.cs @@ -56,7 +56,7 @@ public unsafe partial class TestRenderPipeline : IRenderPipeline ref readonly var pass = ref shaderDescriptor.passes[0]; var emptyKeywords = new LocalKeywordSet(); var compiled = renderSystem.GraphicsEngine.ShaderCompiler.CompilePass(in pass, in config, in emptyKeywords).GetValueOrThrow(); - + _meshletShader = renderSystem.ResourceManager.CreateGraphicsShader(shaderDescriptor, [compiled]); _meshletMaterial = renderSystem.ResourceManager.CreateMaterial(_meshletShader); }