Misaki/GhostEngine
2
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor shader system: arrays, keywords, property syntax

Browse Source 
Major refactor of shader compiler and related systems:
- Switch ShaderDescriptor/PassDescriptor to arrays; remove IPassDescriptor
- Rewrite keywords block parser/semantic analysis for flexible syntax
- Change property initializers to brace syntax `{ ... }`
- Simplify TokenStream API (remove ref index params)
- Make GetBindlessIndex return uint (~0u for not found)
- Update shader compilation and variant logic for new descriptors
- Update test shader syntax to match new property/keyword formats
- Add AGENTS.md agent development guide
- Add Antlr4 dependency to Ghost.DSL
- Miscellaneous code style and error handling improvements
This commit is contained in:
Misaki
2026-01-10 18:36:18 +09:00
parent 6a041f75ba
commit d71bdb3fc9
18 changed files with 548 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
Ghost.DSL/ShaderCompiler/DSLShaderCompiler.cs
View File

@@ -145,8 +145,13 @@ internal static class DSLShaderCompiler
};
}
private static uint CalculateCBufferSize(List<PropertyDescriptor> properties)
private static uint CalculateCBufferSize(ReadOnlySpan<PropertyDescriptor> properties)
{
if (properties.IsEmpty)
{
return 0;
}
var currentOffset = 0u;
foreach (var prop in properties)
@@ -180,7 +185,7 @@ internal static class DSLShaderCompiler
name = p.name,
type = p.type,
defaultValue = p.defaultValue
}).ToList();
}).ToArray();
var shaderLocalProperties = semantics.properties?
.Where(p => p.scope == PropertyScope.Local)
@@ -189,41 +194,37 @@ internal static class DSLShaderCompiler
name = p.name,
type = p.type,
defaultValue = p.defaultValue
}).ToList();
}).ToArray();
if (shaderGlobalProperties != null)
{
descriptor.globalProperties ??= new List<PropertyDescriptor>();
descriptor.globalProperties.AddRange(shaderGlobalProperties);
}
if (shaderLocalProperties != null)
{
descriptor.properties ??= new List<PropertyDescriptor>();
descriptor.properties.AddRange(shaderLocalProperties);
descriptor.cbufferSize = CalculateCBufferSize(descriptor.properties);
}
descriptor.globalProperties = shaderGlobalProperties ?? Array.Empty<PropertyDescriptor>();
descriptor.properties = shaderLocalProperties ?? Array.Empty<PropertyDescriptor>();
descriptor.cbufferSize = CalculateCBufferSize(descriptor.properties);
if (semantics.passes != null)
{
foreach (var pass in semantics.passes)
descriptor.passes = new PassDescriptor[semantics.passes.Count];
for (int i = 0; i < semantics.passes.Count; i++)
{
var pass = semantics.passes[i];
var localPipeline = MeragePipeline(pass.localPipeline, PipelineState.Default);
var fullPass = new PassDescriptor
descriptor.passes[i] = new PassDescriptor
{
uniqueIdentifier = GetPassUniqueId(semantics, pass),
identifier = GetPassUniqueId(semantics, pass),
name = pass.name,
taskShader = pass.taskShader,
meshShader = pass.meshShader,
pixelShader = pass.pixelShader,
localPipeline = localPipeline,
defines = pass.defines,
keywords = pass.keywords,
defines = pass.defines?.ToArray() ?? Array.Empty<string>(),
includes = pass.includes?.ToArray() ?? Array.Empty<string>(),
keywords = pass.keywords?.ToArray() ?? Array.Empty<KeywordsGroup>()
};
descriptor.passes.Add(fullPass);
}
}
else
{
descriptor.passes = Array.Empty<PassDescriptor>();
}
return descriptor;
}
@@ -237,6 +238,11 @@ internal static class DSLShaderCompiler
var lexer = new Lexer(source);
var stream = new TokenStream(lexer.Tokenize());
var shaderInfo = ParseShaders(stream);
if (shaderInfo.Count == 0)
{
return Result.Failure("No shader found in the provided file.");
}
var model = SemanticAnalysis(shaderInfo[0], out var errors);
if (errors.Count != 0 || model == null)
@@ -263,14 +269,21 @@ internal static class DSLShaderCompiler
return Result.Failure("Failed to generate pass files: " + generatedResult.Message);
}
foreach (var pass in desc.passes)
foreach (ref var pass in desc.passes.AsSpan())
{
if (pass is PassDescriptor fullPass)
if (pass.includes == null)
{
fullPass.includes ??= new List<string>();
fullPass.includes.Add(globalPropResult.Value);
fullPass.includes.Add(generatedResult.Value);
pass.includes = new string[2];
}
else
{
Array.Resize(ref pass.includes, pass.includes.Length + 2);
// Shift existing includes to make room for the two new includes at the front.
pass.includes.AsSpan(0, pass.includes.Length - 2).CopyTo(pass.includes.AsSpan(2));
}
pass.includes[0] = globalPropResult.Value;
pass.includes[1] = generatedResult.Value;
}
return desc;
@@ -360,7 +373,7 @@ struct PerMaterialData
return outputFilePath;
}
public static Result<string> GenerateGlobalProperties(List<PropertyDescriptor> globalProperties, string targetDirectory)
public static Result<string> GenerateGlobalProperties(ReadOnlySpan<PropertyDescriptor> globalProperties, string targetDirectory)
{
if (!Directory.Exists(targetDirectory))
{

1
Ghost.DSL/ShaderCompiler/DSLShaderSemantics.cs
View File

@@ -32,6 +32,7 @@ internal class PassSemantic
public ShaderEntryPoint meshShader;
public ShaderEntryPoint pixelShader;
public List<string>? defines;
public List<string>? includes;
public List<KeywordsGroup>? keywords;
public PipelineSemantic? localPipeline;
}

4
Ghost.DSL/ShaderCompiler/DSLShaderSyntax.cs
View File

@@ -11,7 +11,7 @@ internal struct PropertyDeclaration
public Token scope;
public Token type;
public Token name;
public FunctionCallDeclaration? propertyConstructor;
public List<Token>? propertyInitializer;
}
internal struct ValueDeclaration
@@ -44,7 +44,7 @@ internal class PassSyntax
public HlslDeclaration? hlsl;
public List<Token>? defines;
public List<Token>? includes;
public List<FunctionCallDeclaration>? keywords;
public List<List<Token>>? keywords;
public List<FunctionCallDeclaration>? functionCalls;
}

78
Ghost.DSL/ShaderCompiler/Parser/KeywordsBlock.cs
View File

@@ -2,26 +2,42 @@ using Ghost.Core.Graphics;
namespace Ghost.DSL.ShaderCompiler.Parser;
internal class KeywordsBlock : IBlockParser<List<FunctionCallDeclaration>, List<KeywordsGroup>>
internal class KeywordsBlock : IBlockParser<List<List<Token>>, List<KeywordsGroup>>
{
public static bool ShouldEnter(Token token)
{
return token.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.KEYWORDS);
}
public static List<FunctionCallDeclaration> Parse(TokenStreamSlice stream)
public static List<List<Token>> Parse(TokenStreamSlice stream)
{
stream.Expect(TokenType.Keyword);
stream.Expect(TokenType.LBrace);
var keywords = new List<FunctionCallDeclaration>();
var keywords = new List<List<Token>>();
var bodyStream = stream.Slice(stream.Remaining - 1);
while (bodyStream.HasMore)
{
var keywordToken = bodyStream.Expect(TokenType.Identifier);
var args = ParseUtility.ParseFunctionArguments(ref bodyStream, TokenType.Identifier);
keywords.Add(new FunctionCallDeclaration { name = keywordToken, arguments = args });
var keys = new List<Token>();
while (!bodyStream.Match(TokenType.Semicolon))
{
var expectType = TokenType.Identifier;
if (keys.Count == 0)
{
expectType |= TokenType.Keyword;
}
var argument = bodyStream.Expect(expectType);
keys.Add(argument);
if (bodyStream.Match(TokenType.Comma))
{
bodyStream.Consume();
}
}
keywords.Add(keys);
bodyStream.Expect(TokenType.Semicolon);
}
@@ -30,7 +46,7 @@ internal class KeywordsBlock : IBlockParser<List<FunctionCallDeclaration>, List<
return keywords;
}
public static List<KeywordsGroup>? SemanticAnalysis(List<FunctionCallDeclaration>? syntax, List<DSLShaderError> errors)
public static List<KeywordsGroup>? SemanticAnalysis(List<List<Token>>? syntax, List<DSLShaderError> errors)
{
if (syntax == null)
{
@@ -38,42 +54,42 @@ internal class KeywordsBlock : IBlockParser<List<FunctionCallDeclaration>, List<
}
var keywords = new List<KeywordsGroup>(syntax.Count);
foreach (var keyword in syntax)
foreach (var keys in syntax)
{
if (keyword.arguments == null || keyword.arguments.Count == 0)
if (keys.Count == 0)
{
errors.Add(new DSLShaderError
{
message = $"Function '{keyword.name.lexeme}' must have at least one argument.",
line = keyword.name.line,
column = keyword.name.column
});
continue;
}
var group = new KeywordsGroup();
switch (keyword.name.lexeme)
group.space = keys[0].lexeme switch
{
case TokenLexicon.KnownFunctions.LOCAL:
group.space = KeywordSpace.Local;
break;
case TokenLexicon.KnownFunctions.GLOBAL:
group.space = KeywordSpace.Global;
break;
default:
TokenLexicon.KnownFunctions.LOCAL => KeywordSpace.Local,
TokenLexicon.KnownFunctions.GLOBAL => KeywordSpace.Global,
_ => KeywordSpace.Local
};
for (var i = 0; i < keys.Count; i++)
{
var token = keys[i];
if (i == 0 && token.type == TokenType.Keyword)
{
continue;
}
if (token.type != TokenType.Identifier)
{
errors.Add(new DSLShaderError
{
message = $"Unknown function name '{keyword.name.lexeme}'.",
line = keyword.name.line,
column = keyword.name.column
message = $"Invalid keyword '{token.lexeme}' in keywords block.",
line = token.line,
column = token.column
});
continue;
}
}
foreach (var arg in keyword.arguments)
{
group.keywords ??= new List<string>(keyword.arguments.Count);
group.keywords.Add(arg.lexeme);
group.keywords ??= new List<string>(keys.Count);
group.keywords.Add(token.lexeme);
}
keywords.Add(group);

1
Ghost.DSL/ShaderCompiler/Parser/PassBlock.cs
View File

@@ -68,6 +68,7 @@ internal class PassBlock : IBlockParser<PassSyntax, PassSemantic>
{
name = syntax.name.lexeme,
defines = DefinesBlock.SemanticAnalysis(syntax.defines, errors),
includes = IncludesBlock.SemanticAnalysis(syntax.includes, errors),
keywords = KeywordsBlock.SemanticAnalysis(syntax.keywords, errors),
localPipeline = PipelineBlock.SemanticAnalysis(syntax.localPipeline, errors),
};

90
Ghost.DSL/ShaderCompiler/Parser/PropertiesBlock.cs
View File

@@ -218,14 +218,18 @@ internal class PropertiesBlock : IBlockParser<PropertiesSyntax, List<PropertySem
{
case TokenType.Equals:
{
var constructorTypeToken = bodyStream.Expect(TokenType.Identifier);
var args = ParseUtility.ParseFunctionArguments(ref bodyStream, TokenType.Identifier | TokenType.Number);
shaderProperty.propertyConstructor = new FunctionCallDeclaration
bodyStream.Expect(TokenType.LBrace);
while (!bodyStream.Match(TokenType.RBrace))
{
name = constructorTypeToken,
arguments = args
};
var token = bodyStream.Consume();
if (!token.Match(TokenType.Comma))
{
shaderProperty.propertyInitializer ??= new List<Token>();
shaderProperty.propertyInitializer.Add(token);
}
}
bodyStream.Expect(TokenType.RBrace);
bodyStream.Expect(TokenType.Semicolon);
break;
@@ -282,9 +286,9 @@ internal class PropertiesBlock : IBlockParser<PropertiesSyntax, List<PropertySem
continue;
}
if (property.propertyConstructor != null)
if (property.propertyInitializer != null)
{
flowControl = ValidatePropertyConstructor(errors, property, model);
flowControl = ValidatePropertyInitializer(errors, property, model);
if (!flowControl)
{
continue;
@@ -346,14 +350,14 @@ internal class PropertiesBlock : IBlockParser<PropertiesSyntax, List<PropertySem
return true;
}
private static bool ValidatePropertyConstructor(List<DSLShaderError> errors, PropertyDeclaration property, PropertySemantic model)
private static bool ValidatePropertyInitializer(List<DSLShaderError> errors, PropertyDeclaration property, PropertySemantic model)
{
var constructor = property.propertyConstructor;
if (!constructor.HasValue)
var initializer = property.propertyInitializer;
if (initializer == null)
{
errors.Add(new DSLShaderError
{
message = "Shader property constructor is null.",
message = "Shader property initializer is null.",
line = property.name.line,
column = property.name.column
});
@@ -361,63 +365,25 @@ internal class PropertiesBlock : IBlockParser<PropertiesSyntax, List<PropertySem
return false;
}
var constructorValue = constructor.Value;
if (string.IsNullOrWhiteSpace(constructorValue.name.lexeme))
{
errors.Add(new DSLShaderError
{
message = "Shader property constructor has an empty name.",
line = constructorValue.name.line,
column = constructorValue.name.column
});
return false;
}
if (constructorValue.name.lexeme != property.type.lexeme)
{
errors.Add(new DSLShaderError
{
message = $"Shader property constructor name '{constructorValue.name.lexeme}' does not match property type '{property.type.lexeme}'.",
line = constructorValue.name.line,
column = constructorValue.name.column
});
return false;
}
if (!s_propTypeInfo.TryGetValue(model.type, out var info))
{
errors.Add(new DSLShaderError
{
message = $"No constructor metadata registered for property type '{model.type}'.",
line = constructorValue.name.line,
column = constructorValue.name.column
message = $"No initializer metadata registered for property type '{model.type}'.",
line = property.name.line,
column = property.name.column
});
return false;
}
// Count check
if (constructorValue.arguments == null)
if (initializer.Count != info.ArgCount)
{
errors.Add(new DSLShaderError
{
message = "Shader property constructor arguments are null.",
line = constructorValue.name.line,
column = constructorValue.name.column
});
return false;
}
if (constructorValue.arguments.Count != info.ArgCount)
{
errors.Add(new DSLShaderError
{
message = $"Shader property constructor for type '{property.type.lexeme}' expects {info.ArgCount} argument(s), but got {constructorValue.arguments.Count}.",
line = constructorValue.name.line,
column = constructorValue.name.column
message = $"Shader property constructor for type '{property.type.lexeme}' expects {info.ArgCount} argument(s), but got {initializer.Count}.",
line = property.name.line,
column = property.name.column
});
return false;
@@ -425,9 +391,9 @@ internal class PropertiesBlock : IBlockParser<PropertiesSyntax, List<PropertySem
// Type check (uniform requirement for all args)
var hasError = false;
for (var i = 0; i < constructorValue.arguments.Count; i++)
for (var i = 0; i < initializer.Count; i++)
{
var arg = constructorValue.arguments[i];
var arg = initializer[i];
if (!arg.Match(info.ArgTokenType))
{
errors.Add(new DSLShaderError
@@ -451,15 +417,15 @@ internal class PropertiesBlock : IBlockParser<PropertiesSyntax, List<PropertySem
{
try
{
model.defaultValue = info.Builder(constructorValue.arguments, errors);
model.defaultValue = info.Builder(initializer, errors);
}
catch (Exception ex)
{
errors.Add(new DSLShaderError
{
message = $"Failed to construct default value for property '{property.name.lexeme}': {ex.Message}",
line = constructorValue.name.line,
column = constructorValue.name.column
line = property.name.line,
column = property.name.column
});
return false;

61
Ghost.DSL/ShaderCompiler/TokenStream.cs
View File

@@ -2,12 +2,17 @@ namespace Ghost.DSL.ShaderCompiler;
internal static class TokenStreamImple
{
public static Token Peek(ReadOnlySpan<Token> tokens, ref int index, int length)
public static Token Peek(ReadOnlySpan<Token> tokens, int index, int length)
{
return index + length < tokens.Length ? tokens[index + length] : throw new InvalidOperationException("No more tokens available");
if (index + length < tokens.Length)
{
return tokens[index + length];
}
throw new IndexOutOfRangeException();
}
public static bool TryPeek(ReadOnlySpan<Token> tokens, ref int index, int length, out Token token)
public static bool TryPeek(ReadOnlySpan<Token> tokens, int index, int length, out Token token)
{
if (index + length < tokens.Length)
{
@@ -31,29 +36,33 @@ internal static class TokenStreamImple
return false;
}
public static Token Consume(ReadOnlySpan<Token> tokens, ref int index)
public static Token Consume(ReadOnlySpan<Token> tokens, ref int index, int count = 1)
{
return index < tokens.Length ? tokens[index++] : throw new InvalidOperationException("No more tokens available");
if (index + count <= tokens.Length)
{
index += count;
return tokens[index - 1];
}
throw new IndexOutOfRangeException();
}
public static bool Match(ReadOnlySpan<Token> tokens, ref int index, TokenType type, string? lexeme)
public static bool Match(ReadOnlySpan<Token> tokens, int index, TokenType type, string? lexeme)
{
var t = Peek(tokens, ref index, 0);
var t = Peek(tokens, index, 0);
if (!t.Match(type, lexeme))
{
return false;
}
//index++;
return true;
}
public static int MatchMany(ReadOnlySpan<Token> tokens, ref int index, TokenType type, string? lexeme)
public static int MatchMany(ReadOnlySpan<Token> tokens, int index, TokenType type, string? lexeme)
{
var count = 0;
while (TryPeek(tokens, ref index, 0, out var t) && t.Match(type, lexeme))
while (TryPeek(tokens, index, 0, out var t) && t.Match(type, lexeme))
{
index++;
count++;
}
@@ -62,7 +71,7 @@ internal static class TokenStreamImple
public static Token Expect(ReadOnlySpan<Token> tokens, ref int index, TokenType type, string? lexeme)
{
if (!TryPeek(tokens, ref index, 0, out var t))
if (!TryPeek(tokens, index, 0, out var t))
{
throw new InvalidOperationException("Expected token but reached end of stream");
}
@@ -110,17 +119,17 @@ internal class TokenStream
public Token Peek(int length = 0)
{
return TokenStreamImple.Peek(_tokens, ref _index, length);
return TokenStreamImple.Peek(_tokens, _index, length);
}
public bool TryPeek(out Token token)
{
return TokenStreamImple.TryPeek(_tokens, ref _index, 0, out token);
return TokenStreamImple.TryPeek(_tokens, _index, 0, out token);
}
public bool TryPeek(int length, out Token token)
{
return TokenStreamImple.TryPeek(_tokens, ref _index, length, out token);
return TokenStreamImple.TryPeek(_tokens, _index, length, out token);
}
public bool TryConsume(out Token token)
@@ -128,19 +137,19 @@ internal class TokenStream
return TokenStreamImple.TryConsume(_tokens, ref _index, out token);
}
public Token Consume()
public Token Consume(int count = 1)
{
return TokenStreamImple.Consume(_tokens, ref _index);
return TokenStreamImple.Consume(_tokens, ref _index, count);
}
public bool Match(TokenType type, string? lexeme = null)
{
return TokenStreamImple.Match(_tokens, ref _index, type, lexeme);
return TokenStreamImple.Match(_tokens, _index, type, lexeme);
}
public int MatchMany(TokenType type, string? lexeme = null)
{
return TokenStreamImple.MatchMany(_tokens, ref _index, type, lexeme);
return TokenStreamImple.MatchMany(_tokens, _index, type, lexeme);
}
public Token Expect(TokenType type, string? lexeme = null)
@@ -224,17 +233,17 @@ internal ref struct TokenStreamSlice
public Token Peek(int length = 0)
{
return TokenStreamImple.Peek(_tokens, ref _index, length);
return TokenStreamImple.Peek(_tokens, _index, length);
}
public bool TryPeek(out Token token)
{
return TokenStreamImple.TryPeek(_tokens, ref _index, 0, out token);
return TokenStreamImple.TryPeek(_tokens, _index, 0, out token);
}
public bool TryPeek(int length, out Token token)
{
return TokenStreamImple.TryPeek(_tokens, ref _index, length, out token);
return TokenStreamImple.TryPeek(_tokens, _index, length, out token);
}
public bool TryConsume(out Token token)
@@ -242,19 +251,19 @@ internal ref struct TokenStreamSlice
return TokenStreamImple.TryConsume(_tokens, ref _index, out token);
}
public Token Consume()
public Token Consume(int count = 1)
{
return TokenStreamImple.Consume(_tokens, ref _index);
return TokenStreamImple.Consume(_tokens, ref _index, count);
}
public bool Match(TokenType type, string? lexeme = null)
{
return TokenStreamImple.Match(_tokens, ref _index, type, lexeme);
return TokenStreamImple.Match(_tokens, _index, type, lexeme);
}
public int MatchMany(TokenType type, string? lexeme = null)
{
return TokenStreamImple.MatchMany(_tokens, ref _index, type, lexeme);
return TokenStreamImple.MatchMany(_tokens, _index, type, lexeme);
}
public Token Expect(TokenType type, string? lexeme = null)