Misaki/GhostEngine
2
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor render graph & DSL; remove material system

Browse Source 
- Major optimization of Ghost.RenderGraph.Concept: pooled resources, zero-allocation hot paths, explicit queue types, and batch barrier APIs.
- Migrated Ghost.DSL shader compiler to ANTLR4-based parser; removed hand-written parser, added grammar files and semantic model conversion.
- Added CollectionPool/ListPool for pooled list management.
- Updated documentation for new architecture and performance.
- Removed Ghost.Shader.Concept (material/material system) from repo and solution.
- README.md replaced with a brief project statement.
This commit is contained in:
Misaki
2026-01-11 13:28:17 +09:00
parent d71bdb3fc9
commit 87e315a588
63 changed files with 1841 additions and 6085 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
Ghost.DSL/ShaderCompiler/DSLShaderCompiler.cs
View File

@@ -1,6 +1,6 @@
using Ghost.Core;
using Ghost.Core.Graphics;
using Ghost.DSL.ShaderCompiler.Parser;
using Ghost.DSL.ShaderParser;
using System.Text;
namespace Ghost.DSL.ShaderCompiler;
@@ -22,107 +22,6 @@ internal static class DSLShaderCompiler
private const string _GLOBAL_PROPERTY_FILE_NAME = "GlobalData.g.hlsl";
private const string _GENERATED_FILE_HEADER = "// Auto-generated shader file. Please do not edit this file directly.";
// private struct ShaderInheritance
// {
// public DSLShaderSemantics? parent;
// public List<ShaderInheritance>? children;
// }
public static List<DSLShaderSyntax> ParseShaders(TokenStream stream)
{
var shaders = new List<DSLShaderSyntax>();
while (stream.TryPeek(out var nextToken))
{
if (ShaderBlock.ShouldEnter(nextToken))
{
var shader = ShaderBlock.Parse(stream.SliceNextBlock());
shaders.Add(shader);
}
else if (nextToken.Match(TokenType.EndOfFile))
{
stream.Consume();
}
else
{
throw new Exception($"Unexpected token '{nextToken.lexeme}' at top level. Expected 'shader' declaration.");
}
}
return shaders;
}
public static DSLShaderSemantics? SemanticAnalysis(DSLShaderSyntax syntax, out List<DSLShaderError> errors)
{
errors = new List<DSLShaderError>();
if (string.IsNullOrWhiteSpace(syntax.name.lexeme))
{
errors.Add(new DSLShaderError
{
message = "Shader name cannot be empty.",
line = syntax.name.line,
column = syntax.name.column
});
return null;
}
var shaderModel = ShaderBlock.SemanticAnalysis(syntax, errors);
return shaderModel;
}
private static List<DSLShaderSemantics>? TopologicalSort(ReadOnlySpan<DSLShaderSemantics> semantics)
{
var inDegrees = new Dictionary<string, int>();
var childrenMap = new Dictionary<string, List<string>>();
var semanticsMap = new Dictionary<string, DSLShaderSemantics>();
foreach (var s in semantics)
{
inDegrees[s.name] = 0;
childrenMap[s.name] = new List<string>();
semanticsMap[s.name] = s;
}
foreach (var s in semantics)
{
if (!string.IsNullOrEmpty(s.fallback) && semanticsMap.ContainsKey(s.fallback))
{
childrenMap[s.fallback].Add(s.name);
inDegrees[s.name]++;
}
}
var queue = new Queue<DSLShaderSemantics>();
foreach (var s in semantics)
{
if (inDegrees[s.name] == 0)
{
queue.Enqueue(s);
}
}
var sortedList = new List<DSLShaderSemantics>();
while (queue.Count > 0)
{
var current = queue.Dequeue();
sortedList.Add(current);
foreach (var childName in childrenMap[current.name])
{
inDegrees[childName]--;
if (inDegrees[childName] == 0)
{
queue.Enqueue(semanticsMap[childName]);
}
}
}
// If there's a cycle, the graph will not be fully traversed.
return sortedList.Count == semantics.Length ? sortedList : null;
}
private static string GetPassUniqueId(DSLShaderSemantics shader, PassSemantic pass)
{
return $"{shader.name}_{pass.name}";
@@ -175,7 +74,8 @@ internal static class DSLShaderCompiler
{
var descriptor = new ShaderDescriptor
{
name = semantics.name
name = semantics.name,
hlsl = semantics.hlsl
};
var shaderGlobalProperties = semantics.properties?
@@ -217,7 +117,8 @@ internal static class DSLShaderCompiler
localPipeline = localPipeline,
defines = pass.defines?.ToArray() ?? Array.Empty<string>(),
includes = pass.includes?.ToArray() ?? Array.Empty<string>(),
keywords = pass.keywords?.ToArray() ?? Array.Empty<KeywordsGroup>()
keywords = pass.keywords?.ToArray() ?? Array.Empty<KeywordsGroup>(),
hlsl = pass.hlsl
};
}
}
@@ -235,15 +136,27 @@ internal static class DSLShaderCompiler
{
var source = File.ReadAllText(shaderPath);
var lexer = new Lexer(source);
var stream = new TokenStream(lexer.Tokenize());
var shaderInfo = ParseShaders(stream);
if (shaderInfo.Count == 0)
// Use ANTLR4 parser
var shaderModels = AntlrShaderCompiler.ParseShaders(source, out var parseErrors);
if (parseErrors.Count != 0)
{
var errorMessages = new StringBuilder();
foreach (var error in parseErrors)
{
errorMessages.AppendLine(error.ToString());
}
return Result.Failure("Failed to parse shader due to errors:\n" + errorMessages.ToString());
}
if (shaderModels.Count == 0)
{
return Result.Failure("No shader found in the provided file.");
}
var model = SemanticAnalysis(shaderInfo[0], out var errors);
// Convert to semantics
var model = AntlrShaderCompiler.ConvertToSemantics(shaderModels[0], out var errors);
if (errors.Count != 0 || model == null)
{

11
Ghost.DSL/ShaderCompiler/DSLShaderSemantics.cs
View File

@@ -8,7 +8,7 @@ public enum PropertyScope
Local,
}
internal class PropertySemantic
public class PropertySemantic
{
public PropertyScope scope;
public ShaderPropertyType type;
@@ -16,7 +16,7 @@ internal class PropertySemantic
public object? defaultValue;
}
internal class PipelineSemantic
public class PipelineSemantic
{
public ZTest? zTest;
public ZWrite? zWrite;
@@ -25,22 +25,23 @@ internal class PipelineSemantic
public ColorWriteMask? colorMask;
}
internal class PassSemantic
public class PassSemantic
{
public string name = string.Empty;
public ShaderEntryPoint taskShader;
public ShaderEntryPoint meshShader;
public ShaderEntryPoint pixelShader;
public string? hlsl;
public List<string>? defines;
public List<string>? includes;
public List<KeywordsGroup>? keywords;
public PipelineSemantic? localPipeline;
}
internal class DSLShaderSemantics
public class DSLShaderSemantics
{
public string name = string.Empty;
public string fallback = string.Empty;
public string? hlsl;
public List<PropertySemantic>? properties;
public PipelineSemantic? pipeline;
public List<PassSemantic>? passes;

58
Ghost.DSL/ShaderCompiler/DSLShaderSyntax.cs
View File

@@ -1,58 +0,0 @@
namespace Ghost.DSL.ShaderCompiler;
internal struct FunctionCallDeclaration
{
public Token name;
public List<Token>? arguments;
}
internal struct PropertyDeclaration
{
public Token scope;
public Token type;
public Token name;
public List<Token>? propertyInitializer;
}
internal struct ValueDeclaration
{
public Token name;
public Token value;
}
internal struct HlslDeclaration
{
public List<Token>? tokens;
}
internal class PropertiesSyntax
{
public List<PropertyDeclaration>? properties;
public List<FunctionCallDeclaration>? functionCalls;
}
internal class PipelineSyntax
{
public List<ValueDeclaration>? values;
public List<FunctionCallDeclaration>? functionCalls;
}
internal class PassSyntax
{
public Token name;
public PipelineSyntax? localPipeline;
public HlslDeclaration? hlsl;
public List<Token>? defines;
public List<Token>? includes;
public List<List<Token>>? keywords;
public List<FunctionCallDeclaration>? functionCalls;
}
internal class DSLShaderSyntax
{
public Token name;
public PropertiesSyntax? properties;
public PipelineSyntax? pipeline;
public List<PassSyntax>? passes;
public List<FunctionCallDeclaration>? functionCalls;
}

310
Ghost.DSL/ShaderCompiler/Lexer.cs
View File

@@ -1,310 +0,0 @@
namespace Ghost.DSL.ShaderCompiler;
public class Lexer
{
private readonly string _source;
private int _pos = 0;
private int _line = 1; // Lines typically start at 1
private int _column = 1; // Columns typically start at 1
public Lexer(string source)
{
_source = source;
}
public IEnumerable<Token> Tokenize()
{
while (!IsAtEnd())
{
var token = ScanToken();
if (token != Token.Null)
{
yield return token;
}
}
yield return new Token(TokenType.EndOfFile, string.Empty, _line, _column);
}
#region Core Scanning Logic
private Token ScanToken()
{
var c = Consume();
// Rule 1: Skip whitespace and handle line tracking
if (char.IsWhiteSpace(c))
{
HandleWhitespace(c);
return Token.Null;
}
// Rule 2: Handle comments
if (c == '/')
{
if (HandleComments())
{
return Token.Null;
}
// If not a comment, fall through to handle '/' as an operator if needed
}
// Rule 3: Handle string literals
if (c == '"')
{
return ScanStringLiteral();
}
// Rule 4: Handle numeric literals
if (char.IsDigit(c) || (c == '.' && char.IsDigit(Peek())))
{
return ScanNumericLiteral(c);
}
// Rule 5: Handle identifiers and keywords
if (char.IsLetter(c) || c == '_')
{
return ScanIdentifierOrKeyword(c);
}
// Rule 6: Handle single-character tokens (punctuation)
var punctuationToken = ScanPunctuation(c);
if (punctuationToken != Token.Null)
{
return punctuationToken;
}
// Rule 7: Skip unknown characters (could log warning in production)
return Token.Null;
}
#endregion
#region Rule Implementations
private void HandleWhitespace(char c)
{
if (c == '\n')
{
_line++;
_column = 1;
}
else if (c == '\r')
{
// Handle Windows line endings - peek for \n
if (Peek() == '\n')
{
Consume();
}
_line++;
_column = 1;
}
else
{
_column++;
}
}
private bool HandleComments()
{
var next = Peek();
if (next == '/') // Single-line comment
{
return ScanSingleLineComment();
}
else if (next == '*') // Multi-line comment
{
return ScanMultiLineComment();
}
return false; // Not a comment
}
private bool ScanSingleLineComment()
{
// Skip the second '/'
Consume();
// Consume until end of line
while (!IsAtEnd() && Peek() != '\n' && Peek() != '\r')
{
Consume();
}
return true;
}
private bool ScanMultiLineComment()
{
// Skip the '*'
Consume();
while (!IsAtEnd())
{
var c = Consume();
if (c == '\n')
{
_line++;
_column = 1;
}
else if (c == '*' && Peek() == '/')
{
Consume(); // Consume closing '/'
return true;
}
else
{
_column++;
}
}
// Unclosed comment - could throw error in production
return true;
}
private Token ScanStringLiteral()
{
var startLine = _line;
var startColumn = _column - 1; // Account for opening quote
var start = _pos;
while (!IsAtEnd() && Peek() != '"')
{
var c = Peek();
if (c == '\n')
{
_line++;
_column = 1;
}
else if (c == '\\')
{
// Handle escape sequences
Consume(); // Skip backslash
if (!IsAtEnd())
{
Consume();
}
continue;
}
Consume();
}
if (IsAtEnd())
{
// Unterminated string - could throw error in production
var unterminatedText = _source[start.._pos];
return new Token(TokenType.StringLiteral, unterminatedText, startLine, startColumn);
}
var text = _source[start.._pos];
Consume(); // Consume closing quote
return new Token(TokenType.StringLiteral, text, startLine, startColumn);
}
private Token ScanNumericLiteral(char firstChar)
{
var startColumn = _column - 1;
var start = _pos - 1; // Include the first character
var hasDot = firstChar == '.';
while (!IsAtEnd())
{
var c = Peek();
if (char.IsDigit(c))
{
Consume();
}
else if (c == '.' && !hasDot)
{
hasDot = true;
Consume();
}
else
{
break;
}
}
var number = _source[start.._pos];
return new Token(TokenType.Number, number, _line, startColumn);
}
private Token ScanIdentifierOrKeyword(char firstChar)
{
var startColumn = _column - 1;
var start = _pos - 1; // Include the first character
while (!IsAtEnd() && (char.IsLetterOrDigit(Peek()) || Peek() == '_'))
{
Consume();
}
var text = _source[start.._pos];
var tokenType = DetermineIdentifierType(text);
return new Token(tokenType, text, _line, startColumn);
}
private Token ScanPunctuation(char c)
{
var startColumn = _column - 1;
return c switch
{
'=' => new Token(TokenType.Equals, "=", _line, startColumn),
';' => new Token(TokenType.Semicolon, ";", _line, startColumn),
',' => new Token(TokenType.Comma, ",", _line, startColumn),
'{' => new Token(TokenType.LBrace, "{", _line, startColumn),
'}' => new Token(TokenType.RBrace, "}", _line, startColumn),
'(' => new Token(TokenType.LParen, "(", _line, startColumn),
')' => new Token(TokenType.RParen, ")", _line, startColumn),
_ => Token.Null // Unknown punctuation
};
}
#endregion
#region Classification Rules
private static TokenType DetermineIdentifierType(string text)
{
// Rule: Check if it's a known keyword first
if (TokenLexicon.IsKeyword(text))
{
return TokenType.Keyword;
}
// Rule: All other identifiers are treated as identifiers
// (Could extend this to handle functions, types, etc. as separate token types)
return TokenType.Identifier;
}
#endregion
#region Helper Methods
private bool IsAtEnd() => _pos >= _source.Length;
private char Consume()
{
if (IsAtEnd())
return '\0';
var c = _source[_pos];
_pos++;
_column++;
return c;
}
private char Peek() => IsAtEnd() ? '\0' : _source[_pos];
private char PeekNext() => _pos + 1 >= _source.Length ? '\0' : _source[_pos + 1];
#endregion
}

45
Ghost.DSL/ShaderCompiler/Parser/DefinesBlock.cs
View File

@@ -1,45 +0,0 @@
namespace Ghost.DSL.ShaderCompiler.Parser;
internal class DefinesBlock : IBlockParser<List<Token>, List<string>>
{
public static bool ShouldEnter(Token token)
{
return token.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.DEFINES);
}
public static List<Token> Parse(TokenStreamSlice stream)
{
stream.Expect(TokenType.Keyword);
stream.Expect(TokenType.LBrace);
var defines = new List<Token>();
var bodyStream = stream.Slice(stream.Remaining - 1);
while (bodyStream.HasMore)
{
var defineToken = bodyStream.Expect(TokenType.Identifier);
defines.Add(defineToken);
bodyStream.Expect(TokenType.Semicolon);
}
stream.Expect(TokenType.RBrace);
return defines;
}
public static List<string>? SemanticAnalysis(List<Token>? syntax, List<DSLShaderError> errors)
{
if (syntax == null)
{
return null;
}
var defines = new List<string>(syntax.Count);
foreach (var defineToken in syntax)
{
defines.Add(defineToken.lexeme);
}
return defines;
}
}

8
Ghost.DSL/ShaderCompiler/Parser/IBlockParser.cs
View File

@@ -1,8 +0,0 @@
namespace Ghost.DSL.ShaderCompiler.Parser;
internal interface IBlockParser<T, U>
{
public static abstract bool ShouldEnter(Token token);
public static abstract T? Parse(TokenStreamSlice ts);
public static abstract U? SemanticAnalysis(T? syntax, List<DSLShaderError> errors);
}

59
Ghost.DSL/ShaderCompiler/Parser/IncludesBlock.cs
View File

@@ -1,59 +0,0 @@
namespace Ghost.DSL.ShaderCompiler.Parser;
internal class IncludesBlock : IBlockParser<List<Token>, List<string>>
{
public static bool ShouldEnter(Token token)
{
return token.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.INCLUDES);
}
public static List<Token> Parse(TokenStreamSlice stream)
{
stream.Expect(TokenType.Keyword);
stream.Expect(TokenType.LBrace);
var includes = new List<Token>();
var bodyStream = stream.Slice(stream.Remaining - 1);
while (bodyStream.HasMore)
{
var includeToken = bodyStream.Expect(TokenType.StringLiteral);
includes.Add(includeToken);
bodyStream.Expect(TokenType.Semicolon);
}
stream.Expect(TokenType.RBrace);
return includes;
}
public static List<string>? SemanticAnalysis(List<Token>? syntax, List<DSLShaderError> errors)
{
if (syntax == null || syntax.Count == 0)
{
return null;
}
var includes = new List<string>(syntax.Count);
foreach (var includeToken in syntax)
{
var path = includeToken.lexeme;
if (File.Exists(path))
{
includes.Add(path);
}
else
{
errors.Add(new DSLShaderError
{
message = $"Included file '{path}' not found.",
line = includeToken.line,
column = includeToken.column
});
continue;
}
}
return includes;
}
}

100
Ghost.DSL/ShaderCompiler/Parser/KeywordsBlock.cs
View File

@@ -1,100 +0,0 @@
using Ghost.Core.Graphics;
namespace Ghost.DSL.ShaderCompiler.Parser;
internal class KeywordsBlock : IBlockParser<List<List<Token>>, List<KeywordsGroup>>
{
public static bool ShouldEnter(Token token)
{
return token.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.KEYWORDS);
}
public static List<List<Token>> Parse(TokenStreamSlice stream)
{
stream.Expect(TokenType.Keyword);
stream.Expect(TokenType.LBrace);
var keywords = new List<List<Token>>();
var bodyStream = stream.Slice(stream.Remaining - 1);
while (bodyStream.HasMore)
{
var keys = new List<Token>();
while (!bodyStream.Match(TokenType.Semicolon))
{
var expectType = TokenType.Identifier;
if (keys.Count == 0)
{
expectType |= TokenType.Keyword;
}
var argument = bodyStream.Expect(expectType);
keys.Add(argument);
if (bodyStream.Match(TokenType.Comma))
{
bodyStream.Consume();
}
}
keywords.Add(keys);
bodyStream.Expect(TokenType.Semicolon);
}
stream.Expect(TokenType.RBrace);
return keywords;
}
public static List<KeywordsGroup>? SemanticAnalysis(List<List<Token>>? syntax, List<DSLShaderError> errors)
{
if (syntax == null)
{
return null;
}
var keywords = new List<KeywordsGroup>(syntax.Count);
foreach (var keys in syntax)
{
if (keys.Count == 0)
{
continue;
}
var group = new KeywordsGroup();
group.space = keys[0].lexeme switch
{
TokenLexicon.KnownFunctions.LOCAL => KeywordSpace.Local,
TokenLexicon.KnownFunctions.GLOBAL => KeywordSpace.Global,
_ => KeywordSpace.Local
};
for (var i = 0; i < keys.Count; i++)
{
var token = keys[i];
if (i == 0 && token.type == TokenType.Keyword)
{
continue;
}
if (token.type != TokenType.Identifier)
{
errors.Add(new DSLShaderError
{
message = $"Invalid keyword '{token.lexeme}' in keywords block.",
line = token.line,
column = token.column
});
continue;
}
group.keywords ??= new List<string>(keys.Count);
group.keywords.Add(token.lexeme);
}
keywords.Add(group);
}
return keywords;
}
}

71
Ghost.DSL/ShaderCompiler/Parser/ParseUtility.cs
View File

@@ -1,71 +0,0 @@
namespace Ghost.DSL.ShaderCompiler.Parser;
internal static class ParseUtility
{
public static List<Token> ParseFunctionArguments(ref TokenStreamSlice stream, TokenType tokenType)
{
var args = new List<Token>();
stream.Expect(TokenType.LParen);
while (!stream.Peek().type.Equals(TokenType.RParen))
{
var argToken = stream.Expect(tokenType);
args.Add(argToken);
if (stream.Peek().type == TokenType.Comma)
{
stream.Consume();
}
else
{
break;
}
}
stream.Expect(TokenType.RParen);
return args;
}
public static FunctionCallDeclaration ParseFunction(ref TokenStreamSlice stream, TokenType tokenType)
{
var functionToken = stream.Expect(TokenType.Identifier);
var args = ParseFunctionArguments(ref stream, tokenType);
stream.Expect(TokenType.Semicolon);
return new FunctionCallDeclaration
{
name = functionToken,
arguments = args
};
}
public static bool TrySliceLine(ref TokenStreamSlice stream, out TokenStreamSlice lineStream)
{
var length = 0;
if (!stream.TryPeek(out var nextToken))
{
lineStream = default;
return false;
}
while (!nextToken.Match(TokenType.Semicolon) && !nextToken.Match(TokenType.RBrace))
{
length++;
if (!stream.TryPeek(length, out nextToken))
{
break;
}
}
if (length > 0)
{
lineStream = stream.Slice(length);
stream.Consume(); // Consume the semicolon
return true;
}
lineStream = default;
return false;
}
}

141
Ghost.DSL/ShaderCompiler/Parser/PassBlock.cs
View File

@@ -1,141 +0,0 @@
using Ghost.Core.Graphics;
namespace Ghost.DSL.ShaderCompiler.Parser;
// TODO: Add pass template support.
// Pass templates let user to inject their own custom code into the generated HLSL code.
// This is useful for adding custom lighting models, custom shadowing techniques, or other advanced effects without touching the core shader code.
internal class PassBlock : IBlockParser<PassSyntax, PassSemantic>
{
public static bool ShouldEnter(Token token)
{
return token.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.PASS);
}
public static PassSyntax Parse(TokenStreamSlice stream)
{
var pass = new PassSyntax();
stream.Expect(TokenType.Keyword);
pass.name = stream.Expect(TokenType.StringLiteral);
stream.Expect(TokenType.LBrace);
var bodyStream = stream.Slice(stream.Remaining - 1);
while (bodyStream.TryPeek(out var nextToken))
{
if (DefinesBlock.ShouldEnter(nextToken))
{
pass.defines = DefinesBlock.Parse(bodyStream.SliceNextBlock());
}
else if (KeywordsBlock.ShouldEnter(nextToken))
{
pass.keywords = KeywordsBlock.Parse(bodyStream.SliceNextBlock());
}
else if (PipelineBlock.ShouldEnter(nextToken))
{
pass.localPipeline = PipelineBlock.Parse(bodyStream.SliceNextBlock());
}
else if (IncludesBlock.ShouldEnter(nextToken))
{
pass.includes = IncludesBlock.Parse(bodyStream.SliceNextBlock());
}
else if (nextToken.Match(TokenType.Identifier))
{
var func = ParseUtility.ParseFunction(ref bodyStream, TokenType.StringLiteral);
pass.functionCalls ??= new();
pass.functionCalls.Add(func);
}
else
{
throw new Exception($"Unexpected token '{nextToken}' in pass body.");
}
}
stream.Expect(TokenType.RBrace);
return pass;
}
public static PassSemantic? SemanticAnalysis(PassSyntax? syntax, List<DSLShaderError> errors)
{
if (syntax == null)
{
return null;
}
var semantic = new PassSemantic
{
name = syntax.name.lexeme,
defines = DefinesBlock.SemanticAnalysis(syntax.defines, errors),
includes = IncludesBlock.SemanticAnalysis(syntax.includes, errors),
keywords = KeywordsBlock.SemanticAnalysis(syntax.keywords, errors),
localPipeline = PipelineBlock.SemanticAnalysis(syntax.localPipeline, errors),
};
if (syntax.functionCalls != null)
{
foreach (var func in syntax.functionCalls)
{
switch (func.name.lexeme)
{
case TokenLexicon.KnownFunctions.TASK_SHADER:
AnalysisShaderEntry(errors, func, ref semantic.taskShader);
break;
case TokenLexicon.KnownFunctions.MESH_SHADER:
AnalysisShaderEntry(errors, func, ref semantic.meshShader);
break;
case TokenLexicon.KnownFunctions.PIXEL_SHADER:
AnalysisShaderEntry(errors, func, ref semantic.pixelShader);
break;
default:
errors.Add(new DSLShaderError
{
message = $"Unknown function '{func.name.lexeme}' in pass {syntax.name.lexeme}.",
line = func.name.line,
column = func.name.column
});
break;
}
}
}
if (semantic.meshShader.shader == null || semantic.pixelShader.shader == null)
{
// TODO: Inheritance from base pass.
// TODO: Add mesh shader support.
errors.Add(new DSLShaderError
{
message = $"Pass {syntax.name.lexeme} must contain a mesh shader (ms) and a pixel shader (ps) declaration.",
line = syntax.name.line,
column = syntax.name.column
});
}
return semantic;
}
private static void AnalysisShaderEntry(List<DSLShaderError> errors, FunctionCallDeclaration func, ref ShaderEntryPoint shaderEntryPoint)
{
if (func.arguments?.Count != 2)
{
errors.Add(new DSLShaderError
{
message = "Shader declaration requires exactly two arguments: (shaderPath, entryPoint).",
line = func.name.line,
column = func.name.column
});
}
else
{
shaderEntryPoint = new ShaderEntryPoint
{
shader = func.arguments[0].lexeme,
entry = func.arguments[1].lexeme
};
}
}
}

143
Ghost.DSL/ShaderCompiler/Parser/PipelineBlock.cs
View File

@@ -1,143 +0,0 @@
using Ghost.Core.Graphics;
namespace Ghost.DSL.ShaderCompiler.Parser;
internal class PipelineBlock : IBlockParser<PipelineSyntax, PipelineSemantic>
{
public static bool ShouldEnter(Token token)
{
return token.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.PIPELINE);
}
public static PipelineSyntax Parse(TokenStreamSlice stream)
{
stream.Expect(TokenType.Keyword);
stream.Expect(TokenType.LBrace);
var pipeline = new PipelineSyntax();
var bodyStream = stream.Slice(stream.Remaining - 1);
while (bodyStream.HasMore)
{
var stateToken = bodyStream.Expect(TokenType.Identifier);
bodyStream.Expect(TokenType.Equals);
var valueToken = bodyStream.Expect(TokenType.Identifier | TokenType.Number);
pipeline.values ??= new();
pipeline.values.Add(new ValueDeclaration
{
name = stateToken,
value = valueToken
});
bodyStream.Expect(TokenType.Semicolon);
}
stream.Expect(TokenType.RBrace);
return pipeline;
}
public static PipelineSemantic? SemanticAnalysis(PipelineSyntax? syntax, List<DSLShaderError> errors)
{
if (syntax == null)
{
return null;
}
var semantic = new PipelineSemantic();
if (syntax.values != null)
{
foreach (var valueDecl in syntax.values)
{
switch (valueDecl.name.lexeme)
{
case TokenLexicon.KnownPipelineProperties.ZTEST:
if (Enum.TryParse<ZTest>(valueDecl.value.lexeme, true, out var zTest))
{
semantic.zTest = zTest;
}
else
{
errors.Add(new DSLShaderError
{
message = $"Invalid ZTest option: {valueDecl.value.lexeme}",
line = valueDecl.value.line,
column = valueDecl.value.column
});
}
break;
case TokenLexicon.KnownPipelineProperties.ZWRITE:
if (Enum.TryParse<ZWrite>(valueDecl.value.lexeme, true, out var zWrite))
{
semantic.zWrite = zWrite;
}
else
{
errors.Add(new DSLShaderError
{
message = $"Invalid ZWrite option: {valueDecl.value.lexeme}",
line = valueDecl.value.line,
column = valueDecl.value.column
});
}
break;
case TokenLexicon.KnownPipelineProperties.CULL:
if (Enum.TryParse<Cull>(valueDecl.value.lexeme, true, out var cull))
{
semantic.cull = cull;
}
else
{
errors.Add(new DSLShaderError
{
message = $"Invalid Cull option: {valueDecl.value.lexeme}",
line = valueDecl.value.line,
column = valueDecl.value.column
});
}
break;
case TokenLexicon.KnownPipelineProperties.BLEND:
if (Enum.TryParse<Blend>(valueDecl.value.lexeme, true, out var blend))
{
semantic.blend = blend;
}
else
{
errors.Add(new DSLShaderError
{
message = $"Invalid Blend option: {valueDecl.value.lexeme}",
line = valueDecl.value.line,
column = valueDecl.value.column
});
}
break;
case TokenLexicon.KnownPipelineProperties.COLORMASK:
if (Enum.TryParse<ColorWriteMask>(valueDecl.value.lexeme, true, out var colorMask))
{
semantic.colorMask = colorMask;
}
else
{
errors.Add(new DSLShaderError
{
message = $"Invalid Color Mask value: {valueDecl.value.lexeme}",
line = valueDecl.value.line,
column = valueDecl.value.column
});
}
break;
default:
break;
}
}
}
return semantic;
}
}

437
Ghost.DSL/ShaderCompiler/Parser/PropertiesBlock.cs
View File

@@ -1,437 +0,0 @@
using Ghost.Core.Graphics;
using Misaki.HighPerformance.Mathematics;
using System.Globalization;
namespace Ghost.DSL.ShaderCompiler.Parser;
internal class PropertiesBlock : IBlockParser<PropertiesSyntax, List<PropertySemantic>>
{
private delegate object? PropertyValueBuilder(List<Token> tokens, List<DSLShaderError> errors);
private sealed record PropTypeInfo(int ArgCount, TokenType ArgTokenType, PropertyValueBuilder? Builder);
private static readonly Dictionary<ShaderPropertyType, PropTypeInfo> s_propTypeInfo = new()
{
// Floats
[ShaderPropertyType.Float] = new(1, TokenType.Number, (syntax, errors) => ParseFloatValue(syntax[0], errors)),
[ShaderPropertyType.Float2] = new(2, TokenType.Number, (syntax, errors) => new float2(
ParseFloatValue(syntax[0], errors),
ParseFloatValue(syntax[1], errors))),
[ShaderPropertyType.Float3] = new(3, TokenType.Number, (syntax, errors) => new float3(
ParseFloatValue(syntax[0], errors),
ParseFloatValue(syntax[1], errors),
ParseFloatValue(syntax[2], errors))),
[ShaderPropertyType.Float4] = new(4, TokenType.Number, (syntax, errors) => new float4(
ParseFloatValue(syntax[0], errors),
ParseFloatValue(syntax[1], errors),
ParseFloatValue(syntax[2], errors),
ParseFloatValue(syntax[3], errors))),
// Ints
[ShaderPropertyType.Int] = new(1, TokenType.Number, (syntax, errors) => ParseIntValue(syntax[0], errors)),
[ShaderPropertyType.Int2] = new(2, TokenType.Number, (syntax, errors) => new int2(
ParseIntValue(syntax[0], errors),
ParseIntValue(syntax[1], errors))),
[ShaderPropertyType.Int3] = new(3, TokenType.Number, (syntax, errors) => new int3(
ParseIntValue(syntax[0], errors),
ParseIntValue(syntax[1], errors),
ParseIntValue(syntax[2], errors))),
[ShaderPropertyType.Int4] = new(4, TokenType.Number, (syntax, errors) => new int4(
ParseIntValue(syntax[0], errors),
ParseIntValue(syntax[1], errors),
ParseIntValue(syntax[2], errors),
ParseIntValue(syntax[3], errors))),
// UInts
[ShaderPropertyType.UInt] = new(1, TokenType.Number, (syntax, errors) => ParseUIntValue(syntax[0], errors)),
[ShaderPropertyType.UInt2] = new(2, TokenType.Number, (syntax, errors) => new uint2(
ParseUIntValue(syntax[0], errors),
ParseUIntValue(syntax[1], errors))),
[ShaderPropertyType.UInt3] = new(3, TokenType.Number, (syntax, errors) => new uint3(
ParseUIntValue(syntax[0], errors),
ParseUIntValue(syntax[1], errors),
ParseUIntValue(syntax[2], errors))),
[ShaderPropertyType.UInt4] = new(4, TokenType.Number, (syntax, errors) => new uint4(
ParseUIntValue(syntax[0], errors),
ParseUIntValue(syntax[1], errors),
ParseUIntValue(syntax[2], errors),
ParseUIntValue(syntax[3], errors))),
// Bools (numbers 0/1)
[ShaderPropertyType.Bool] = new(1, TokenType.Number, (syntax, errors) => ParseBoolValue(syntax[0], errors)),
[ShaderPropertyType.Bool2] = new(2, TokenType.Number, (syntax, errors) => new bool2(
ParseBoolValue(syntax[0], errors),
ParseBoolValue(syntax[1], errors))),
[ShaderPropertyType.Bool3] = new(3, TokenType.Number, (syntax, errors) => new bool3(
ParseBoolValue(syntax[0], errors),
ParseBoolValue(syntax[1], errors),
ParseBoolValue(syntax[2], errors))),
[ShaderPropertyType.Bool4] = new(4, TokenType.Number, (syntax, errors) => new bool4(
ParseBoolValue(syntax[0], errors),
ParseBoolValue(syntax[1], errors),
ParseBoolValue(syntax[2], errors),
ParseBoolValue(syntax[3], errors))),
// Textures (single identifier argument)
[ShaderPropertyType.Texture2D] = new(1, TokenType.Identifier, (syntax, errors) => ParseTextureDefault(syntax[0], errors)),
[ShaderPropertyType.Texture3D] = new(1, TokenType.Identifier, (syntax, errors) => ParseTextureDefault(syntax[0], errors)),
[ShaderPropertyType.TextureCube] = new(1, TokenType.Identifier, (syntax, errors) => ParseTextureDefault(syntax[0], errors)),
};
private static float ParseFloatValue(Token token, List<DSLShaderError> errors)
{
if (!float.TryParse(token.lexeme, CultureInfo.InvariantCulture, out var result))
{
errors.Add(new DSLShaderError
{
message = $"Failed to parse float value '{token.lexeme}'.",
line = token.line,
column = token.column
});
}
return result;
}
private static int ParseIntValue(Token token, List<DSLShaderError> errors)
{
if (!int.TryParse(token.lexeme, CultureInfo.InvariantCulture, out var result))
{
errors.Add(new DSLShaderError
{
message = $"Failed to parse int value '{token.lexeme}'.",
line = token.line,
column = token.column
});
}
return result;
}
private static uint ParseUIntValue(Token token, List<DSLShaderError> errors)
{
if (!uint.TryParse(token.lexeme, CultureInfo.InvariantCulture, out var result))
{
errors.Add(new DSLShaderError
{
message = $"Failed to parse uint value '{token.lexeme}'.",
line = token.line,
column = token.column
});
}
return result;
}
private static bool ParseBoolValue(Token token, List<DSLShaderError> errors)
{
if (!bool.TryParse(token.lexeme, out var result))
{
errors.Add(new DSLShaderError
{
message = $"Failed to parse bool value '{token.lexeme}'.",
line = token.line,
column = token.column
});
}
return result;
}
private static string ParseTextureDefault(Token token, List<DSLShaderError> errors)
{
if (!TokenLexicon.IsTextureDefaultValue(token.lexeme))
{
errors.Add(new DSLShaderError
{
message = $"Texture default value '{token.lexeme}' is not valid.",
line = token.line,
column = token.column
});
}
return token.lexeme;
}
public static bool ShouldEnter(Token token)
{
return token.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.PROPERTIES);
}
private static ShaderPropertyType FromString(string type)
{
return type.ToLower() switch
{
TokenLexicon.KnownTypes.FLOAT => ShaderPropertyType.Float,
TokenLexicon.KnownTypes.FLOAT2 => ShaderPropertyType.Float2,
TokenLexicon.KnownTypes.FLOAT3 => ShaderPropertyType.Float3,
TokenLexicon.KnownTypes.FLOAT4 => ShaderPropertyType.Float4,
TokenLexicon.KnownTypes.FLOAT4X4 => ShaderPropertyType.Float4x4,
TokenLexicon.KnownTypes.INT => ShaderPropertyType.Int,
TokenLexicon.KnownTypes.INT2 => ShaderPropertyType.Int2,
TokenLexicon.KnownTypes.INT3 => ShaderPropertyType.Int3,
TokenLexicon.KnownTypes.INT4 => ShaderPropertyType.Int4,
TokenLexicon.KnownTypes.UINT => ShaderPropertyType.UInt,
TokenLexicon.KnownTypes.UINT2 => ShaderPropertyType.UInt2,
TokenLexicon.KnownTypes.UINT3 => ShaderPropertyType.UInt3,
TokenLexicon.KnownTypes.UINT4 => ShaderPropertyType.UInt4,
TokenLexicon.KnownTypes.BOOL => ShaderPropertyType.Bool,
TokenLexicon.KnownTypes.BOOL2 => ShaderPropertyType.Bool2,
TokenLexicon.KnownTypes.BOOL3 => ShaderPropertyType.Bool3,
TokenLexicon.KnownTypes.BOOL4 => ShaderPropertyType.Bool4,
TokenLexicon.KnownTypes.TEXTURE2D => ShaderPropertyType.Texture2D,
TokenLexicon.KnownTypes.TEXTURE3D => ShaderPropertyType.Texture3D,
TokenLexicon.KnownTypes.TEXTURECUBE => ShaderPropertyType.TextureCube,
TokenLexicon.KnownTypes.TEXTURECUBE_ARRAY => ShaderPropertyType.TextureCubeArray,
TokenLexicon.KnownTypes.TEXTURE2D_ARRAY => ShaderPropertyType.Texture2DArray,
TokenLexicon.KnownTypes.SAMPLER => ShaderPropertyType.Sampler,
_ => ShaderPropertyType.None,
};
}
public static PropertiesSyntax Parse(TokenStreamSlice stream)
{
stream.Expect(TokenType.Keyword);
stream.Expect(TokenType.LBrace);
var syntax = new PropertiesSyntax();
var bodyStream = stream.Slice(stream.Remaining - 1);
while (bodyStream.HasMore)
{
var shaderProperty = new PropertyDeclaration();
if (bodyStream.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.GLOBAL)
|| bodyStream.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.LOCAL))
{
var scopeToken = bodyStream.Consume();
shaderProperty.scope = scopeToken;
}
var typeToken = bodyStream.Expect(TokenType.Identifier);
var nameToken = bodyStream.Expect(TokenType.Identifier);
shaderProperty.type = typeToken;
shaderProperty.name = nameToken;
var nextToken = bodyStream.Consume();
switch (nextToken.type)
{
case TokenType.Equals:
{
bodyStream.Expect(TokenType.LBrace);
while (!bodyStream.Match(TokenType.RBrace))
{
var token = bodyStream.Consume();
if (!token.Match(TokenType.Comma))
{
shaderProperty.propertyInitializer ??= new List<Token>();
shaderProperty.propertyInitializer.Add(token);
}
}
bodyStream.Expect(TokenType.RBrace);
bodyStream.Expect(TokenType.Semicolon);
break;
}
case TokenType.Semicolon:
break;
default:
throw new Exception($"Unexpected token '{nextToken.lexeme}' in property declaration.");
}
syntax.properties ??= new();
syntax.properties.Add(shaderProperty);
}
stream.Expect(TokenType.RBrace);
return syntax;
}
public static List<PropertySemantic>? SemanticAnalysis(PropertiesSyntax? syntax, List<DSLShaderError> errors)
{
if (syntax == null)
{
return null;
}
var models = new List<PropertySemantic>();
var usedPropertyNames = new HashSet<string>();
if (syntax.properties != null)
{
foreach (var property in syntax.properties)
{
var model = new PropertySemantic
{
scope = property.scope.lexeme switch
{
TokenLexicon.KnownKeywords.GLOBAL => PropertyScope.Global,
TokenLexicon.KnownKeywords.LOCAL => PropertyScope.Local,
_ => PropertyScope.Local,
}
};
var flowControl = ValidatePropertyType(errors, property, model);
if (!flowControl)
{
continue;
}
flowControl = ValidatePropertyName(errors, usedPropertyNames, property, model);
if (!flowControl)
{
continue;
}
if (property.propertyInitializer != null)
{
flowControl = ValidatePropertyInitializer(errors, property, model);
if (!flowControl)
{
continue;
}
}
usedPropertyNames.Add(property.name.lexeme);
models.Add(model);
}
}
return models;
}
private static bool ValidatePropertyType(List<DSLShaderError> errors, PropertyDeclaration property, PropertySemantic model)
{
if (!TokenLexicon.IsType(property.type.lexeme))
{
errors.Add(new DSLShaderError
{
message = $"Shader property type '{property.type.lexeme}' is not a valid type.",
line = property.type.line,
column = property.type.column
});
return false;
}
model.type = FromString(property.type.lexeme);
return true;
}
private static bool ValidatePropertyName(List<DSLShaderError> errors, HashSet<string> usedPropertyNames, PropertyDeclaration property, PropertySemantic model)
{
if (string.IsNullOrWhiteSpace(property.name.lexeme))
{
errors.Add(new DSLShaderError
{
message = "Shader property has an empty name.",
line = property.name.line,
column = property.name.column
});
return false;
}
else if (usedPropertyNames.Contains(property.name.lexeme))
{
errors.Add(new DSLShaderError
{
message = $"Shader property name '{property.name.lexeme}' is duplicated.",
line = property.name.line,
column = property.name.column
});
return false;
}
model.name = property.name.lexeme;
return true;
}
private static bool ValidatePropertyInitializer(List<DSLShaderError> errors, PropertyDeclaration property, PropertySemantic model)
{
var initializer = property.propertyInitializer;
if (initializer == null)
{
errors.Add(new DSLShaderError
{
message = "Shader property initializer is null.",
line = property.name.line,
column = property.name.column
});
return false;
}
if (!s_propTypeInfo.TryGetValue(model.type, out var info))
{
errors.Add(new DSLShaderError
{
message = $"No initializer metadata registered for property type '{model.type}'.",
line = property.name.line,
column = property.name.column
});
return false;
}
if (initializer.Count != info.ArgCount)
{
errors.Add(new DSLShaderError
{
message = $"Shader property constructor for type '{property.type.lexeme}' expects {info.ArgCount} argument(s), but got {initializer.Count}.",
line = property.name.line,
column = property.name.column
});
return false;
}
// Type check (uniform requirement for all args)
var hasError = false;
for (var i = 0; i < initializer.Count; i++)
{
var arg = initializer[i];
if (!arg.Match(info.ArgTokenType))
{
errors.Add(new DSLShaderError
{
message = $"Shader property constructor argument {i} expects token kind '{info.ArgTokenType}', but got '{arg.type}'.",
line = arg.line,
column = arg.column
});
hasError = true;
}
}
if (hasError)
{
return false;
}
// Build default Value if we have a builder (textures currently null / TODO)
if (info.Builder != null)
{
try
{
model.defaultValue = info.Builder(initializer, errors);
}
catch (Exception ex)
{
errors.Add(new DSLShaderError
{
message = $"Failed to construct default value for property '{property.name.lexeme}': {ex.Message}",
line = property.name.line,
column = property.name.column
});
return false;
}
}
return true;
}
}

114
Ghost.DSL/ShaderCompiler/Parser/ShaderBlock.cs
View File

@@ -1,114 +0,0 @@
namespace Ghost.DSL.ShaderCompiler.Parser;
internal class ShaderBlock : IBlockParser<DSLShaderSyntax, DSLShaderSemantics>
{
public static bool ShouldEnter(Token token)
{
return token.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.SHADER);
}
public static DSLShaderSyntax Parse(TokenStreamSlice stream)
{
var shader = new DSLShaderSyntax();
stream.Expect(TokenType.Keyword);
shader.name = stream.Expect(TokenType.StringLiteral);
stream.Expect(TokenType.LBrace);
var bodyStream = stream.Slice(stream.Remaining - 1);
while (bodyStream.TryPeek(out var nextToken))
{
if (PropertiesBlock.ShouldEnter(nextToken))
{
shader.properties = PropertiesBlock.Parse(bodyStream.SliceNextBlock());
}
else if (PipelineBlock.ShouldEnter(nextToken))
{
shader.pipeline = PipelineBlock.Parse(bodyStream.SliceNextBlock());
}
else if (PassBlock.ShouldEnter(nextToken))
{
shader.passes ??= new();
shader.passes.Add(PassBlock.Parse(bodyStream.SliceNextBlock()));
}
else if (nextToken.Match(TokenType.Identifier))
{
var func = ParseUtility.ParseFunction(ref bodyStream, TokenType.StringLiteral | TokenType.Number | TokenType.Identifier);
shader.functionCalls ??= new();
shader.functionCalls.Add(func);
}
else
{
throw new Exception($"Unexpected token '{nextToken}' in shader body.");
}
}
stream.Expect(TokenType.RBrace);
return shader;
}
public static DSLShaderSemantics? SemanticAnalysis(DSLShaderSyntax? syntax, List<DSLShaderError> errors)
{
if (syntax == null)
{
return null;
}
var shaderModel = new DSLShaderSemantics
{
name = syntax.name.lexeme,
properties = PropertiesBlock.SemanticAnalysis(syntax.properties, errors),
pipeline = PipelineBlock.SemanticAnalysis(syntax.pipeline, errors)
};
if (syntax.passes != null)
{
foreach (var passSyntax in syntax.passes)
{
var passModel = PassBlock.SemanticAnalysis(passSyntax, errors);
if (passModel != null)
{
shaderModel.passes ??= new();
shaderModel.passes.Add(passModel);
}
}
}
if (syntax.functionCalls != null)
{
foreach (var func in syntax.functionCalls)
{
switch (func.name.lexeme)
{
case TokenLexicon.KnownFunctions.FALLBACK:
if (func.arguments == null || func.arguments.Count != 1)
{
errors.Add(new DSLShaderError
{
message = "Fallback declaration requires exactly one arguments: (fallback shader name).",
line = func.name.line,
column = func.name.column
});
continue;
}
shaderModel.fallback = func.arguments[0].lexeme;
break;
default:
errors.Add(new DSLShaderError
{
message = $"Unknown function '{func.name.lexeme}' in shader.",
line = func.name.line,
column = func.name.column
});
break;
}
}
}
return shaderModel;
}
}

242
Ghost.DSL/ShaderCompiler/Token.cs
View File

@@ -1,242 +0,0 @@
namespace Ghost.DSL.ShaderCompiler;
[Flags]
public enum TokenType
{
None = 0,
// Literals
Identifier = 1 << 0, // variable names, function names, etc.
Keyword = 1 << 1, // shader, properties, pipeline, pass, etc.
Number = 1 << 2, // numeric literals (123, 45.67, .5)
StringLiteral = 1 << 3, // "ForwardVS.hlsl"
// Operators and Punctuation
Equals = 1 << 4, // =
Semicolon = 1 << 5, // ;
Comma = 1 << 6, // ,
// Delimiters
LBrace = 1 << 7, // {
RBrace = 1 << 8, // }
LParen = 1 << 9, // (
RParen = 1 << 10, // )
// Special
EndOfFile = 1 << 11, // EOF
// Future extensions (commented out for now)
// LBracket = 1 << 12, // [
// RBracket = 1 << 13, // ]
// Dot = 1 << 14, // .
// Colon = 1 << 15, // :
// Plus = 1 << 16, // +
// Minus = 1 << 17, // -
// Star = 1 << 18, // *
// Slash = 1 << 19, // /
}
public readonly struct Token : IEquatable<Token>
{
public readonly TokenType type;
public readonly string lexeme;
public readonly int line;
public readonly int column;
public static Token Null => new Token(TokenType.None, string.Empty, -1, -1);
public Token(TokenType type, string lexeme, int line, int column)
{
this.type = type;
this.lexeme = lexeme;
this.line = line;
this.column = column;
}
public override readonly string ToString()
{
return $"{type}('{lexeme}') at {line}:{column}";
}
public bool Equals(Token other)
{
return type == other.type && lexeme == other.lexeme && line == other.line && column == other.column;
}
public override int GetHashCode()
{
return HashCode.Combine(type, lexeme, line, column);
}
public override bool Equals(object? obj)
{
return obj is Token token && Equals(token);
}
public static bool operator ==(Token left, Token right)
{
return left.Equals(right);
}
public static bool operator !=(Token left, Token right)
{
return !(left == right);
}
}
public static class TokenExtensions
{
public static bool Match(this Token token, TokenType type, string? lexeme = null)
{
if (!type.HasFlag(token.type))
{
return false;
}
if (!string.IsNullOrEmpty(lexeme) && token.lexeme != lexeme)
{
return false;
}
return true;
}
public static void Expect(this Token token, TokenType type, string? lexeme = null)
{
if (!token.Match(type, lexeme))
{
var expected = lexeme != null ? $"{type}('{lexeme}')" : type.ToString();
throw new Exception($"Unexpected token at line {token.line}, column {token.column}. Expected {expected}, got {token.type}('{token.lexeme}').");
}
}
}
internal static class TokenLexicon
{
public static class KnownKeywords
{
public const string SHADER = "shader";
public const string PROPERTIES = "properties";
public const string PIPELINE = "pipeline";
public const string PASS = "pass";
public const string DEFINES = "defines";
public const string KEYWORDS = "keywords";
public const string INCLUDES = "includes";
public const string GLOBAL = "global";
public const string LOCAL = "local";
}
public static class KnownPipelineProperties
{
public const string ZTEST = "ztest";
public const string ZWRITE = "zwrite";
public const string CULL = "cull";
public const string BLEND = "blend";
public const string COLORMASK = "color_mask";
}
public static class KnownFunctions
{
public const string TASK_SHADER = "ts";
public const string MESH_SHADER = "ms";
public const string PIXEL_SHADER = "ps";
public const string COMPUTE_SHADER = "cs";
public const string LOCAL = "local";
public const string GLOBAL = "global";
public const string FALLBACK = "fallback";
}
public static class KnownTypes
{
// Basic types
public const string FLOAT = "float";
public const string FLOAT2 = "float2";
public const string FLOAT3 = "float3";
public const string FLOAT4 = "float4";
public const string FLOAT4X4 = "float4x4";
public const string INT = "int";
public const string INT2 = "int2";
public const string INT3 = "int3";
public const string INT4 = "int4";
public const string UINT = "uint";
public const string UINT2 = "uint2";
public const string UINT3 = "uint3";
public const string UINT4 = "uint4";
public const string BOOL = "bool";
public const string BOOL2 = "bool2";
public const string BOOL3 = "bool3";
public const string BOOL4 = "bool4";
// Texture types
public const string TEXTURE2D = "tex2d";
public const string TEXTURE2D_ARRAY = "tex2d_arr";
public const string TEXTURE3D = "tex3d";
public const string TEXTURECUBE = "texcube";
public const string TEXTURECUBE_ARRAY = "texcube_arr";
public const string SAMPLER = "sampler";
}
public static class KnownTextureValue
{
public const string WHITE = "white";
public const string BLACK = "black";
public const string GREY = "grey";
public const string NORMAL = "normal";
public const string TRANSPARENT = "transparent";
}
private static readonly HashSet<string> s_keywords = new()
{
KnownKeywords.SHADER,
KnownKeywords.PROPERTIES,
KnownKeywords.PIPELINE,
KnownKeywords.PASS,
KnownKeywords.DEFINES,
KnownKeywords.KEYWORDS,
KnownKeywords.INCLUDES,
KnownKeywords.GLOBAL,
KnownKeywords.LOCAL,
};
private static readonly HashSet<string> s_functions = new()
{
KnownFunctions.TASK_SHADER,
KnownFunctions.PIXEL_SHADER,
KnownFunctions.MESH_SHADER,
KnownFunctions.COMPUTE_SHADER,
KnownFunctions.LOCAL,
KnownFunctions.GLOBAL,
};
private static readonly HashSet<string> s_types = new()
{
KnownTypes.FLOAT, KnownTypes.FLOAT2, KnownTypes.FLOAT3, KnownTypes.FLOAT4, KnownTypes.FLOAT4X4,
KnownTypes.INT, KnownTypes.INT2, KnownTypes.INT3, KnownTypes.INT4,
KnownTypes.UINT, KnownTypes.UINT2, KnownTypes.UINT3, KnownTypes.UINT4,
KnownTypes.BOOL, KnownTypes.BOOL2, KnownTypes.BOOL3, KnownTypes.BOOL4,
KnownTypes.TEXTURE2D, KnownTypes.TEXTURE2D_ARRAY, KnownTypes.TEXTURE3D,
KnownTypes.TEXTURECUBE, KnownTypes.TEXTURECUBE_ARRAY,
KnownTypes.SAMPLER,
};
private static readonly HashSet<string> s_textureDefaultValues = new()
{
KnownTextureValue.WHITE,
KnownTextureValue.BLACK,
KnownTextureValue.GREY,
KnownTextureValue.NORMAL,
KnownTextureValue.TRANSPARENT,
};
public static bool IsKeyword(string lexeme) => s_keywords.Contains(lexeme);
public static bool IsFunction(string lexeme) => s_functions.Contains(lexeme);
public static bool IsType(string lexeme) => s_types.Contains(lexeme);
public static bool IsTextureDefaultValue(string lexeme) => s_textureDefaultValues.Contains(lexeme);
}

321
Ghost.DSL/ShaderCompiler/TokenStream.cs
View File

@@ -1,321 +0,0 @@
namespace Ghost.DSL.ShaderCompiler;
internal static class TokenStreamImple
{
public static Token Peek(ReadOnlySpan<Token> tokens, int index, int length)
{
if (index + length < tokens.Length)
{
return tokens[index + length];
}
throw new IndexOutOfRangeException();
}
public static bool TryPeek(ReadOnlySpan<Token> tokens, int index, int length, out Token token)
{
if (index + length < tokens.Length)
{
token = tokens[index + length];
return true;
}
token = default;
return false;
}
public static bool TryConsume(ReadOnlySpan<Token> tokens, ref int index, out Token token)
{
if (index < tokens.Length)
{
token = tokens[index++];
return true;
}
token = default;
return false;
}
public static Token Consume(ReadOnlySpan<Token> tokens, ref int index, int count = 1)
{
if (index + count <= tokens.Length)
{
index += count;
return tokens[index - 1];
}
throw new IndexOutOfRangeException();
}
public static bool Match(ReadOnlySpan<Token> tokens, int index, TokenType type, string? lexeme)
{
var t = Peek(tokens, index, 0);
if (!t.Match(type, lexeme))
{
return false;
}
return true;
}
public static int MatchMany(ReadOnlySpan<Token> tokens, int index, TokenType type, string? lexeme)
{
var count = 0;
while (TryPeek(tokens, index, 0, out var t) && t.Match(type, lexeme))
{
count++;
}
return count;
}
public static Token Expect(ReadOnlySpan<Token> tokens, ref int index, TokenType type, string? lexeme)
{
if (!TryPeek(tokens, index, 0, out var t))
{
throw new InvalidOperationException("Expected token but reached end of stream");
}
if (!t.Match(type, lexeme))
{
throw new InvalidOperationException($"Expected token {type}('{lexeme ?? "*"}') but got {t}");
}
index++;
return t;
}
}
internal class TokenStream
{
private readonly Token[] _tokens;
private int _index = 0;
public int Length => _tokens.Length;
public int Remaining => _tokens.Length - _index;
public bool HasMore => _index < _tokens.Length;
public int Position
{
get => _index;
set
{
if (value < 0 || value > _tokens.Length)
{
throw new ArgumentOutOfRangeException(nameof(value), "Position must be within the bounds of the token stream.");
}
_index = value;
}
}
public TokenStream(Token[] tokens)
{
_tokens = tokens;
}
public TokenStream(IEnumerable<Token> tokens)
{
_tokens = tokens.ToArray();
}
public Token Peek(int length = 0)
{
return TokenStreamImple.Peek(_tokens, _index, length);
}
public bool TryPeek(out Token token)
{
return TokenStreamImple.TryPeek(_tokens, _index, 0, out token);
}
public bool TryPeek(int length, out Token token)
{
return TokenStreamImple.TryPeek(_tokens, _index, length, out token);
}
public bool TryConsume(out Token token)
{
return TokenStreamImple.TryConsume(_tokens, ref _index, out token);
}
public Token Consume(int count = 1)
{
return TokenStreamImple.Consume(_tokens, ref _index, count);
}
public bool Match(TokenType type, string? lexeme = null)
{
return TokenStreamImple.Match(_tokens, _index, type, lexeme);
}
public int MatchMany(TokenType type, string? lexeme = null)
{
return TokenStreamImple.MatchMany(_tokens, _index, type, lexeme);
}
public Token Expect(TokenType type, string? lexeme = null)
{
return TokenStreamImple.Expect(_tokens, ref _index, type, lexeme);
}
public TokenStreamSlice Slice(int length = -1)
{
if (length <= 0)
{
length = _tokens.Length - _index;
}
var slice = _tokens.AsSpan().Slice(_index, length);
_index += length;
return new TokenStreamSlice(slice);
}
public TokenStreamSlice SliceNextBlock()
{
var length = 0;
var lBraceCount = 0;
var rBraceCount = 0;
Token nextToken;
do
{
nextToken = Peek(length);
if (length > 0 && lBraceCount > 0 && lBraceCount == rBraceCount)
{
break;
}
if (nextToken.Match(TokenType.LBrace))
{
lBraceCount++;
}
else if (nextToken.Match(TokenType.RBrace))
{
rBraceCount++;
}
length++;
}
while (_index + length < _tokens.Length);
return Slice(length);
}
}
internal ref struct TokenStreamSlice
{
private readonly ReadOnlySpan<Token> _tokens;
private int _index;
public readonly int Length => _tokens.Length;
public readonly int Remaining => _tokens.Length - _index;
public readonly bool HasMore => _index < _tokens.Length;
public int Position
{
readonly get => _index;
set
{
if (value < 0 || value > _tokens.Length)
{
throw new ArgumentOutOfRangeException(nameof(value), "Position must be within the bounds of the token stream.");
}
_index = value;
}
}
internal TokenStreamSlice(ReadOnlySpan<Token> tokens)
{
_tokens = tokens;
_index = 0;
}
public Token Peek(int length = 0)
{
return TokenStreamImple.Peek(_tokens, _index, length);
}
public bool TryPeek(out Token token)
{
return TokenStreamImple.TryPeek(_tokens, _index, 0, out token);
}
public bool TryPeek(int length, out Token token)
{
return TokenStreamImple.TryPeek(_tokens, _index, length, out token);
}
public bool TryConsume(out Token token)
{
return TokenStreamImple.TryConsume(_tokens, ref _index, out token);
}
public Token Consume(int count = 1)
{
return TokenStreamImple.Consume(_tokens, ref _index, count);
}
public bool Match(TokenType type, string? lexeme = null)
{
return TokenStreamImple.Match(_tokens, _index, type, lexeme);
}
public int MatchMany(TokenType type, string? lexeme = null)
{
return TokenStreamImple.MatchMany(_tokens, _index, type, lexeme);
}
public Token Expect(TokenType type, string? lexeme = null)
{
return TokenStreamImple.Expect(_tokens, ref _index, type, lexeme);
}
public TokenStreamSlice Slice(int length = -1)
{
if (length <= 0)
{
length = _tokens.Length - _index;
}
var slice = _tokens.Slice(_index, length);
_index += length;
return new TokenStreamSlice(slice);
}
public TokenStreamSlice SliceNextBlock()
{
var length = 0;
var lBraceCount = 0;
var rBraceCount = 0;
do
{
var nextToken = Peek(length);
if (length > 0 && lBraceCount > 0 && lBraceCount == rBraceCount)
{
break;
}
if (nextToken.Match(TokenType.LBrace))
{
lBraceCount++;
}
else if (nextToken.Match(TokenType.RBrace))
{
rBraceCount++;
}
length++;
}
while (_index + length < _tokens.Length);
if (lBraceCount != rBraceCount)
{
throw new InvalidOperationException("Unmatched braces in token stream.");
}
return Slice(length);
}
}