Refactoring Rendering backend

This commit is contained in:
2025-10-05 16:26:37 +09:00
parent a39f377533
commit 01a850ff94
99 changed files with 5056 additions and 5136 deletions

View File

@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("Ghost.Shader.Test")]

View File

@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<Compile Remove="Combinators\**" />
<EmbeddedResource Remove="Combinators\**" />
<None Remove="Combinators\**" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Misaki.HighPerformance.Mathematics" Version="1.1.0" />
</ItemGroup>
</Project>

148
Ghost.Shader/Lexer.cs Normal file
View File

@@ -0,0 +1,148 @@
namespace Ghost.Shader;
public class Lexer
{
private readonly string _source;
private int _pos = 0;
private int _line = 0;
private int _column = 0;
public Lexer(string source) => _source = source;
public IEnumerable<Token> Tokenize()
{
while (_pos < _source.Length)
{
var c = _source[_pos];
// Skip whitespace
if (char.IsWhiteSpace(c))
{
if (c == '\n')
{
_line++;
_column = 0;
}
else
{
_column++;
}
_pos++;
continue;
}
// Punctuation
switch (c)
{
case '=':
yield return MakeToken(TokenType.Equals, "=");
break;
case ';':
yield return MakeToken(TokenType.Semicolon, ";");
break;
case ',':
yield return MakeToken(TokenType.Comma, ",");
break;
case '{':
yield return MakeToken(TokenType.LBrace, "{");
break;
case '}':
yield return MakeToken(TokenType.RBrace, "}");
break;
case '(':
yield return MakeToken(TokenType.LParen, "(");
break;
case ')':
yield return MakeToken(TokenType.RParen, ")");
break;
case '"':
yield return ReadString();
break;
default:
if (char.IsLetter(c) || c == '_')
{
yield return ReadIdentifierOrKeyword();
}
else if (char.IsDigit(c) || c == '.')
{
yield return ReadNumber();
}
else
{
_pos++; // skip unknown
}
break;
}
}
yield return new Token(TokenType.EndOfFile, "", _line, _column);
}
private Token MakeToken(TokenType type, string lexeme)
{
var token = new Token(type, lexeme, _line, _column);
_pos++;
_column++;
return token;
}
private Token ReadString()
{
var startCol = _column;
_pos++;
_column++; // skip "
var start = _pos;
while (_pos < _source.Length && _source[_pos] != '"')
{
_pos++;
_column++;
}
var text = _source.Substring(start, _pos - start);
_pos++;
_column++; // skip closing "
return new Token(TokenType.StringLiteral, text, _line, startCol);
}
private Token ReadIdentifierOrKeyword()
{
var startCol = _column;
var start = _pos;
while (_pos < _source.Length && (char.IsLetterOrDigit(_source[_pos]) || _source[_pos] == '_'))
{
_pos++;
_column++;
}
var text = _source.Substring(start, _pos - start);
// Optional: detect keywords
if (TokenLexicon.IsKeyword(text))
{
return new Token(TokenType.Keyword, text, _line, startCol);
}
return new Token(TokenType.Identifier, text, _line, startCol);
}
private Token ReadNumber()
{
var startCol = _column;
var start = _pos;
while (_pos < _source.Length && (char.IsDigit(_source[_pos]) || _source[_pos] == '.'))
{
_pos++;
_column++;
}
var num = _source.Substring(start, _pos - start);
return new Token(TokenType.Number, num, _line, startCol);
}
}

View File

@@ -0,0 +1,58 @@
namespace Ghost.Shader;
internal static class ParseUtility
{
public static List<Token> ParseFunctionArguments(ref TokenStreamSlice stream, TokenType tokenType)
{
var args = new List<Token>();
stream.Expect(TokenType.LParen);
while (!stream.Peek().type.Equals(TokenType.RParen))
{
var argToken = stream.Expect(tokenType);
args.Add(argToken);
if (stream.Peek().type == TokenType.Comma)
{
stream.Consume();
}
else
{
break;
}
}
stream.Expect(TokenType.RParen);
return args;
}
public static bool TrySliceLine(ref TokenStreamSlice stream, out TokenStreamSlice lineStream)
{
var length = 0;
if (!stream.TryPeek(out var nextToken))
{
lineStream = default;
return false;
}
while (!nextToken.Match(TokenType.Semicolon) && !nextToken.Match(TokenType.RBrace))
{
length++;
if (!stream.TryPeek(length, out nextToken))
{
break;
}
}
if (length > 0)
{
lineStream = stream.Slice(length);
stream.Consume(); // Consume the semicolon
return true;
}
lineStream = default;
return false;
}
}

View File

@@ -0,0 +1,29 @@
namespace Ghost.Shader.ParserBlock;
internal class DefinesBlock : IBlockParser<List<string>>
{
public static bool ShouldEnter(Token token)
{
return token.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.DEFINES);
}
public static List<string> Parse(TokenStreamSlice stream)
{
stream.Expect(TokenType.Keyword);
stream.Expect(TokenType.LBrace);
var defines = new List<string>();
var bodyStream = stream.Slice(stream.Remaining - 1);
while (bodyStream.HasMore)
{
var defineToken = bodyStream.Expect(TokenType.Identifier);
defines.Add(defineToken.lexeme);
bodyStream.Expect(TokenType.Semicolon);
}
stream.Expect(TokenType.RBrace);
return defines;
}
}

View File

@@ -0,0 +1,13 @@
namespace Ghost.Shader.ParserBlock;
internal interface IBlockParser<T>
{
public static abstract bool ShouldEnter(Token token);
public static abstract T Parse(TokenStreamSlice ts);
}
internal interface IBlockParser<T, U> : IBlockParser<T>
{
public U SemanticAnalysis(T syntax, List<ShaderError> errors);
}

View File

@@ -0,0 +1,29 @@
namespace Ghost.Shader.ParserBlock;
internal class IncludesBlock : IBlockParser<List<string>>
{
public static bool ShouldEnter(Token token)
{
return token.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.INCLUDES);
}
public static List<string> Parse(TokenStreamSlice stream)
{
stream.Expect(TokenType.Keyword);
stream.Expect(TokenType.LBrace);
var includes = new List<string>();
var bodyStream = stream.Slice(stream.Remaining - 1);
while (bodyStream.HasMore)
{
var includeToken = bodyStream.Expect(TokenType.StringLiteral);
includes.Add(includeToken.lexeme);
bodyStream.Expect(TokenType.Semicolon);
}
stream.Expect(TokenType.RBrace);
return includes;
}
}

View File

@@ -0,0 +1,30 @@
namespace Ghost.Shader.ParserBlock;
internal class KeywordsBlock : IBlockParser<List<FunctionCall>>
{
public static bool ShouldEnter(Token token)
{
return token.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.KEYWORDS);
}
public static List<FunctionCall> Parse(TokenStreamSlice stream)
{
stream.Expect(TokenType.Keyword);
stream.Expect(TokenType.LBrace);
var keywords = new List<FunctionCall>();
var bodyStream = stream.Slice(stream.Remaining - 1);
while (bodyStream.HasMore)
{
var keywordToken = bodyStream.Expect(TokenType.Identifier);
var args = ParseUtility.ParseFunctionArguments(ref bodyStream, TokenType.Identifier);
keywords.Add(new FunctionCall { name = keywordToken, arguments = args });
bodyStream.Expect(TokenType.Semicolon);
}
stream.Expect(TokenType.RBrace);
return keywords;
}
}

View File

@@ -0,0 +1,63 @@
namespace Ghost.Shader.ParserBlock;
internal class PassBlock : IBlockParser<ShaderPassSyntax>
{
public static bool ShouldEnter(Token token)
{
return token.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.PASS);
}
public static ShaderPassSyntax Parse(TokenStreamSlice stream)
{
var pass = new ShaderPassSyntax();
stream.Expect(TokenType.Keyword);
pass.name = stream.Expect(TokenType.StringLiteral);
stream.Expect(TokenType.LBrace);
var bodyStream = stream.Slice(stream.Remaining - 1);
while (bodyStream.TryPeek(out var nextToken))
{
if (DefinesBlock.ShouldEnter(nextToken))
{
pass.defines = DefinesBlock.Parse(bodyStream.SliceNextBlock());
}
else if (IncludesBlock.ShouldEnter(nextToken))
{
pass.includes = IncludesBlock.Parse(bodyStream.SliceNextBlock());
}
else if (KeywordsBlock.ShouldEnter(nextToken))
{
pass.keywords = KeywordsBlock.Parse(bodyStream.SliceNextBlock());
}
else if (PipelineBlock.ShouldEnter(nextToken))
{
pass.overridePipeline = PipelineBlock.Parse(bodyStream.SliceNextBlock());
}
else if (nextToken.Match(TokenType.Identifier, "vs"))
{
bodyStream.Expect(TokenType.Identifier, "vs");
var vsArgs = ParseUtility.ParseFunctionArguments(ref bodyStream, TokenType.StringLiteral);
pass.vertexShader = vsArgs[0];
pass.vertexEntry = vsArgs[1];
bodyStream.Expect(TokenType.Semicolon);
}
else if (nextToken.Match(TokenType.Identifier, "ps"))
{
bodyStream.Expect(TokenType.Identifier, "ps");
var psArgs = ParseUtility.ParseFunctionArguments(ref bodyStream, TokenType.StringLiteral);
pass.pixelShader = psArgs[0];
pass.pixelEntry = psArgs[1];
bodyStream.Expect(TokenType.Semicolon);
}
else
{
throw new Exception($"Unexpected token '{nextToken.lexeme}' in pass body.");
}
}
stream.Expect(TokenType.RBrace);
return pass;
}
}

View File

@@ -0,0 +1,217 @@
using static Ghost.Shader.TokenLexicon;
namespace Ghost.Shader.ParserBlock;
internal class PipelineBlock : IBlockParser<PipelineStateSyntax, PipelineStateModel>
{
public static bool ShouldEnter(Token token)
{
return token.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.PIPELINE);
}
public static PipelineStateSyntax Parse(TokenStreamSlice stream)
{
stream.Expect(TokenType.Keyword);
stream.Expect(TokenType.LBrace);
var pipeline = new PipelineStateSyntax();
var bodyStream = stream.Slice(stream.Remaining - 1);
while (bodyStream.HasMore)
{
var stateToken = bodyStream.Expect(TokenType.Identifier);
bodyStream.Expect(TokenType.Equals);
var valueToken = bodyStream.Expect(TokenType.Identifier | TokenType.Number);
switch (stateToken.lexeme)
{
case KnownPipelineProperties.ZTEST:
pipeline.zTest = valueToken;
break;
case KnownPipelineProperties.ZWRITE:
pipeline.zWrite = valueToken;
break;
case KnownPipelineProperties.CULL:
pipeline.cull = valueToken;
break;
case KnownPipelineProperties.BLEND:
pipeline.blend = valueToken;
break;
case KnownPipelineProperties.COLORMASK:
pipeline.colorMask = valueToken;
break;
default:
throw new InvalidDataException($"Unknown pipeline state: {stateToken.lexeme}");
}
bodyStream.Expect(TokenType.Semicolon);
}
stream.Expect(TokenType.RBrace);
return pipeline;
}
public PipelineStateModel SemanticAnalysis(PipelineStateSyntax syntax, List<ShaderError> errors)
{
var model = new PipelineStateModel();
// ZTest
if (!syntax.zTest.Match(TokenType.None))
{
switch (syntax.zTest.lexeme)
{
case "disable":
model.zTest = ZTestOptions.Disabled;
break;
case "less":
model.zTest = ZTestOptions.Less;
break;
case "less_equal":
model.zTest = ZTestOptions.LessEqual;
break;
case "equal":
model.zTest = ZTestOptions.Equal;
break;
case "greater_equal":
model.zTest = ZTestOptions.GreaterEqual;
break;
case "greater":
model.zTest = ZTestOptions.Greater;
break;
case "not_equal":
model.zTest = ZTestOptions.NotEqual;
break;
case "always":
model.zTest = ZTestOptions.Always;
break;
default:
errors.Add(new ShaderError
{
message = $"Invalid ZTest option: {syntax.zTest.lexeme}",
line = syntax.zTest.line,
column = syntax.zTest.column
});
break;
}
}
else
{
model.zTest = ZTestOptions.LessEqual;
}
// ZWrite
if (!syntax.zWrite.Match(TokenType.None))
{
switch (syntax.zWrite.lexeme)
{
case "on":
model.zWrite = ZWriteOptions.On;
break;
case "off":
model.zWrite = ZWriteOptions.Off;
break;
default:
errors.Add(new ShaderError
{
message = $"Invalid ZWrite option: {syntax.zWrite.lexeme}",
line = syntax.zWrite.line,
column = syntax.zWrite.column
});
break;
}
}
else
{
model.zWrite = ZWriteOptions.On;
}
// Cull
if (!syntax.cull.Match(TokenType.None))
{
switch (syntax.cull.lexeme)
{
case "off":
model.cull = CullOptions.Off;
break;
case "front":
model.cull = CullOptions.Front;
break;
case "back":
model.cull = CullOptions.Back;
break;
default:
errors.Add(new ShaderError
{
message = $"Invalid Cull option: {syntax.cull.lexeme}",
line = syntax.cull.line,
column = syntax.cull.column
});
break;
}
}
else
{
model.cull = CullOptions.Back;
}
// Blend
if (!syntax.blend.Match(TokenType.None))
{
switch (syntax.blend.lexeme)
{
case "opaque":
model.blend = BlendOptions.Opaque;
break;
case "alpha":
model.blend = BlendOptions.Alpha;
break;
case "additive":
model.blend = BlendOptions.Additive;
break;
case "multiply":
model.blend = BlendOptions.Multiply;
break;
case "premultiplied":
model.blend = BlendOptions.PremultipliedAlpha;
break;
default:
errors.Add(new ShaderError
{
message = $"Invalid Blend option: {syntax.blend.lexeme}",
line = syntax.blend.line,
column = syntax.blend.column
});
break;
}
}
else
{
model.blend = BlendOptions.Opaque;
}
// Color Mask
if (!syntax.colorMask.Match(TokenType.None))
{
if (uint.TryParse(syntax.colorMask.lexeme, out var colorMask))
{
model.colorMask = colorMask;
}
else
{
errors.Add(new ShaderError
{
message = $"Invalid Color Mask value: {syntax.colorMask.lexeme}",
line = syntax.colorMask.line,
column = syntax.colorMask.column
});
}
}
else
{
model.colorMask = 0xF; // Default to RGBA
}
return model;
}
}

View File

@@ -0,0 +1,337 @@
using Misaki.HighPerformance.Mathematics;
using System.Globalization;
namespace Ghost.Shader.ParserBlock;
internal class PropertiesBlock : IBlockParser<List<PropertySyntax>, List<PropertyModel>>
{
private sealed record PropTypeInfo(int ArgCount, TokenType ArgTokenType, Func<List<Token>, object?>? Builder);
private static readonly Dictionary<ShaderPropertyType, PropTypeInfo> s_propTypeInfo = new()
{
// Floats
[ShaderPropertyType.Float] = new(1, TokenType.Number, a => float.Parse(a[0].lexeme, CultureInfo.InvariantCulture)),
[ShaderPropertyType.Float2] = new(2, TokenType.Number, a => new float2(
float.Parse(a[0].lexeme, CultureInfo.InvariantCulture),
float.Parse(a[1].lexeme, CultureInfo.InvariantCulture))),
[ShaderPropertyType.Float3] = new(3, TokenType.Number, a => new float3(
float.Parse(a[0].lexeme, CultureInfo.InvariantCulture),
float.Parse(a[1].lexeme, CultureInfo.InvariantCulture),
float.Parse(a[2].lexeme, CultureInfo.InvariantCulture))),
[ShaderPropertyType.Float4] = new(4, TokenType.Number, a => new float4(
float.Parse(a[0].lexeme, CultureInfo.InvariantCulture),
float.Parse(a[1].lexeme, CultureInfo.InvariantCulture),
float.Parse(a[2].lexeme, CultureInfo.InvariantCulture),
float.Parse(a[3].lexeme, CultureInfo.InvariantCulture))),
// Ints
[ShaderPropertyType.Int] = new(1, TokenType.Number, a => int.Parse(a[0].lexeme, CultureInfo.InvariantCulture)),
[ShaderPropertyType.Int2] = new(2, TokenType.Number, a => new int2(
int.Parse(a[0].lexeme, CultureInfo.InvariantCulture),
int.Parse(a[1].lexeme, CultureInfo.InvariantCulture))),
[ShaderPropertyType.Int3] = new(3, TokenType.Number, a => new int3(
int.Parse(a[0].lexeme, CultureInfo.InvariantCulture),
int.Parse(a[1].lexeme, CultureInfo.InvariantCulture),
int.Parse(a[2].lexeme, CultureInfo.InvariantCulture))),
[ShaderPropertyType.Int4] = new(4, TokenType.Number, a => new int4(
int.Parse(a[0].lexeme, CultureInfo.InvariantCulture),
int.Parse(a[1].lexeme, CultureInfo.InvariantCulture),
int.Parse(a[2].lexeme, CultureInfo.InvariantCulture),
int.Parse(a[3].lexeme, CultureInfo.InvariantCulture))),
// UInts
[ShaderPropertyType.UInt] = new(1, TokenType.Number, a => uint.Parse(a[0].lexeme, CultureInfo.InvariantCulture)),
[ShaderPropertyType.UInt2] = new(2, TokenType.Number, a => new uint2(
uint.Parse(a[0].lexeme, CultureInfo.InvariantCulture),
uint.Parse(a[1].lexeme, CultureInfo.InvariantCulture))),
[ShaderPropertyType.UInt3] = new(3, TokenType.Number, a => new uint3(
uint.Parse(a[0].lexeme, CultureInfo.InvariantCulture),
uint.Parse(a[1].lexeme, CultureInfo.InvariantCulture),
uint.Parse(a[2].lexeme, CultureInfo.InvariantCulture))),
[ShaderPropertyType.UInt4] = new(4, TokenType.Number, a => new uint4(
uint.Parse(a[0].lexeme, CultureInfo.InvariantCulture),
uint.Parse(a[1].lexeme, CultureInfo.InvariantCulture),
uint.Parse(a[2].lexeme, CultureInfo.InvariantCulture),
uint.Parse(a[3].lexeme, CultureInfo.InvariantCulture))),
// Bools (numbers 0/1)
[ShaderPropertyType.Bool] = new(1, TokenType.Number, a => a[0].lexeme != "0"),
[ShaderPropertyType.Bool2] = new(2, TokenType.Number, a => new bool2(a[0].lexeme != "0", a[1].lexeme != "0")),
[ShaderPropertyType.Bool3] = new(3, TokenType.Number, a => new bool3(a[0].lexeme != "0", a[1].lexeme != "0", a[2].lexeme != "0")),
[ShaderPropertyType.Bool4] = new(4, TokenType.Number, a => new bool4(a[0].lexeme != "0", a[1].lexeme != "0", a[2].lexeme != "0", a[3].lexeme != "0")),
// Textures (single identifier argument currently no default object built)
[ShaderPropertyType.Texture2D] = new(1, TokenType.Identifier, null),
[ShaderPropertyType.Texture3D] = new(1, TokenType.Identifier, null),
[ShaderPropertyType.TextureCube] = new(1, TokenType.Identifier, null),
};
public static bool ShouldEnter(Token token)
{
return token.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.PROPERTIES);
}
private static ShaderPropertyType FromString(string type)
{
return type.ToLower() switch
{
"float" => ShaderPropertyType.Float,
"float2" => ShaderPropertyType.Float2,
"float3" => ShaderPropertyType.Float3,
"float4" => ShaderPropertyType.Float4,
"int" => ShaderPropertyType.Int,
"int2" => ShaderPropertyType.Int2,
"int3" => ShaderPropertyType.Int3,
"int4" => ShaderPropertyType.Int4,
"uint" => ShaderPropertyType.UInt,
"uint2" => ShaderPropertyType.UInt2,
"uint3" => ShaderPropertyType.UInt3,
"uint4" => ShaderPropertyType.UInt4,
"bool" => ShaderPropertyType.Bool,
"bool2" => ShaderPropertyType.Bool2,
"bool3" => ShaderPropertyType.Bool3,
"bool4" => ShaderPropertyType.Bool4,
"texture2d" => ShaderPropertyType.Texture2D,
"texture3d" => ShaderPropertyType.Texture3D,
"texturecube" => ShaderPropertyType.TextureCube,
_ => ShaderPropertyType.None,
};
}
public static List<PropertySyntax> Parse(TokenStreamSlice stream)
{
stream.Expect(TokenType.Keyword);
stream.Expect(TokenType.LBrace);
var properties = new List<PropertySyntax>();
var bodyStream = stream.Slice(stream.Remaining - 1);
while (bodyStream.HasMore)
{
var shaderProperty = new PropertySyntax();
var typeToken = bodyStream.Expect(TokenType.Identifier);
var nameToken = bodyStream.Expect(TokenType.Identifier);
shaderProperty.type = typeToken;
shaderProperty.name = nameToken;
var nextToken = bodyStream.Consume();
switch (nextToken.type)
{
case TokenType.Equals:
{
var constructorTypeToken = bodyStream.Expect(TokenType.Identifier);
var args = ParseUtility.ParseFunctionArguments(ref bodyStream, TokenType.Identifier | TokenType.Number);
shaderProperty.propertyConstructor = new FunctionCall
{
name = constructorTypeToken,
arguments = args
};
bodyStream.Expect(TokenType.Semicolon);
break;
}
case TokenType.Semicolon:
break;
default:
throw new Exception($"Unexpected token '{nextToken.lexeme}' in property declaration.");
}
properties.Add(shaderProperty);
}
stream.Expect(TokenType.RBrace);
return properties;
}
public List<PropertyModel> SemanticAnalysis(List<PropertySyntax> syntax, List<ShaderError> errors)
{
var models = new List<PropertyModel>();
var usedPropertyNames = new HashSet<string>();
foreach (var property in syntax)
{
var model = new PropertyModel();
var flowControl = ValidatePropertyType(errors, property, model);
if (!flowControl)
{
continue;
}
flowControl = ValidatePropertyName(errors, usedPropertyNames, property, model);
if (!flowControl)
{
continue;
}
if (property.propertyConstructor != null)
{
flowControl = ValidatePropertyConstructor(errors, property, model);
if (!flowControl)
{
continue;
}
}
usedPropertyNames.Add(property.name.lexeme);
models.Add(model);
}
return models;
}
private static bool ValidatePropertyType(List<ShaderError> errors, PropertySyntax property, PropertyModel model)
{
if (!TokenLexicon.IsType(property.type.lexeme))
{
errors.Add(new ShaderError
{
message = $"Shader property type '{property.type.lexeme}' is not a valid type.",
line = property.type.line,
column = property.type.column
});
return false;
}
model.type = FromString(property.type.lexeme);
return true;
}
private static bool ValidatePropertyName(List<ShaderError> errors, HashSet<string> usedPropertyNames, PropertySyntax property, PropertyModel model)
{
if (string.IsNullOrWhiteSpace(property.name.lexeme))
{
errors.Add(new ShaderError
{
message = "Shader property has an empty name.",
line = property.name.line,
column = property.name.column
});
return false;
}
else if (usedPropertyNames.Contains(property.name.lexeme))
{
errors.Add(new ShaderError
{
message = $"Shader property name '{property.name.lexeme}' is duplicated.",
line = property.name.line,
column = property.name.column
});
return false;
}
model.name = property.name.lexeme;
return true;
}
private static bool ValidatePropertyConstructor(List<ShaderError> errors, PropertySyntax property, PropertyModel model)
{
var constructor = property.propertyConstructor;
if (constructor == null)
{
errors.Add(new ShaderError
{
message = "Shader property constructor is null.",
line = property.name.line,
column = property.name.column
});
return false;
}
if (string.IsNullOrWhiteSpace(constructor.name.lexeme))
{
errors.Add(new ShaderError
{
message = "Shader property constructor has an empty name.",
line = constructor.name.line,
column = constructor.name.column
});
return false;
}
if (constructor.name.lexeme != property.type.lexeme)
{
errors.Add(new ShaderError
{
message = $"Shader property constructor name '{constructor.name.lexeme}' does not match property type '{property.type.lexeme}'.",
line = constructor.name.line,
column = constructor.name.column
});
return false;
}
if (!s_propTypeInfo.TryGetValue(model.type, out var info))
{
errors.Add(new ShaderError
{
message = $"No constructor metadata registered for property type '{model.type}'.",
line = constructor.name.line,
column = constructor.name.column
});
return false;
}
// Count check
if (constructor.arguments.Count != info.ArgCount)
{
errors.Add(new ShaderError
{
message = $"Shader property constructor for type '{property.type.lexeme}' expects {info.ArgCount} argument(s), but got {constructor.arguments.Count}.",
line = constructor.name.line,
column = constructor.name.column
});
return false;
}
// Type check (uniform requirement for all args)
var hasError = false;
for (var i = 0; i < constructor.arguments.Count; i++)
{
var arg = constructor.arguments[i];
if (!arg.Match(info.ArgTokenType))
{
errors.Add(new ShaderError
{
message = $"Shader property constructor argument {i} expects token kind '{info.ArgTokenType}', but got '{arg.type}'.",
line = arg.line,
column = arg.column
});
hasError = true;
}
}
if (hasError)
{
return false;
}
// Build default value if we have a builder (textures currently null / TODO)
if (info.Builder != null)
{
try
{
model.defaultValue = info.Builder(constructor.arguments);
}
catch (Exception ex)
{
errors.Add(new ShaderError
{
message = $"Failed to construct default value for property '{property.name.lexeme}': {ex.Message}",
line = constructor.name.line,
column = constructor.name.column
});
return false;
}
}
return true;
}
}

View File

@@ -0,0 +1,43 @@
namespace Ghost.Shader.ParserBlock;
internal class ShaderBlock : IBlockParser<ShaderSyntax>
{
public static bool ShouldEnter(Token token)
{
return token.Match(TokenType.Keyword, TokenLexicon.KnownKeywords.SHADER);
}
public static ShaderSyntax Parse(TokenStreamSlice stream)
{
var shader = new ShaderSyntax();
stream.Expect(TokenType.Keyword);
shader.name = stream.Expect(TokenType.StringLiteral);
stream.Expect(TokenType.LBrace);
var bodyStream = stream.Slice(stream.Remaining - 1);
while (bodyStream.TryPeek(out var nextToken))
{
if (PropertiesBlock.ShouldEnter(nextToken))
{
shader.properties = PropertiesBlock.Parse(bodyStream.SliceNextBlock());
}
else if (PipelineBlock.ShouldEnter(nextToken))
{
shader.pipeline = PipelineBlock.Parse(bodyStream.SliceNextBlock());
}
else if (PassBlock.ShouldEnter(nextToken))
{
shader.passes.Add(PassBlock.Parse(bodyStream.SliceNextBlock()));
}
else
{
throw new Exception($"Unexpected token '{nextToken.lexeme}' in shader body.");
}
}
stream.Expect(TokenType.RBrace);
return shader;
}
}

View File

@@ -0,0 +1,35 @@
namespace Ghost.Shader;
public enum ZTestOptions
{
Disabled,
Less,
LessEqual,
Equal,
GreaterEqual,
Greater,
NotEqual,
Always
}
public enum ZWriteOptions
{
Off,
On
}
public enum CullOptions
{
Off,
Front,
Back
}
public enum BlendOptions
{
Opaque,
Alpha,
Additive,
Multiply,
PremultipliedAlpha
}

View File

@@ -0,0 +1,58 @@
using Ghost.Shader.ParserBlock;
namespace Ghost.Shader;
public struct ShaderError
{
public string message;
public int line;
public int column;
public readonly override string ToString()
{
return $"Error at {line}:{column} - {message}";
}
}
internal static class ShaderCompiler
{
public static List<ShaderSyntax> ParseShaders(TokenStream stream)
{
var shaders = new List<ShaderSyntax>();
while (stream.TryPeek(out var nextToken))
{
if (ShaderBlock.ShouldEnter(nextToken))
{
var shader = ShaderBlock.Parse(stream.SliceNextBlock());
shaders.Add(shader);
}
else if (nextToken.Match(TokenType.EndOfFile))
{
stream.Consume();
}
else
{
throw new Exception($"Unexpected token '{nextToken.lexeme}' at top level. Expected 'shader' declaration.");
}
}
return shaders;
}
public static ShaderModel SemanticAnalysis(ShaderSyntax syntax, out List<ShaderError> errors)
{
var shaderModel = new ShaderModel();
errors = new List<ShaderError>();
shaderModel.name = syntax.name.lexeme;
var propertiesBlock = new PropertiesBlock();
shaderModel.properties = propertiesBlock.SemanticAnalysis(syntax.properties, errors);
var pipelineBlock = new PipelineBlock();
shaderModel.pipeline = pipelineBlock.SemanticAnalysis(syntax.pipeline, errors);
return shaderModel;
}
}

View File

@@ -0,0 +1,34 @@
namespace Ghost.Shader;
public enum ShaderPropertyType
{
None,
Float, Float2, Float3, Float4,
Int, Int2, Int3, Int4,
UInt, UInt2, UInt3, UInt4,
Bool, Bool2, Bool3, Bool4,
Texture2D, Texture3D, TextureCube,
}
internal class PropertyModel
{
public ShaderPropertyType type;
public string name = string.Empty;
public object? defaultValue;
}
internal class PipelineStateModel
{
public ZTestOptions zTest;
public ZWriteOptions zWrite;
public CullOptions cull;
public BlendOptions blend;
public uint colorMask;
}
internal class ShaderModel
{
public string name = string.Empty;
public List<PropertyModel> properties = new();
public PipelineStateModel pipeline = new();
}

View File

@@ -0,0 +1,44 @@
namespace Ghost.Shader;
public class FunctionCall
{
public Token name;
public List<Token> arguments = new();
}
public class PropertySyntax
{
public Token type;
public Token name;
public FunctionCall? propertyConstructor;
}
public class PipelineStateSyntax
{
public Token zTest;
public Token zWrite;
public Token cull;
public Token blend;
public Token colorMask;
}
public class ShaderPassSyntax
{
public Token name;
public Token vertexShader;
public Token vertexEntry;
public Token pixelShader;
public Token pixelEntry;
public List<string>? defines;
public List<string>? includes;
public List<FunctionCall>? keywords;
public PipelineStateSyntax? overridePipeline;
}
public class ShaderSyntax
{
public Token name;
public List<PropertySyntax> properties = new();
public PipelineStateSyntax pipeline = new();
public List<ShaderPassSyntax> passes = new();
}

138
Ghost.Shader/Token.cs Normal file
View File

@@ -0,0 +1,138 @@
namespace Ghost.Shader;
[Flags]
public enum TokenType
{
None = 0,
Identifier = 1 << 0, // variable names, types, etc.
Keyword = 1 << 1, // shader, properties, pipeline, pass, etc.
Number = 1 << 2, // numeric literals
StringLiteral = 1 << 3, // "ForwardVS.hlsl"
Equals = 1 << 4, // =
Semicolon = 1 << 5, // ;
Comma = 1 << 6, // ,
LBrace = 1 << 7, // {
RBrace = 1 << 8, // }
LParen = 1 << 9, // (
RParen = 1 << 10, // )
EndOfFile = 1 << 11 // EOF
}
public readonly struct Token
{
public readonly TokenType type;
public readonly string lexeme;
public readonly int line;
public readonly int column;
public Token(TokenType type, string lexeme, int line, int column)
{
this.type = type;
this.lexeme = lexeme;
this.line = line;
this.column = column;
}
public override readonly string ToString()
{
return $"{type}('{lexeme}') at {line}:{column}";
}
}
public static class TokenExtensions
{
public static bool Match(this Token token, TokenType type, string? lexeme = null)
{
if (!type.HasFlag(token.type))
{
return false;
}
if (!string.IsNullOrEmpty(lexeme) && token.lexeme != lexeme)
{
return false;
}
return true;
}
public static void Expect(this Token token, TokenType type, string? lexeme = null)
{
if (!token.Match(type, lexeme))
{
var expected = lexeme != null ? $"{type}('{lexeme}')" : type.ToString();
throw new Exception($"Unexpected token at line {token.line}, column {token.column}. Expected {expected}, got {token.type}('{token.lexeme}').");
}
}
}
public static class TokenLexicon
{
public static class KnownKeywords
{
public const string SHADER = "shader";
public const string PROPERTIES = "properties";
public const string PIPELINE = "pipeline";
public const string PASS = "pass";
public const string DEFINES = "defines";
public const string KEYWORDS = "keywords";
public const string INCLUDES = "includes";
}
public static class KnownPipelineProperties
{
public const string ZTEST = "ztest";
public const string ZWRITE = "zwrite";
public const string CULL = "cull";
public const string BLEND = "blend";
public const string COLORMASK = "color_mask";
}
private static readonly HashSet<string> s_keywords = new()
{
KnownKeywords.SHADER,
KnownKeywords.PROPERTIES,
KnownKeywords.PIPELINE,
KnownKeywords.PASS,
KnownKeywords.DEFINES,
KnownKeywords.KEYWORDS,
KnownKeywords.INCLUDES,
};
private static readonly HashSet<string> s_function = new()
{
"vs",
"ps",
"ms",
"cs",
"dynamic",
"static",
};
private static readonly HashSet<string> s_types = new()
{
"float",
"float2",
"float3",
"float4",
"int",
"int2",
"int3",
"int4",
"uint",
"uint2",
"uint3",
"uint4",
"bool",
"bool2",
"bool3",
"bool4",
"texture2d",
"texture3d",
"texturecube",
};
public static bool IsKeyword(string lexeme) => s_keywords.Contains(lexeme);
public static bool IsFunction(string lexeme) => s_function.Contains(lexeme);
public static bool IsType(string lexeme) => s_types.Contains(lexeme);
}

285
Ghost.Shader/TokenStream.cs Normal file
View File

@@ -0,0 +1,285 @@
namespace Ghost.Shader;
internal static class TokenStreamImple
{
public static Token Peek(ReadOnlySpan<Token> tokens, ref int index, int length)
{
return index + length < tokens.Length ? tokens[index + length] : throw new InvalidOperationException("No more tokens available");
}
public static bool TryPeek(ReadOnlySpan<Token> tokens, ref int index, int length, out Token token)
{
if (index + length < tokens.Length)
{
token = tokens[index + length];
return true;
}
token = default;
return false;
}
public static bool TryConsume(ReadOnlySpan<Token> tokens, ref int index, out Token token)
{
if (index < tokens.Length)
{
token = tokens[index++];
return true;
}
token = default;
return false;
}
public static Token Consume(ReadOnlySpan<Token> tokens, ref int index)
{
return index < tokens.Length ? tokens[index++] : throw new InvalidOperationException("No more tokens available");
}
public static bool Match(ReadOnlySpan<Token> tokens, ref int index, TokenType type, string? lexeme = null)
{
var t = Peek(tokens, ref index, 0);
if (!t.Match(type, lexeme))
{
return false;
}
index++;
return true;
}
public static Token Expect(ReadOnlySpan<Token> tokens, ref int index, TokenType type, string? lexeme = null)
{
if (!TryPeek(tokens, ref index, 0, out var t))
{
throw new InvalidOperationException("Expected token but reached end of stream");
}
if (!t.Match(type, lexeme))
{
throw new InvalidOperationException($"Expected token {type}('{lexeme ?? "*"}') but got {t}");
}
index++;
return t;
}
}
internal class TokenStream
{
private readonly Token[] _tokens;
private int _index = 0;
public int Length => _tokens.Length;
public int Remaining => _tokens.Length - _index;
public bool HasMore => _index < _tokens.Length;
public int Position
{
get => _index;
set
{
if (value < 0 || value > _tokens.Length)
{
throw new ArgumentOutOfRangeException(nameof(value), "Position must be within the bounds of the token stream.");
}
_index = value;
}
}
public TokenStream(Token[] tokens)
{
_tokens = tokens;
}
public Token Peek(int length = 0)
{
return TokenStreamImple.Peek(_tokens, ref _index, length);
}
public bool TryPeek(out Token token)
{
return TokenStreamImple.TryPeek(_tokens, ref _index, 0, out token);
}
public bool TryPeek(int length, out Token token)
{
return TokenStreamImple.TryPeek(_tokens, ref _index, length, out token);
}
public bool TryConsume(out Token token)
{
return TokenStreamImple.TryConsume(_tokens, ref _index, out token);
}
public Token Consume()
{
return TokenStreamImple.Consume(_tokens, ref _index);
}
public bool Match(TokenType type, string? lexeme = null)
{
return TokenStreamImple.Match(_tokens, ref _index, type, lexeme);
}
public Token Expect(TokenType type, string? lexeme = null)
{
return TokenStreamImple.Expect(_tokens, ref _index, type, lexeme);
}
public TokenStreamSlice Slice(int length = -1)
{
if (length <= 0)
{
length = _tokens.Length - _index;
}
var slice = _tokens.AsSpan().Slice(_index, length);
_index += length;
return new TokenStreamSlice(slice);
}
public TokenStreamSlice SliceNextBlock()
{
var length = 0;
var lBraceCount = 0;
var rBraceCount = 0;
Token nextToken;
do
{
nextToken = Peek(length);
if (length > 0 && lBraceCount > 0 && lBraceCount == rBraceCount)
{
break;
}
if (nextToken.Match(TokenType.LBrace))
{
lBraceCount++;
}
else if (nextToken.Match(TokenType.RBrace))
{
rBraceCount++;
}
length++;
}
while (_index + length < _tokens.Length);
return Slice(length);
}
}
internal ref struct TokenStreamSlice
{
private readonly ReadOnlySpan<Token> _tokens;
private int _index;
public readonly int Length => _tokens.Length;
public readonly int Remaining => _tokens.Length - _index;
public readonly bool HasMore => _index < _tokens.Length;
public int Position
{
readonly get => _index;
set
{
if (value < 0 || value > _tokens.Length)
{
throw new ArgumentOutOfRangeException(nameof(value), "Position must be within the bounds of the token stream.");
}
_index = value;
}
}
internal TokenStreamSlice(ReadOnlySpan<Token> tokens)
{
_tokens = tokens;
_index = 0;
}
public Token Peek(int length = 0)
{
return TokenStreamImple.Peek(_tokens, ref _index, length);
}
public bool TryPeek(out Token token)
{
return TokenStreamImple.TryPeek(_tokens, ref _index, 0, out token);
}
public bool TryPeek(int length, out Token token)
{
return TokenStreamImple.TryPeek(_tokens, ref _index, length, out token);
}
public bool TryConsume(out Token token)
{
return TokenStreamImple.TryConsume(_tokens, ref _index, out token);
}
public Token Consume()
{
return TokenStreamImple.Consume(_tokens, ref _index);
}
public bool Match(TokenType type, string? lexeme = null)
{
return TokenStreamImple.Match(_tokens, ref _index, type, lexeme);
}
public Token Expect(TokenType type, string? lexeme = null)
{
return TokenStreamImple.Expect(_tokens, ref _index, type, lexeme);
}
public TokenStreamSlice Slice(int length = -1)
{
if (length <= 0)
{
length = _tokens.Length - _index;
}
var slice = _tokens.Slice(_index, length);
_index += length;
return new TokenStreamSlice(slice);
}
public TokenStreamSlice SliceNextBlock()
{
var length = 0;
var lBraceCount = 0;
var rBraceCount = 0;
do
{
var nextToken = Peek(length);
if (length > 0 && lBraceCount > 0 && lBraceCount == rBraceCount)
{
break;
}
if (nextToken.Match(TokenType.LBrace))
{
lBraceCount++;
}
else if (nextToken.Match(TokenType.RBrace))
{
rBraceCount++;
}
length++;
}
while (_index + length < _tokens.Length);
if (lBraceCount != rBraceCount)
{
throw new InvalidOperationException("Unmatched braces in token stream.");
}
return Slice(length);
}
}