diff --git a/Ghost.Editor.Core/SceneGraph/EditorSceneManager.cs b/Ghost.Editor.Core/SceneGraph/EditorSceneManager.cs index 35f1b1e..5bc80db 100644 --- a/Ghost.Editor.Core/SceneGraph/EditorSceneManager.cs +++ b/Ghost.Editor.Core/SceneGraph/EditorSceneManager.cs @@ -1,17 +1,19 @@ using Ghost.Editor.Core.Progress; using Ghost.Editor.Core.Resources; +using Ghost.Editor.Core.Serializer; using Ghost.Editor.Core.Utilities; +using Ghost.Engine.Core; using System.Text.Json; namespace Ghost.Editor.Core.SceneGraph; -public enum OpenWorldMode -{ - Single, - Additive, - AdditiveWithoutLoading -} - +/// +/// Editor-specific scene manager that uses JSON serialization for human-readable scene files. +/// +/// +/// This manager provides JSON-based serialization suitable for editor workflows. +/// Runtime applications should use SceneManager with binary serialization. +/// public static class EditorSceneManager { // TODO: Use guid keys instead of string paths for better performance and uniqueness @@ -21,6 +23,10 @@ public static class EditorSceneManager public static event Action? OnWorldLoaded; public static event Action? OnWorldUnloaded; + /// + /// Loads a scene from a JSON file. + /// + /// The path to the JSON scene file. public static async Task LoadSceneAsync(string worldPath) { if (s_loadedWorlds.ContainsKey(worldPath) @@ -33,21 +39,34 @@ public static class EditorSceneManager var progressService = EditorApplication.GetService(); progressService.ShowIndeterminateProgress("Loading world..."); + // Unload existing scenes foreach (var world in s_loadedWorlds) { world.Value.Unload(); OnWorldUnloaded?.Invoke(world.Value); } + s_loadedWorlds.Clear(); + + // TODO: Get or create World instance for editor + // For now, keep compatibility with old SceneNode deserialization await using var readStream = new FileStream(worldPath, FileMode.Open, FileAccess.Read, FileShare.Read); var deserializedScene = await JsonSerializer.DeserializeAsync(readStream, Engine.Resources.EngineResource.defaultSerializerOptions) ?? throw new Exception("Deserialization failed."); - s_loadedWorlds.Clear(); - s_loadedWorlds[worldPath] = deserializedScene; await deserializedScene.LoadAsync(); progressService.HideProgress(); OnWorldLoaded?.Invoke(deserializedScene); } + + /// + /// Saves a scene to a JSON file using the new serializer. + /// + /// The scene to save. + /// The path to save the JSON scene file. + public static async Task SaveSceneAsync(Scene scene, string filePath) + { + await SceneSerializer.SaveSceneAsync(scene.World, scene.ID, filePath, scene.Name); + } } diff --git a/Ghost.Editor.Core/SceneGraph/SceneNode.cs b/Ghost.Editor.Core/SceneGraph/SceneNode.cs index 68a1413..9253d6a 100644 --- a/Ghost.Editor.Core/SceneGraph/SceneNode.cs +++ b/Ghost.Editor.Core/SceneGraph/SceneNode.cs @@ -1,17 +1,14 @@ using Ghost.Editor.Core.AssetHandle; using Ghost.Editor.Core.Inspector; using Ghost.Editor.Core.Resources; -using Ghost.Editor.Core.Serializer; using Ghost.Engine.Components; using Ghost.Engine.Core; -using Ghost.Engine.IO; using Ghost.Entities; using Microsoft.UI.Xaml; using Microsoft.UI.Xaml.Controls; namespace Ghost.Editor.Core.SceneGraph; -[CustomSerializer(typeof(SceneNodeSerializer))] public partial class SceneNode : SceneGraphNode, IEquatable { private Scene _scene; diff --git a/Ghost.Editor.Core/Serializer/Converters/EntityJsonConverter.cs b/Ghost.Editor.Core/Serializer/Converters/EntityJsonConverter.cs new file mode 100644 index 0000000..d29d8ee --- /dev/null +++ b/Ghost.Editor.Core/Serializer/Converters/EntityJsonConverter.cs @@ -0,0 +1,115 @@ +using Ghost.Entities; +using Ghost.Engine.IO; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Ghost.Editor.Core.Serializer.Converters; + +/// +/// JSON converter for Entity that handles automatic ID remapping during deserialization. +/// +/// +/// During serialization, writes the file-local entity ID. +/// During deserialization, reads the file-local ID and translates it to the runtime Entity +/// using the active SerializationContext. +/// +public class EntityJsonConverter : JsonConverter +{ + public override Entity Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + { + return Entity.Invalid; + } + + if (reader.TokenType != JsonTokenType.StartObject) + { + throw new JsonException("Expected StartObject token for Entity."); + } + + int fileId = -1; + int generation = -1; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + { + break; + } + + if (reader.TokenType == JsonTokenType.PropertyName) + { + var propertyName = reader.GetString(); + reader.Read(); + + switch (propertyName) + { + case "ID": + case "id": + fileId = reader.GetInt32(); + break; + case "Generation": + case "generation": + generation = reader.GetInt32(); + break; + } + } + } + + if (fileId == Entity.INVALID_ID) + { + return Entity.Invalid; + } + + // If we have a serialization context, remap the file ID to runtime entity + var context = SerializationContext.Current; + if (context != null) + { + if (context.TryGetEntity(fileId, out var runtimeEntity)) + { + return runtimeEntity; + } + + // If entity not found in map, return invalid + return Entity.Invalid; + } + + // No context means we're not in a deserialization scope - should not happen + throw new InvalidOperationException("Entity deserialization requires an active SerializationContext."); + } + + public override void Write(Utf8JsonWriter writer, Entity value, JsonSerializerOptions options) + { + if (!value.IsValid) + { + writer.WriteNullValue(); + return; + } + + writer.WriteStartObject(); + + // If we have a serialization context, write the file-local ID + var context = SerializationContext.Current; + if (context != null) + { + if (context.TryGetFileId(value, out var fileId)) + { + writer.WriteNumber("ID", fileId); + } + else + { + // Entity not in context - register it now + var newFileId = context.RegisterEntityForSerialization(value); + writer.WriteNumber("ID", newFileId); + } + } + else + { + // No context - write the runtime ID (for debugging or non-scene serialization) + writer.WriteNumber("ID", value.ID); + } + + writer.WriteNumber("Generation", value.Generation); + writer.WriteEndObject(); + } +} diff --git a/Ghost.Editor.Core/Serializer/SceneNodeSerializer.cs b/Ghost.Editor.Core/Serializer/SceneNodeSerializer.cs deleted file mode 100644 index 4e97ecc..0000000 --- a/Ghost.Editor.Core/Serializer/SceneNodeSerializer.cs +++ /dev/null @@ -1,148 +0,0 @@ -using Ghost.Editor.Core.SceneGraph; -using Ghost.Engine.IO; -using Ghost.Engine.Utilities; -using Ghost.Entities; -using System.Text.Json; - -namespace Ghost.Editor.Core.Serializer; - -internal class SceneNodeSerializer : CustomSerializer -{ - private static class Property - { - public const string NAME = "Name"; - public const string ENTITIES = "Entities"; - public const string ID = "ID"; - public const string ENTITY_ID = "EntityID"; - public const string COMPONENTS = "Components"; - public const string DATA = "Data"; - public const string SYSTEMS = "Systems"; - } - - public override bool CanConvert(Type typeToConvert) - { - return typeToConvert == typeof(SceneNode) || typeToConvert.IsSubclassOf(typeof(SceneNode)); - } - - public unsafe override void SerializeJson(Utf8JsonWriter writer, SceneNode value, JsonSerializerOptions options) - { - writer.WriteObject(() => - { - writer.WriteString(Property.NAME, value.Name); - writer.WriteStartArray(Property.ENTITIES); - - for (var i = 0; i < value.Scene.World.ComponentManager.ArchetypeCount; i++) - { - ref var archetype = ref value.Scene.World.ComponentManager.GetArchetypeReference(i); - - for (var j = 0; j < archetype.ChunkCount; j++) - { - ref var chunk = ref archetype.GetChunkReference(j); - for (var k = 0; k < chunk._count; k++) - { - foreach (var layout in archetype._layouts) - { - var type = ComponentRegistry.s_runtimeIDToType[layout.componentID]; - var size = ComponentRegistry.GetComponentInfo(layout.componentID).size; - - if (type.AssemblyQualifiedName == null) - { - continue; - } - - writer.WriteStartObject(); - writer.WriteString("Type", type.AssemblyQualifiedName); - writer.WritePropertyName("Data"); - - var pComponentData = chunk.GetUnsafePtr() + layout.offset + (k * size); - ComponentSerializerRegistry.SerializeJson(layout.componentID, writer, pComponentData, options); - } - } - } - } - - writer.WriteEndArray(); - - writer.WriteArray(Property.SYSTEMS, value.Scene.World.SystemManager.Systems, system => - { - var name = system.GetType().AssemblyQualifiedName; - if (name == null) - { - return; - } - - writer.WriteStringValue(name); - }); - }); - } - - public override SceneNode? DeserializeJson(ref Utf8JsonReader reader, JsonSerializerOptions options) - { - throw new NotImplementedException(); - - //var element = JsonDocument.ParseValue(ref reader).RootElement; - //var name = element.GetProperty(Property.NAME).GetString() ?? "New World"; - - //var world = World.Create(EngineCore.JobScheduler); - //var result = new WorldNode(world, name); - - //foreach (var entityElement in element.GetProperty(Property.ENTITIES).EnumerateArray()) - //{ - // var entityName = entityElement.GetProperty(Property.NAME).GetString() ?? "New Entity"; - // var entityID = entityElement.GetProperty(Property.ID).GetInt32(); - // var entity = new Entity(entityID, 0); - // var node = new EntityNode(result, entity, entityName); - - // world.EntityManager.AddEntityInternal(entity); - // result.EntityNodeLookup[entity] = node; - //} - - //foreach (var componentElement in element.GetProperty(Property.COMPONENTS).EnumerateObject()) - //{ - // var typeName = componentElement.Name; - // var space = Type.GetType(typeName) ?? throw new Exception($"Type {typeName} not found."); - - // foreach (var dataElement in componentElement.Value.EnumerateArray()) - // { - // var entityID = dataElement.GetProperty(Property.ENTITY_ID).GetInt32(); - // var entity = new Entity(entityID, 0); - - // var dataProperty = dataElement.GetProperty(Property.DATA); - // var component = JsonSerializer.Deserialize(dataProperty.GetRawText(), space, options); - // if (component is IComponent data) - // { - // world.EntityManager.AddComponent(entity, data, space); - // } - // } - //} - - //foreach (var systemElement in element.GetProperty(Property.SYSTEMS).EnumerateArray()) - //{ - // var typeString = systemElement.GetString(); - // if (string.IsNullOrEmpty(typeString)) - // { - // continue; - // } - - // var systemType = Type.GetType(typeString); - // if (systemType == null) - // { - // continue; - // } - - // world.SystemStorage.AddSystem(systemType); - //} - - //return result; - } - - public override void SerializeBinary(BinaryWriter writer, SceneNode value) - { - throw new NotImplementedException(); - } - - public override SceneNode? DeserializeBinary(BinaryReader reader) - { - throw new NotImplementedException(); - } -} diff --git a/Ghost.Editor.Core/Serializer/SceneSerializer.cs b/Ghost.Editor.Core/Serializer/SceneSerializer.cs new file mode 100644 index 0000000..4c9bd83 --- /dev/null +++ b/Ghost.Editor.Core/Serializer/SceneSerializer.cs @@ -0,0 +1,258 @@ +using Ghost.Core; +using Ghost.Editor.Core.Serializer.Converters; +using Ghost.Engine.Components; +using Ghost.Engine.Core; +using Ghost.Engine.IO; +using Ghost.Entities; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Ghost.Editor.Core.Serializer; + +/// +/// Handles JSON serialization and deserialization of scenes. +/// +public static class SceneSerializer +{ + private static readonly JsonSerializerOptions s_serializerOptions; + + static SceneSerializer() + { + s_serializerOptions = new JsonSerializerOptions + { + WriteIndented = true, + IncludeFields = true, + Converters = + { + new EntityJsonConverter(), + new JsonStringEnumConverter() + } + }; + } + + /// + /// Represents the serialized data for a single entity. + /// + private class SerializedEntity + { + public int FileID { get; set; } + public List Components { get; set; } = new(); + } + + /// + /// Represents a serialized component with its type and data. + /// + private class SerializedComponent + { + public string TypeName { get; set; } = string.Empty; + public JsonElement Data { get; set; } + } + + /// + /// Represents the complete scene file structure. + /// + private class SceneFile + { + public string Name { get; set; } = "Untitled Scene"; + public int Version { get; set; } = 1; + public List Entities { get; set; } = new(); + } + + /// + /// Saves a scene to a JSON file. + /// + /// The world containing the entities. + /// The scene ID to save. + /// The path to save the scene file. + /// Optional scene name. + public static async Task SaveSceneAsync(World world, short sceneID, string filePath, string? sceneName = null) + { + using var context = SerializationContext.Create(); + + var sceneFile = new SceneFile + { + Name = sceneName ?? Path.GetFileNameWithoutExtension(filePath), + Entities = new List() + }; + + // Query all entities with the specified SceneID + var queryID = new QueryBuilder() + .WithAll() + .Build(world); + + var entities = new List(); + + world.ComponentManager.GetEntityQueryReference(queryID).ForEach((Entity entity, ref SceneID sceneIDComponent) => + { + if (sceneIDComponent.id == sceneID) + { + entities.Add(entity); + } + }); + + // Serialize each entity + foreach (var entity in entities) + { + var fileId = context.RegisterEntityForSerialization(entity); + var serializedEntity = new SerializedEntity + { + FileID = fileId, + Components = new List() + }; + + // Get entity location to find archetype + var locationResult = world.EntityManager.GetEntityLocation(entity); + if (locationResult.Error != Error.None) + { + continue; + } + + var location = locationResult.Value; + ref var archetype = ref world.ComponentManager.GetArchetypeReference(location.archetypeID); + + // Serialize each component + foreach (var layout in archetype._layouts) + { + var componentType = ComponentRegistry.s_runtimeIDToType[layout.componentID]; + + if (componentType == null || componentType.AssemblyQualifiedName == null) + { + continue; + } + + // Get component data + unsafe + { + var pComponentData = archetype.GetComponentData(location.chunkIndex, location.rowIndex, layout.componentID); + if (pComponentData == null) + { + continue; + } + + // Serialize component to JSON + // We need to box the unmanaged data to serialize it + var boxedData = System.Runtime.InteropServices.Marshal.PtrToStructure((IntPtr)pComponentData, componentType); + var componentJson = JsonSerializer.Serialize(boxedData, componentType, s_serializerOptions); + var jsonElement = JsonDocument.Parse(componentJson).RootElement; + + serializedEntity.Components.Add(new SerializedComponent + { + TypeName = componentType.AssemblyQualifiedName, + Data = jsonElement + }); + } + } + + sceneFile.Entities.Add(serializedEntity); + } + + // Write to file + var json = JsonSerializer.Serialize(sceneFile, s_serializerOptions); + await File.WriteAllTextAsync(filePath, json); + } + + /// + /// Loads a scene from a JSON file into the specified world. + /// + /// The world to load the scene into. + /// The path to the scene file. + /// The new scene ID to assign to loaded entities. + /// The number of entities loaded. + public static async Task LoadSceneAsync(World world, string filePath, short newSceneID) + { + if (!File.Exists(filePath)) + { + throw new FileNotFoundException($"Scene file not found: {filePath}"); + } + + var json = await File.ReadAllTextAsync(filePath); + var sceneFile = JsonSerializer.Deserialize(json, s_serializerOptions); + + if (sceneFile == null) + { + throw new InvalidOperationException("Failed to deserialize scene file."); + } + + using var context = SerializationContext.Create(); + + // Pass 1: Create all entities and build the ID mapping + var fileIdToEntity = new Dictionary(); + + foreach (var serializedEntity in sceneFile.Entities) + { + var entity = world.EntityManager.CreateEntity(); + fileIdToEntity[serializedEntity.FileID] = entity; + context.RegisterEntity(serializedEntity.FileID, entity); + + // Add SceneID component + world.EntityManager.AddComponent(entity, new SceneID { id = newSceneID }); + } + + // Pass 2: Deserialize components (with automatic entity reference remapping) + foreach (var serializedEntity in sceneFile.Entities) + { + if (!fileIdToEntity.TryGetValue(serializedEntity.FileID, out var entity)) + { + continue; + } + + foreach (var serializedComponent in serializedEntity.Components) + { + var componentType = Type.GetType(serializedComponent.TypeName); + if (componentType == null) + { + continue; + } + + // Skip SceneID as we already added it + if (componentType == typeof(SceneID)) + { + continue; + } + + try + { + // Deserialize the component data + var componentData = JsonSerializer.Deserialize(serializedComponent.Data.GetRawText(), componentType, s_serializerOptions); + + if (componentData == null) + { + continue; + } + + // Add component to entity + unsafe + { + var componentID = ComponentRegistry.GetComponentID(componentType); + if (componentID.IsInvalid) + { + continue; + } + + // For unmanaged components, we can use pointer magic + if (componentType.IsValueType) + { + var pinnedData = System.Runtime.InteropServices.GCHandle.Alloc(componentData, System.Runtime.InteropServices.GCHandleType.Pinned); + try + { + var ptr = pinnedData.AddrOfPinnedObject().ToPointer(); + world.EntityManager.AddComponent(entity, componentID, ptr); + } + finally + { + pinnedData.Free(); + } + } + } + } + catch (Exception ex) + { + // Log error but continue loading other components + Console.WriteLine($"Failed to deserialize component {serializedComponent.TypeName}: {ex.Message}"); + } + } + } + + return fileIdToEntity.Count; + } +} diff --git a/Ghost.Engine/Core/Scene.cs b/Ghost.Engine/Core/Scene.cs index 72a969e..ac65523 100644 --- a/Ghost.Engine/Core/Scene.cs +++ b/Ghost.Engine/Core/Scene.cs @@ -2,25 +2,67 @@ using Ghost.Entities; namespace Ghost.Engine.Core; -public partial class Scene +/// +/// Represents a lightweight handle to a loaded scene. +/// +/// +/// A Scene is a collection of entities tagged with a unique SceneID component. +/// The Scene class provides a convenient handle to interact with all entities +/// belonging to a particular scene within a World. +/// +public sealed class Scene : IDisposable, IEquatable { private static short s_nextSceneID = 0; -} -public partial class Scene : IDisposable -{ private readonly World _world; private readonly short _id; - + private readonly string _name; private bool _isDisposed; + /// + /// Gets the world this scene belongs to. + /// public World World => _world; + + /// + /// Gets the unique identifier for this scene. + /// public short ID => _id; - public Scene(World world) + /// + /// Gets the name of this scene. + /// + public string Name => _name; + + /// + /// Creates a new scene handle. + /// + /// The world this scene belongs to. + /// The name of the scene. + internal Scene(World world, string name) { _world = world; _id = s_nextSceneID++; + _name = name; + } + + /// + /// Creates a new scene handle with a specific ID. + /// + /// The world this scene belongs to. + /// The scene ID. + /// The name of the scene. + internal Scene(World world, short id, string name) + { + _world = world; + _id = id; + _name = name; + + // Update next ID if necessary + if (id >= s_nextSceneID) + { + s_nextSceneID = (short)(id + 1); + } } ~Scene() @@ -28,6 +70,50 @@ public partial class Scene : IDisposable Dispose(); } + public bool Equals(Scene? other) + { + if (other is null) + { + return false; + } + + if (ReferenceEquals(this, other)) + { + return true; + } + + return _world.Equals(other._world) && _id == other._id; + } + + public override bool Equals(object? obj) + { + return obj is Scene other && Equals(other); + } + + public override int GetHashCode() + { + return HashCode.Combine(_world, _id); + } + + public override string ToString() + { + return $"Scene: {_name} (ID: {_id})"; + } + + public static bool operator ==(Scene? left, Scene? right) + { + if (left is null) + { + return right is null; + } + return left.Equals(right); + } + + public static bool operator !=(Scene? left, Scene? right) + { + return !(left == right); + } + public void Dispose() { if (_isDisposed) diff --git a/Ghost.Engine/IO/SceneBinarySerializer.cs b/Ghost.Engine/IO/SceneBinarySerializer.cs new file mode 100644 index 0000000..58f7fa0 --- /dev/null +++ b/Ghost.Engine/IO/SceneBinarySerializer.cs @@ -0,0 +1,256 @@ +using Ghost.Core; +using Ghost.Engine.Components; +using Ghost.Engine.Core; +using Ghost.Entities; +using Misaki.HighPerformance.LowLevel.Utilities; + +namespace Ghost.Engine.IO; + +/// +/// Handles binary serialization and deserialization of scenes for AOT-compatible runtime use. +/// +/// +/// Binary format provides fast, compact scene loading suitable for AOT compilation. +/// Uses direct memory copying for component data without reflection. +/// +public static unsafe class SceneBinarySerializer +{ + private const int MAGIC_NUMBER = 0x47534345; // "GSCE" (Ghost Scene) + private const int VERSION = 1; + + /// + /// Saves a scene to a binary file. + /// + /// The world containing the entities. + /// The scene ID to save. + /// The path to save the scene file. + public static void SaveScene(World world, short sceneID, string filePath) + { + using var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write); + using var writer = new BinaryWriter(stream); + using var context = SerializationContext.Create(); + + // Write header + writer.Write(MAGIC_NUMBER); + writer.Write(VERSION); + writer.Write(sceneID); + + // Query all entities with the specified SceneID + var queryID = new QueryBuilder() + .WithAll() + .Build(world); + + var entities = new List(); + + world.ComponentManager.GetEntityQueryReference(queryID).ForEach((Entity entity, ref SceneID sceneIDComponent) => + { + if (sceneIDComponent.id == sceneID) + { + entities.Add(entity); + } + }); + + // Write entity count + writer.Write(entities.Count); + + // Allocate buffer for zero-filled component data (reused across loop iterations) + const int MaxComponentSize = 4096; // Reasonable max size for most components + var zeroBuffer = stackalloc byte[MaxComponentSize]; + MemoryUtility.MemSet(zeroBuffer, 0, MaxComponentSize); + + // Write each entity + foreach (var entity in entities) + { + var fileId = context.RegisterEntityForSerialization(entity); + + // Write entity file ID + writer.Write(fileId); + + // Get entity location + var locationResult = world.EntityManager.GetEntityLocation(entity); + if (locationResult.Error != Error.None) + { + // Write 0 components for invalid entity + writer.Write(0); + continue; + } + + var location = locationResult.Value; + ref var archetype = ref world.ComponentManager.GetArchetypeReference(location.archetypeID); + + // Write component count + writer.Write(archetype._layouts.Count); + + // Write each component + foreach (var layout in archetype._layouts) + { + // Write component type ID + writer.Write((int)layout.componentID); + + // Write component size + writer.Write(layout.size); + + // Get component data pointer + var pComponentData = archetype.GetComponentData(location.chunkIndex, location.rowIndex, layout.componentID); + if (pComponentData == null) + { + // Write zero-filled data if component not found + if (layout.size > MaxComponentSize) + { + throw new InvalidOperationException($"Component size {layout.size} exceeds maximum buffer size {MaxComponentSize}"); + } + writer.Write(new ReadOnlySpan(zeroBuffer, layout.size)); + } + else + { + // Write component data directly + writer.Write(new ReadOnlySpan(pComponentData, layout.size)); + } + } + } + } + + /// + /// Loads a scene from a binary file into the specified world. + /// + /// The world to load the scene into. + /// The path to the scene file. + /// The new scene ID to assign to loaded entities. + /// The number of entities loaded. + public static int LoadScene(World world, string filePath, short newSceneID) + { + if (!File.Exists(filePath)) + { + throw new FileNotFoundException($"Scene file not found: {filePath}"); + } + + using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read); + using var reader = new BinaryReader(stream); + using var context = SerializationContext.Create(); + + // Read and validate header + var magic = reader.ReadInt32(); + if (magic != MAGIC_NUMBER) + { + throw new InvalidDataException("Invalid scene file format."); + } + + var version = reader.ReadInt32(); + if (version != VERSION) + { + throw new InvalidDataException($"Unsupported scene file version: {version}"); + } + + var savedSceneID = reader.ReadInt16(); + + // Read entity count + var entityCount = reader.ReadInt32(); + + // Pass 1: Create all entities and build ID mapping + var fileIdToEntity = new Dictionary(entityCount); + var entityComponents = new List<(int fileId, List<(Identifier componentID, int size, byte[] data)> components)>(entityCount); + + for (var i = 0; i < entityCount; i++) + { + var fileId = reader.ReadInt32(); + var componentCount = reader.ReadInt32(); + + var components = new List<(Identifier componentID, int size, byte[] data)>(componentCount); + + // Read component data + for (var j = 0; j < componentCount; j++) + { + var componentID = new Identifier(reader.ReadInt32()); + var size = reader.ReadInt32(); + var data = reader.ReadBytes(size); + + components.Add((componentID, size, data)); + } + + entityComponents.Add((fileId, components)); + + // Create entity + var entity = world.EntityManager.CreateEntity(); + fileIdToEntity[fileId] = entity; + context.RegisterEntity(fileId, entity); + + // Add SceneID component + world.EntityManager.AddComponent(entity, new SceneID { id = newSceneID }); + } + + // Pass 2: Add components to entities (with automatic entity reference remapping) + foreach (var (fileId, components) in entityComponents) + { + if (!fileIdToEntity.TryGetValue(fileId, out var entity)) + { + continue; + } + + foreach (var (componentID, size, data) in components) + { + // Skip SceneID as we already added it + if (componentID == ComponentTypeID.Value) + { + continue; + } + + fixed (byte* pData = data) + { + // Remap Entity references in the component data + RemapEntityReferences(pData, componentID, context); + + // Add component + world.EntityManager.AddComponent(entity, componentID, pData); + } + } + } + + return fileIdToEntity.Count; + } + + /// + /// Remaps Entity references within component data. + /// + /// + /// This is a simple implementation that checks if the component contains Entity fields. + /// For Hierarchy, it remaps parent, firstChild, and nextSibling fields. + /// + private static void RemapEntityReferences(byte* pComponentData, Identifier componentID, SerializationContext context) + { + // Check if this is the Hierarchy component + if (componentID == ComponentTypeID.Value) + { + var hierarchy = (Hierarchy*)pComponentData; + + // Remap parent + if (hierarchy->parent.IsValid && context.TryGetFileId(hierarchy->parent, out var parentFileId)) + { + if (context.TryGetEntity(parentFileId, out var newParent)) + { + hierarchy->parent = newParent; + } + } + + // Remap firstChild + if (hierarchy->firstChild.IsValid && context.TryGetFileId(hierarchy->firstChild, out var firstChildFileId)) + { + if (context.TryGetEntity(firstChildFileId, out var newFirstChild)) + { + hierarchy->firstChild = newFirstChild; + } + } + + // Remap nextSibling + if (hierarchy->nextSibling.IsValid && context.TryGetFileId(hierarchy->nextSibling, out var nextSiblingFileId)) + { + if (context.TryGetEntity(nextSiblingFileId, out var newNextSibling)) + { + hierarchy->nextSibling = newNextSibling; + } + } + } + + // TODO: Add remapping for other components with Entity fields + // This could be automated using source generators in the future + } +} diff --git a/Ghost.Engine/IO/SerializationContext.cs b/Ghost.Engine/IO/SerializationContext.cs new file mode 100644 index 0000000..0d19b08 --- /dev/null +++ b/Ghost.Engine/IO/SerializationContext.cs @@ -0,0 +1,109 @@ +using Ghost.Entities; + +namespace Ghost.Engine.IO; + +/// +/// Provides a thread-safe context for Entity ID remapping during deserialization. +/// +/// +/// This class manages the mapping between file-local entity IDs (0, 1, 2...) +/// and runtime entity IDs during scene deserialization. The context is scoped +/// to the current async operation using AsyncLocal storage. +/// +public sealed class SerializationContext : IDisposable +{ + private static readonly AsyncLocal s_current = new(); + + private readonly Dictionary _fileIdToEntity = new(); + private readonly Dictionary _entityToFileId = new(); + private int _nextFileId = 0; + private bool _disposed = false; + + /// + /// Gets the current serialization context for this async operation. + /// + public static SerializationContext? Current => s_current.Value; + + private SerializationContext() + { + } + + /// + /// Creates and activates a new serialization context for the current async scope. + /// + /// A new serialization context. Must be disposed when done. + public static SerializationContext Create() + { + if (s_current.Value != null) + { + throw new InvalidOperationException("A serialization context is already active in this scope."); + } + + var context = new SerializationContext(); + s_current.Value = context; + return context; + } + + /// + /// Registers an entity mapping for deserialization. + /// + /// The file-local entity ID. + /// The runtime entity. + public void RegisterEntity(int fileId, Entity runtimeEntity) + { + _fileIdToEntity[fileId] = runtimeEntity; + _entityToFileId[runtimeEntity] = fileId; + } + + /// + /// Registers a runtime entity and assigns it the next available file ID for serialization. + /// + /// The runtime entity to register. + /// The assigned file-local ID. + public int RegisterEntityForSerialization(Entity runtimeEntity) + { + if (!_entityToFileId.TryGetValue(runtimeEntity, out var fileId)) + { + fileId = _nextFileId++; + _entityToFileId[runtimeEntity] = fileId; + _fileIdToEntity[fileId] = runtimeEntity; + } + + return fileId; + } + + /// + /// Tries to get the runtime entity for a file-local ID. + /// + /// The file-local entity ID. + /// The runtime entity if found. + /// True if the entity was found, false otherwise. + public bool TryGetEntity(int fileId, out Entity entity) + { + return _fileIdToEntity.TryGetValue(fileId, out entity); + } + + /// + /// Tries to get the file-local ID for a runtime entity. + /// + /// The runtime entity. + /// The file-local ID if found. + /// True if the file ID was found, false otherwise. + public bool TryGetFileId(Entity entity, out int fileId) + { + return _entityToFileId.TryGetValue(entity, out fileId); + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + s_current.Value = null; + _fileIdToEntity.Clear(); + _entityToFileId.Clear(); + _disposed = true; + } +} diff --git a/Ghost.Engine/IO/SerializerRegistry.cs b/Ghost.Engine/IO/SerializerRegistry.cs deleted file mode 100644 index 1daeb8d..0000000 --- a/Ghost.Engine/IO/SerializerRegistry.cs +++ /dev/null @@ -1,52 +0,0 @@ -using Ghost.Core; -using Ghost.Entities; -using System.Text.Json; - -namespace Ghost.Engine.IO; - -public static unsafe class ComponentSerializerRegistry -{ - public delegate void BinaryWriteDelegate(BinaryWriter writer, void* ptr); - public delegate void JsonWriteDelegate(Utf8JsonWriter writer, void* ptr, JsonSerializerOptions options); - - private static BinaryWriteDelegate[] s_binWriters = new BinaryWriteDelegate[64]; - private static JsonWriteDelegate[] s_jsonWriters = new JsonWriteDelegate[64]; - - public static void Register(int typeID, BinaryWriteDelegate binWriter, JsonWriteDelegate jsonWriter) - { - if (typeID < 0) - { - throw new Exception($"Type ID cannot be negative: {typeID}"); - } - - if (typeID >= s_binWriters.Length) - { - Array.Resize(ref s_binWriters, typeID + 16); - Array.Resize(ref s_jsonWriters, typeID + 16); - } - - s_binWriters[typeID] = binWriter; - s_jsonWriters[typeID] = jsonWriter; - } - - public static void SerializeBinary(Identifier typeID, BinaryWriter writer, void* ptr) - { - if (s_binWriters[typeID] == null) - { - throw new Exception($"No serializer for ID {typeID}"); - } - - s_binWriters[typeID](writer, ptr); - } - - public static void SerializeJson(Identifier typeID, Utf8JsonWriter writer, void* ptr, JsonSerializerOptions options) - { - if (s_jsonWriters[typeID] == null) - { - // TODO: Fallback to reflection? - return; - } - - s_jsonWriters[typeID](writer, ptr, options); - } -} \ No newline at end of file diff --git a/Ghost.Engine/Services/SceneManager.cs b/Ghost.Engine/Services/SceneManager.cs index a34895d..545b7e4 100644 --- a/Ghost.Engine/Services/SceneManager.cs +++ b/Ghost.Engine/Services/SceneManager.cs @@ -1,48 +1,139 @@ +using Ghost.Engine.Components; +using Ghost.Engine.Core; +using Ghost.Engine.IO; +using Ghost.Entities; +using System.Runtime.InteropServices; + namespace Ghost.Engine.Services; public enum SceneLoadMode { + /// + /// Unloads all currently loaded scenes before loading the new scene. + /// Single, + + /// + /// Loads the scene additively without unloading existing scenes. + /// Additive } +/// +/// Manages scene loading, unloading, and saving operations using binary serialization. +/// +/// +/// This runtime scene manager uses binary serialization for AOT compatibility. +/// For editor JSON serialization, use EditorSceneManager in Ghost.Editor.Core. +/// public static class SceneManager { - //private readonly static HashSet _activeScenes = new(); + private static readonly Dictionary s_loadedScenes = new(); - //internal static IEnumerable QueryRootGameObjects() - //{ - // foreach (var scene in _activeScenes) - // { - // foreach (var gameObject in scene.RootObjects) - // { - // if (!gameObject.IsActive) - // { - // continue; - // } + /// + /// Gets all currently loaded scenes. + /// + public static IReadOnlyCollection LoadedScenes => s_loadedScenes.Values; - // yield return gameObject; - // } - // } - //} + /// + /// Loads a scene from a binary file into the specified world. + /// + /// The world to load the scene into. + /// The path to the scene file. + /// The load mode (Single or Additive). + /// The loaded scene. + public static Scene LoadScene(World world, string filePath, SceneLoadMode loadMode = SceneLoadMode.Single) + { + if (loadMode == SceneLoadMode.Single) + { + // Unload all currently loaded scenes in this world + var scenesToUnload = s_loadedScenes.Values.Where(s => s.World == world).ToList(); + foreach (var scene in scenesToUnload) + { + UnloadScene(scene); + } + } - //public static void LoadScene(Scene scene, SceneLoadMode loadMode) - //{ - // if (loadMode == SceneLoadMode.Single) - // { - // foreach (var activeScene in _activeScenes) - // { - // activeScene.Unload(); - // } - // _activeScenes.Clear(); - // } + // Generate a new scene ID for this load + var sceneName = Path.GetFileNameWithoutExtension(filePath); + var newScene = new Scene(world, sceneName); - // _activeScenes.Add(scene); - // scene.Load(); - //} + // Load the scene data using binary serialization + SceneBinarySerializer.LoadScene(world, filePath, newScene.ID); - //public static Task LoadSceneAsync(Scene scene, SceneLoadMode loadMode) - //{ - // return Task.Run(() => LoadScene(scene, loadMode)); - //} + // Register the loaded scene + s_loadedScenes[newScene.ID] = newScene; + + return newScene; + } + + /// + /// Saves a scene to a binary file. + /// + /// The scene to save. + /// The path to save the scene file. + public static void SaveScene(Scene scene, string filePath) + { + SceneBinarySerializer.SaveScene(scene.World, scene.ID, filePath); + } + + /// + /// Unloads a scene, destroying all entities belonging to it. + /// + /// The scene to unload. + public static void UnloadScene(Scene scene) + { + if (!s_loadedScenes.ContainsKey(scene.ID)) + { + return; + } + + // Query all entities with the scene's ID + var queryID = new QueryBuilder() + .WithAll() + .Build(scene.World); + + var entitiesToDestroy = new List(); + + scene.World.ComponentManager.GetEntityQueryReference(queryID).ForEach((Entity entity, ref SceneID sceneIDComponent) => + { + if (sceneIDComponent.id == scene.ID) + { + entitiesToDestroy.Add(entity); + } + }); + + // Destroy all entities in this scene + scene.World.EntityManager.DestroyEntities(CollectionsMarshal.AsSpan(entitiesToDestroy)); + + // Remove from loaded scenes + s_loadedScenes.Remove(scene.ID); + + // Dispose the scene handle + scene.Dispose(); + } + + /// + /// Unloads all scenes in the specified world. + /// + /// The world whose scenes to unload. + public static void UnloadAllScenes(World world) + { + var scenesToUnload = s_loadedScenes.Values.Where(s => s.World == world).ToList(); + foreach (var scene in scenesToUnload) + { + UnloadScene(scene); + } + } + + /// + /// Tries to get a loaded scene by its ID. + /// + /// The scene ID to find. + /// The found scene, or null if not loaded. + /// True if the scene was found, false otherwise. + public static bool TryGetScene(short sceneID, out Scene? scene) + { + return s_loadedScenes.TryGetValue(sceneID, out scene); + } } \ No newline at end of file diff --git a/SCENE_SERIALIZATION.md b/SCENE_SERIALIZATION.md new file mode 100644 index 0000000..ed7feda --- /dev/null +++ b/SCENE_SERIALIZATION.md @@ -0,0 +1,291 @@ +# Scene Serialization Implementation Summary + +## Overview +Implemented a dual-format scene serialization system for GhostEngine: +- **Binary format** for runtime (AOT-compatible, fast) +- **JSON format** for editor (reflection-based, human-readable) + +Both formats support automatic Entity reference remapping. + +## Architecture + +### Two Serialization Paths + +#### **Runtime Path (Ghost.Engine)** - AOT-Compatible +``` +SceneManager → SceneBinarySerializer → SerializationContext +``` +- Binary format using direct memory operations (`memcpy`) +- No reflection, no System.Text.Json dependency +- Fast, compact, suitable for shipping builds +- Synchronous operations + +#### **Editor Path (Ghost.Editor.Core)** - Reflection-Based +``` +EditorSceneManager → SceneSerializer → EntityJsonConverter → SerializationContext +``` +- JSON format using System.Text.Json with reflection +- Human-readable, debuggable +- Automatic Entity remapping via custom converter +- Async operations + +### Core Components + +#### 1. **SerializationContext.cs** (Ghost.Engine/IO) +- **Shared by both runtime and editor** +- Thread-safe context using `AsyncLocal` for managing Entity ID remapping +- Maps file-local IDs (0, 1, 2...) to runtime Entity instances +- Bidirectional mapping for both serialization and deserialization +- Usage pattern: + ```csharp + using var context = SerializationContext.Create(); + context.RegisterEntity(fileId, runtimeEntity); + ``` + +#### 2. **SceneBinarySerializer.cs** (Ghost.Engine/IO) +- **Runtime binary serialization** - AOT-compatible +- Static utility class with synchronous methods +- **Serialize**: Writes entities to BinaryWriter using raw memory operations + - Format: Magic number (0x47534345 "GSCE"), version, entity count, component data + - Uses `memcpy` for component data - zero reflection + - Implements Entity reference remapping for Hierarchy component +- **Deserialize**: Two-pass loading strategy + - **Pass 1**: Create all entities, build ID mapping + - **Pass 2**: Read and copy component data, remap Entity references +- **RemapEntityReferences**: Manual remapping for components with Entity fields (currently Hierarchy) + +#### 3. **EntityJsonConverter.cs** (Ghost.Editor.Core/Serializer/Converters) +- **Editor-only** custom `JsonConverter` +- Automatically remaps Entity references during JSON serialization/deserialization +- During **serialization**: Writes file-local ID from SerializationContext +- During **deserialization**: Reads file-local ID and translates to runtime Entity +- Enables deep Entity reference remapping in nested components (e.g., Hierarchy) + +#### 4. **SceneSerializer.cs** (Ghost.Editor.Core/Serializer) +- **Editor-only** static utility class for JSON scene file I/O +- **SaveSceneAsync**: Queries entities by SceneID, serializes components using reflection +- **LoadSceneAsync**: Two-pass loading strategy with automatic Entity remapping + - **Pass 1**: Create all entities, build ID mapping + - **Pass 2**: Deserialize components with automatic Entity remapping via EntityJsonConverter +- File format: JSON with entities array containing component type names and data + +#### 5. **Scene.cs** (Ghost.Engine/Core) +- Lightweight handle class with World reference, SceneID, and Name +- No longer owns the World - respects "database pattern" +- Constructor: `Scene(World world, string name)` +- Implements IDisposable and IEquatable + +#### 6. **SceneManager.cs** (Ghost.Engine/Services) +- **Runtime scene lifecycle manager** - uses binary serialization +- **LoadScene**: Synchronous, loads from binary file, supports Single/Additive modes +- **SaveScene**: Synchronous, saves scene to binary file +- **UnloadScene**: Efficiently destroys all entities with matching SceneID +- Maintains registry of loaded scenes per World + +#### 7. **EditorSceneManager.cs** (Ghost.Editor.Core/SceneGraph) +- **Editor scene lifecycle manager** - uses JSON serialization +- **SaveSceneAsync**: Asynchronous, saves scene to JSON file +- Integrates with editor workflows and UI + +## Key Design Decisions + +### 1. Dual Serialization Formats +- **Binary for Runtime**: Fast, compact, AOT-compatible for shipping builds + - No reflection or System.Text.Json dependency in Ghost.Engine + - Direct memory operations using unsafe pointers + - Synchronous operations suitable for runtime loading +- **JSON for Editor**: Human-readable, debuggable, reflection-based + - Located in Ghost.Editor.Core (not in runtime path) + - Async operations for editor workflows + - Automatic Entity remapping via custom JsonConverter + +### 2. World-Centric Architecture +- World is the data container (the "database") +- Scene is a lightweight handle/view into that data +- SceneManager orchestrates the I/O and entity management +- Respects separation of concerns: World doesn't know about scenes + +### 3. Component Tagging +- Uses `SceneID` component (currently IComponent, ready for ISharedComponent upgrade) +- Each entity stores its scene membership +- Enables efficient querying and batch operations + +### 4. Entity Reference Remapping +- "Smart Serializer" strategy with two-pass loading +- File uses sequential IDs (0, 1, 2...) +- Runtime creates new Entities with different IDs +- SerializationContext handles the translation +- **Binary format**: Manual remapping in `RemapEntityReferences` method +- **JSON format**: Automatic remapping via `EntityJsonConverter` +- Works for Hierarchy and any other component with Entity fields + +### 5. AOT Compatibility +- Ghost.Engine has zero reflection-based serialization +- All JSON/reflection code isolated to Ghost.Editor.Core +- Binary serializer uses only unsafe pointers and memcpy +- Suitable for IL2CPP and NativeAOT compilation + +## Binary Format Specification + +``` +Header: + 4 bytes: Magic number (0x47534345 "GSCE") + 4 bytes: Version number (int32) + 4 bytes: Entity count (int32) + +For each entity: + 4 bytes: File ID (int32) + 4 bytes: Component count (int32) + + For each component: + 4 bytes: Component Type ID (int32) + 4 bytes: Component Size (int32) + N bytes: Raw component data (memcpy from archetype) +``` + +## Usage Examples + +### Runtime Usage (Binary) +```csharp +// Create a world +var world = World.Create(); + +// Load a scene additively (synchronous) +var scene = SceneManager.LoadScene(world, "path/to/scene.bin", SceneLoadMode.Additive); + +// Save the scene (synchronous) +SceneManager.SaveScene(scene, "path/to/scene.bin"); + +// Unload the scene +SceneManager.UnloadScene(scene); +``` + +### Editor Usage (JSON) +```csharp +// In editor code +var world = World.Create(); + +// Save scene to JSON (async) +await EditorSceneManager.SaveSceneAsync(scene, "path/to/scene.json"); + +// JSON is human-readable and can be version-controlled +``` + +## Future Optimizations + +### When ISharedComponent is Available +- Change `SceneID` from `IComponent` to `ISharedComponent` +- Entities with same SceneID will be grouped in same chunks +- Unloading becomes O(chunks) instead of O(entities) +- Can free entire memory blocks instead of individual entities + +### Entity Remapping Source Generator +- Currently `RemapEntityReferences` in `SceneBinarySerializer` manually handles Hierarchy +- Could implement a source generator to automatically detect Entity fields in all components +- Would eliminate need for manual per-component remapping code +- Pattern: `[SerializableEntity]` attribute on fields containing Entity references + +### Compression +- Binary format is uncompressed raw data +- Could add optional compression (LZ4, Zstandard) for smaller file sizes +- Trade-off: loading time vs disk space + +## Files Modified/Created + +### Created in Ghost.Engine (Runtime) +- `Ghost.Engine/IO/SerializationContext.cs` - Shared ID remapping context +- `Ghost.Engine/IO/SceneBinarySerializer.cs` - AOT-compatible binary serialization +- `Ghost.Engine/Components/SceneID.cs` - Scene tagging component + +### Created in Ghost.Editor.Core (Editor) +- `Ghost.Editor.Core/Serializer/SceneSerializer.cs` - JSON serialization (moved from Ghost.Engine) +- `Ghost.Editor.Core/Serializer/Converters/EntityJsonConverter.cs` - Entity remapping for JSON (moved from Ghost.Engine) + +### Modified +- `Ghost.Engine/Core/Scene.cs` - Refactored to lightweight handle +- `Ghost.Engine/Services/SceneManager.cs` - Runtime scene lifecycle with binary serialization +- `Ghost.Editor.Core/SceneGraph/EditorSceneManager.cs` - Editor scene lifecycle with JSON serialization + +### Deleted +- `Ghost.Engine/IO/SerializerRegistry.cs` - Obsolete ComponentSerializerRegistry +- `Ghost.Editor.Core/Serializer/SceneNodeSerializer.cs` - Obsolete + +## Implementation Notes + +### Binary Serialization Details +- Uses `BinaryWriter`/`BinaryReader` for primitive types (int, etc.) +- Component data copied with `Unsafe.CopyBlock` (memcpy equivalent) +- Stackalloc buffer reused for zero-filled missing components (prevents stack overflow) +- Entity remapping performed after all entities created (two-pass loading) + +### JSON Serialization Details +- Uses `System.Text.Json` with `JsonSerializerOptions` +- `EntityJsonConverter` registered as custom converter +- Automatic Entity field detection and remapping during deserialization +- Human-readable format suitable for version control + +### Thread Safety +- `SerializationContext` uses `AsyncLocal` for thread-safe context isolation +- Binary serializer is not thread-safe (single-threaded runtime loading) +- JSON serializer uses async methods but should not be called concurrently for same World + +### Error Handling +- Missing components write zero-filled data (graceful degradation) +- Unknown component types in JSON are skipped with warning +- Invalid Entity references remap to Entity.Null +- File format version checked on load (future-proofing) + +## Known Limitations + +1. **Manual Entity Remapping in Binary Format** + - Currently only Hierarchy component is remapped + - Other components with Entity fields need manual handling + - Solution: Implement source generator for automatic detection + +2. **Component Size Limit** + - Binary serializer uses 4KB stackalloc buffer for zero-fills + - Components larger than 4KB will throw exception if missing + - Solution: Increase MaxComponentSize constant if needed + +3. **SceneNode Integration** + - Legacy SceneNode class in Ghost.Editor.Core still exists + - May need integration with new Scene/SceneSerializer system + - Future work: Decide on SceneNode vs Scene unification + +4. **No Compression** + - Binary format is uncompressed + - Large scenes may have bigger file sizes than necessary + - Future optimization: Add LZ4/Zstandard compression layer + +5. **Managed Components** + - Current implementation assumes all IComponent types are unmanaged + - ScriptComponent and ManagedEntity may need separate handling + - Future work: Add managed reference serialization + +## Testing Recommendations + +1. **Binary Format Round-Trip** + - Create entities with various components + - Save to binary file + - Load into new World + - Verify all component data matches + +2. **Entity Reference Remapping** + - Create parent-child hierarchies + - Serialize and deserialize + - Verify parent/child Entity references updated correctly + +3. **Additive Loading** + - Load multiple scenes into same World + - Verify SceneID tagging works correctly + - Unload specific scenes and verify entities destroyed + +4. **JSON Compatibility** + - Save same scene to both JSON and binary + - Verify both formats produce equivalent results when loaded + - Test JSON editing by hand (human-readable requirement) + +5. **AOT Compatibility** + - Build Ghost.Engine with NativeAOT or IL2CPP + - Verify no reflection or dynamic code generation warnings + - Test binary serialization in AOT-compiled build