Refactor ECS core, improve thread safety, add tests

- Switch Scene IDs to ushort, update invalid value logic
- Add thread safety to SceneManager and ComponentRegistry
- Add GHOST_ZERO_INIT_COMPONENT define for editor configs
- Update mesh header counts to use int, not uint
- Add Collect methods for archetype/component cleanup
- Add With/Without/AsView to ComponentSet for easier use
- Refactor World creation/destruction, version handling
- Refactor ResourceStreamingContext to use init-only props
- Add unit tests for entity queries and world lifecycle
- Minor code cleanups and formatting fixes

src/Directory.Build.props

@@ -1,8 +1,8 @@
 <Project>
   <PropertyGroup Condition="'$(Configuration)' == 'Debug_Editor'">
-    <DefineConstants>$(DefineConstants);DEBUG;GHOST_EDITOR;</DefineConstants>
+    <DefineConstants>$(DefineConstants);DEBUG;GHOST_EDITOR;GHOST_ZERO_INIT_COMPONENT;</DefineConstants>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)' == 'Release_Editor'">
-    <DefineConstants>$(DefineConstants);GHOST_EDITOR;MHP_ENABLE_SAFETY_CHECKS;</DefineConstants>
+    <DefineConstants>$(DefineConstants);GHOST_EDITOR;GHOST_ZERO_INIT_COMPONENT;</DefineConstants>
   </PropertyGroup>
 </Project>

src/Editor/Ghost.Editor.Core/Assets/MeshAssetHandler.cs

@@ -2,6 +2,7 @@ using Ghost.Core;
 using Ghost.Core.Utilities;
 using Ghost.Editor.Core.Services;
 using Ghost.Engine;
+using Ghost.Engine.Streaming;
 using Ghost.Graphics.Core;
 using Ghost.Graphics.RHI;
 using Misaki.HighPerformance.Jobs;
@@ -384,16 +385,16 @@ internal class MeshAssetHandler : IImportableAssetHandler, IPackableAssetHandler
         {
             magic = MeshContentHeader.MAGIC,
             version = MeshContentHeader.VERSION,
-            vertexCount = (uint)geometry.Vertices.Count,
+            vertexCount = geometry.Vertices.Count,
-            indexCount = (uint)geometry.Indices.Count,
+            indexCount = geometry.Indices.Count,
-            materialPartCount = (uint)geometry.MaterialParts.Length,
+            materialPartCount = geometry.MaterialParts.Length,
-            meshletCount = (uint)meshletData.GetRef().meshlets.Count,
+            meshletCount = meshletData.GetRef().meshlets.Count,
-            meshletGroupCount = (uint)meshletData.GetRef().groups.Count,
+            meshletGroupCount = meshletData.GetRef().groups.Count,
-            meshletHierarchyNodeCount = (uint)meshletData.GetRef().hierarchyNodes.Count,
+            meshletHierarchyNodeCount = meshletData.GetRef().hierarchyNodes.Count,
-            meshletVertexCount = (uint)meshletData.GetRef().meshletVertices.Count,
+            meshletVertexCount = meshletData.GetRef().meshletVertices.Count,
-            meshletTriangleCount = (uint)meshletData.GetRef().meshletTriangles.Count,
+            meshletTriangleCount = meshletData.GetRef().meshletTriangles.Count,
-            materialSlotCount = (uint)meshletData.GetRef().materialSlotCount,
+            materialSlotCount = meshletData.GetRef().materialSlotCount,
-            lodLevelCount = (uint)meshletData.GetRef().lodLevelCount,
+            lodLevelCount = meshletData.GetRef().lodLevelCount,
             boundsMin = bounds.Min,
             boundsMax = bounds.Max,
         };

src/Editor/Ghost.Editor.Core/Assets/TextureAssetHandler.cs

@@ -1,5 +1,6 @@
 using Ghost.Core;
 using Ghost.Engine;
+using Ghost.Engine.Streaming;
 using Ghost.Graphics.RHI;
 using Ghost.StbI;
 using Misaki.HighPerformance.LowLevel;

src/Editor/Ghost.Editor.Core/Contracts/IAssetRegistry.cs

@@ -1,7 +1,6 @@
 using Ghost.Core;
 using Ghost.Editor.Core.Assets;
 using Ghost.Editor.Core.Services;
-using Ghost.Engine.AssetLoader;
 
 namespace Ghost.Editor.Core.Contracts;
 

src/Runtime/Ghost.Engine/Core/Scene.cs

@@ -10,25 +10,25 @@ namespace Ghost.Engine.Core;
 /// </summary>
 public struct Scene : IEquatable<Scene>
 {
-    public byte id;
+    public ushort id;
 
     /// <summary>
     /// Gets whether this scene is valid.
     /// </summary>
     [JsonIgnore]
-    public readonly bool IsValid => id != 255;
+    public readonly bool IsValid => id != 65535;
 
     /// <summary>
     /// Gets an invalid scene instance.
     /// </summary>
-    public static Scene Invalid => new Scene { id = 255 };
+    public static Scene Invalid => new Scene { id = 65535 };
 
     public readonly bool Equals(Scene other)
     {
         return id == other.id;
     }
 
-    public override bool Equals(object? obj)
+    public readonly override bool Equals(object? obj)
     {
         return obj is Scene other && Equals(other);
     }
@@ -63,14 +63,18 @@ public struct Scene : IEquatable<Scene>
 /// </remarks>
 public static class SceneManager
 {
-    private static byte s_nextSceneID;
+    private static ushort s_nextSceneID;
-    private static readonly Queue<byte> s_recycledSceneIDs = new();
+    private static readonly Queue<ushort> s_recycledSceneIDs = new();
+
+    private static readonly Lock s_creationLock = new();
 
     /// <summary>
     /// Creates a new scene in the world.
     /// </summary>
     /// <returns>The created scene.</returns>
     public static Scene CreateScene()
+    {
+        lock (s_creationLock)
         {
             if (!s_recycledSceneIDs.TryDequeue(out var id))
             {
@@ -79,6 +83,7 @@ public static class SceneManager
 
             return new Scene { id = id };
         }
+    }
 
     /// <summary>
     /// Destroys all entities belonging to the specified scene.

src/Runtime/Ghost.Engine/Streaming/SceneAssetEntry.cs

@@ -244,6 +244,12 @@ internal unsafe class SceneAssetEntry : ProcessableAssetEntry
     {
         // TODO: How can I get this? Ideally the public api will be something like SceneManager.LoadScene(World, Scene, SceneLoadingType).
         // Should we handle the scene loading explicitly instead of auto loading on the first resolve?
+        // For example if we have a component called SceneStreamer{ Scene a; Scene b; }
+        // In save data, we convert the Scene(int) to a asset gui, and convert it back during load. So at ResolveScene stage (before the file even been loaded), we need to call the SceneManager.CreateScene().
+        // Currently we store the world and loading type directly inside the asset entry, but actually that should not be bound with the asset itself, because we may load scene A along at the first time, then we load it additively at the second time.
+        // So, maybe the scene asset entry should only create a unique id from SceneManager.CreateScene() then resolve the scene file without loading it into world.
+        // Then we can load the scene into world using our job system, and user can decide to wait it immediatly (sync) or fire-and-forget (async).
+
         _targetWorld = World.GetWorld(0)!;
         _loadingType = SceneLoadingType.Single;
     }

src/Runtime/Ghost.Entities/Archetype.cs

@@ -703,6 +703,18 @@ internal unsafe struct Archetype : IDisposable
         return Identifier<Archetype>.Invalid;
     }
 
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public readonly void Collect()
+    {
+        for (var i = 0; i < _chunks.Count; i++)
+        {
+            if (_chunks[i]._count == 0)
+            {
+                _chunks[i].Dispose();
+            }
+        }
+    }
+
     public override readonly int GetHashCode()
     {
         return _hash;

src/Runtime/Ghost.Entities/Component.cs

@@ -1,4 +1,5 @@
 using Ghost.Core;
+using Ghost.Core.Utilities;
 using Misaki.HighPerformance.LowLevel.Buffer;
 using Misaki.HighPerformance.LowLevel.Collections;
 using Misaki.HighPerformance.LowLevel.Utilities;
@@ -43,6 +44,7 @@ internal static class ComponentRegistry
     private static readonly List<ComponentInfo> s_registeredComponents = new();
     private static readonly Dictionary<IntPtr, int> s_typeHandleToID = new();
     private static readonly Dictionary<string, int> s_nameToRuntimeID = new();
+    private static readonly Lock s_registerLock = new();
 
 #if DEBUG || GHOST_EDITOR
     internal static readonly Dictionary<int, Type> s_runtimeIDToType = new();
@@ -59,7 +61,7 @@ internal static class ComponentRegistry
         var type = typeof(T);
         var typeHandle = type.TypeHandle.Value;
 
-        lock (s_registeredComponents)
+        lock (s_registerLock)
         {
             if (s_typeHandleToID.TryGetValue(typeHandle, out var existingID))
             {
@@ -95,7 +97,7 @@ internal static class ComponentRegistry
     public static Identifier<IComponent> GetComponentID(Type type)
     {
         var typeHandle = type.TypeHandle.Value;
-        lock (s_registeredComponents)
+        lock (s_registerLock)
         {
             if (s_typeHandleToID.TryGetValue(typeHandle, out var existingID))
             {
@@ -109,7 +111,7 @@ internal static class ComponentRegistry
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     public static Identifier<IComponent> GetComponentIDByName(string fullName)
     {
-		lock (s_registeredComponents)
+        lock (s_registerLock)
         {
             if (s_nameToRuntimeID.TryGetValue(fullName, out var id))
             {
@@ -123,7 +125,7 @@ internal static class ComponentRegistry
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     public static ComponentInfo GetComponentInfo(Identifier<IComponent> typeId)
     {
-        lock (s_registeredComponents)
+        lock (s_registerLock)
         {
             return s_registeredComponents[typeId];
         }
@@ -132,7 +134,7 @@ internal static class ComponentRegistry
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     public static ComponentInfo GetComponentInfo(Type type)
     {
-        lock (s_registeredComponents)
+        lock (s_registerLock)
         {
             var typeId = GetComponentID(type);
             if (typeId.IsInvalid)
@@ -265,6 +267,16 @@ public partial class ComponentManager : IDisposable
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     internal void Clear()
     {
+        for (int i = 0; i < _archetypes.Count; i++)
+        {
+            _archetypes[i].Dispose();
+        }
+
+        for (int i = 0; i < _entityQueries.Count; i++)
+        {
+            _entityQueries[i].Dispose();
+        }
+
         _archetypes.Clear();
         _entityQueries.Clear();
 
@@ -272,6 +284,15 @@ public partial class ComponentManager : IDisposable
         _querieLookup.Clear();
     }
 
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    internal void Collect()
+    {
+        for (int i = 0; i < _archetypes.Count; i++)
+        {
+            _archetypes[i].Collect();
+        }
+    }
+
     /// <summary>
     /// Gets a reference to the entity query with the specified identifier.
     /// </summary>
@@ -326,6 +347,70 @@ public struct ComponentSet : IDisposable, IEquatable<ComponentSet>
         _hashCode = -1;
     }
 
+    public ComponentSet With(AllocationHandle allocationHandle, params ReadOnlySpan<Identifier<IComponent>> components)
+    {
+        if (_components.AsSpan().SequenceEqual(components))
+        {
+            return new ComponentSet { _components = _components.Clone(allocationHandle), _hashCode = _hashCode };
+        }
+
+        var newComponents = new UnsafeArray<Identifier<IComponent>>(_components.Length + components.Length, allocationHandle);
+
+        newComponents.CopyFrom(_components);
+
+        var i = _components.Length;
+        for (int j = 0; j < components.Length; j++)
+        {
+            for (int k = 0; k < i; k++)
+            {
+                if (newComponents[k] != components[j])
+                {
+                    newComponents[i++] = components[j];
+                }
+            }
+        }
+
+        _components.Resize(i);
+        return new ComponentSet { _components = newComponents, _hashCode = -1 };
+    }
+
+    public ComponentSet Without(AllocationHandle allocationHandle, params ReadOnlySpan<Identifier<IComponent>> components)
+    {
+        if (_components.AsSpan().SequenceEqual(components))
+        {
+            return default;
+        }
+
+        var newComponents = new UnsafeArray<Identifier<IComponent>>(_components.Length, allocationHandle);
+
+        var i = 0;
+        for (int j = 0; j < _components.Length; j++)
+        {
+            var found = false;
+            for (var k = 0; k < components.Length; k++)
+            {
+                if (components[k] == _components[j])
+                {
+                    found = true;
+                    break;
+                }
+            }
+
+            if (!found)
+            {
+                newComponents[i++] = _components[j];
+            }
+        }
+
+        newComponents.Resize(i);
+        return new ComponentSet { _components = newComponents, _hashCode = -1 };
+    }
+
+    public readonly ComponentSetView AsView()
+    {
+        return new ComponentSetView(_components);
+    }
+
     public readonly bool Equals(ComponentSet other)
     {
         return _hashCode == other._hashCode;
@@ -363,7 +448,7 @@ public struct ComponentSet : IDisposable, IEquatable<ComponentSet>
 
     public static implicit operator ComponentSetView(in ComponentSet set)
     {
-        return new ComponentSetView(set.Components);
+        return new ComponentSetView(set._components);
     }
 }
 

src/Runtime/Ghost.Entities/EntityManager.cs

@@ -3,7 +3,6 @@ using Misaki.HighPerformance.Collections;
 using Misaki.HighPerformance.LowLevel.Buffer;
 using Misaki.HighPerformance.LowLevel.Collections;
 using Misaki.HighPerformance.LowLevel.Utilities;
-using System.Diagnostics;
 using System.Runtime.CompilerServices;
 
 namespace Ghost.Entities;
@@ -47,6 +46,7 @@ public unsafe partial class EntityManager : IDisposable
     private bool _disposed;
 
     public World World => _world;
+    public int EntityCount => _entityLocations.Count;
 
     internal EntityManager(World world, int initialCapacity)
     {

src/Runtime/Ghost.Entities/World.cs

@@ -15,8 +15,6 @@ public partial class World
     public static int WorldCount => s_worlds.Count - s_freeWorldSlots.Count;
 
     public static World Create(JobScheduler? jobScheduler = null, int entityCapacity = 16)
-    {
-        lock (s_worlds)
     {
         if (s_freeWorldSlots.TryDequeue(out var index))
         {
@@ -30,11 +28,8 @@ public partial class World
 
         return s_worlds[index.Value]!;
     }
-    }
 
     public static void Destroy(Identifier<World> id)
-    {
-        lock (s_worlds)
     {
         if (id.Value < 0 || id.Value >= s_worlds.Count)
         {
@@ -44,6 +39,10 @@ public partial class World
         var world = s_worlds[id.Value];
         world?.Dispose();
     }
+
+    public static void Destroy(World world)
+    {
+        world.Dispose();
     }
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -119,7 +118,7 @@ public partial class World : IDisposable, IEquatable<World>
     /// <summary>
     /// Gets the current version number of the world.
     /// </summary>
-    public uint Version => Interlocked.CompareExchange(ref _version, 0, 0);
+    public uint Version => Volatile.Read(ref _version);
 
     /// <summary>
     /// Gets the main entity command buffer for this world.
@@ -174,7 +173,7 @@ public partial class World : IDisposable, IEquatable<World>
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     internal uint AdvanceVersion()
     {
-        return Interlocked.Increment(ref _version);
+        return _version++;
     }
 
     /// <summary>
@@ -305,11 +304,10 @@ public partial class World : IDisposable, IEquatable<World>
         _componentManager.Dispose();
         _systemManager.Dispose();
 
-        s_freeWorldSlots.Enqueue(_id);
         s_worlds[_id] = null;
+        s_freeWorldSlots.Enqueue(_id);
 
         _disposed = true;
-
         GC.SuppressFinalize(this);
     }
 }

src/Runtime/Ghost.Graphics/IResourceStreamingProcessor.cs

@@ -7,35 +7,27 @@ internal struct ResourceStreamingContext
 {
     public AsyncCopyPipeline CopyPipeline
     {
-        get;
+        get; init;
     }
 
     public ResourceManager ResourceManager
     {
-        get;
+        get; init;
     }
 
     public IResourceDatabase ResourceDatabase
     {
-        get;
+        get; init;
     }
 
     public IResourceAllocator ResourceAllocator
     {
-        get;
+        get; init;
     }
 
     public ICommandBuffer CommandBuffer
     {
-        get; set;
+        get; init;
-    } = null!;
-
-    internal ResourceStreamingContext(AsyncCopyPipeline copyPipeline, ResourceManager resourceManager, IResourceDatabase resourceDatabase, IResourceAllocator resourceAllocator)
-    {
-        CopyPipeline = copyPipeline;
-        ResourceManager = resourceManager;
-        ResourceDatabase = resourceDatabase;
-        ResourceAllocator = resourceAllocator;
     }
 }
 

src/Runtime/Ghost.Graphics/RenderSystem.cs

@@ -244,14 +244,6 @@ public class RenderSystem : IDisposable
 
         var waitHandles = new WaitHandle[] { null!, _shutdownEvent };
 
-        var streamingContext = new ResourceStreamingContext
-        (
-            _asyncCopyPipeline,
-            _resourceManager,
-            _graphicsEngine.ResourceDatabase,
-            _graphicsEngine.ResourceAllocator
-        );
-
         var renderContext = new RenderContext
         (
             _resourceManager,
@@ -321,11 +313,19 @@ public class RenderSystem : IDisposable
                 {
                     cmd.Begin(frameResource.CommandAllocator);
 
-                    streamingContext.CommandBuffer = cmd;
+                    var streamingContext = new ResourceStreamingContext
-                    renderContext.CommandBuffer = cmd;
+                    {
+                        ResourceManager = _resourceManager,
+                        ResourceDatabase = _graphicsEngine.ResourceDatabase,
+                        ResourceAllocator = _graphicsEngine.ResourceAllocator,
+                        CopyPipeline = _asyncCopyPipeline,
+                        CommandBuffer = cmd,
+                    };
 
                     _streamingProcessor.ProcessPendingUploads(streamingContext);
 
+                    renderContext.CommandBuffer = cmd;
+
                     _renderPipeline.Render(renderContext, frameIndex, frameResource.RenderPayload);
                     _swapChainManager.TransitionToPresent(cmd);
 

src/Test/Ghost.UnitTest/ECS/EntityQueryTests.cs

@@ -0,0 +1,71 @@
+using Ghost.Engine.Components;
+using Ghost.Entities;
+using Misaki.HighPerformance.LowLevel.Buffer;
+
+namespace Ghost.UnitTest.ECS;
+
+[TestClass]
+[DoNotParallelize]
+public class EntityQueryTests
+{
+    private struct Enableable : IEnableableComponent
+    {
+    }
+
+    private World _world = null!;
+
+    [TestInitialize]
+    public void Setup()
+    {
+        AllocationManager.Initialize();
+        _world = World.Create(null, 64);
+    }
+
+    [TestCleanup]
+    public void Cleanup()
+    {
+        _world.Dispose();
+        AllocationManager.Dispose();
+    }
+
+    private void CreateDefaultEntities(int count)
+    {
+        var set = new ComponentSetView([ComponentTypeID<MeshInstance>.Value, ComponentTypeID<LocalToWorld>.Value]);
+        _world.EntityManager.CreateEntities(count, set);
+    }
+
+    [TestMethod]
+    public void ComponentSetCreation()
+    {
+        using var set1 = new ComponentSet(AllocationHandle.Persistent, ComponentTypeID<MeshInstance>.Value, ComponentTypeID<LocalToWorld>.Value);
+        Assert.AreEqual(2, set1.Components.Length);
+
+        using var set2 = set1.With(AllocationHandle.Persistent, ComponentTypeID<Camera>.Value);
+        Assert.AreEqual(3, set2.Components.Length);
+        Assert.IsTrue(set2.Components.Contains(ComponentTypeID<Camera>.Value));
+
+        using var set3 = set2.Without(AllocationHandle.Persistent, ComponentTypeID<MeshInstance>.Value);
+        Assert.AreEqual(2, set3.Components.Length);
+        Assert.IsFalse(set3.Components.Contains(ComponentTypeID<MeshInstance>.Value));
+    }
+
+    [TestMethod]
+    public void SimpleQuery_EntityCountShouldEqual()
+    {
+        CreateDefaultEntities(100);
+
+        var queryID = QueryBuilder.New()
+            .WithAll<LocalToWorld, MeshInstance>()
+            .Build(_world);
+
+        ref readonly var query = ref _world.ComponentManager.GetEntityQueryReference(queryID);
+
+        var i = 0;
+        foreach (var (entity, ltw, mesh) in query.GetEntityComponentIterator<LocalToWorld, MeshInstance>())
+        {
+            i++;
+        }
+
+        Assert.AreEqual(100, i);
+    }
+}

src/Test/Ghost.UnitTest/ECS/WorldTests.cs

@@ -0,0 +1,28 @@
+using Ghost.Entities;
+
+namespace Ghost.UnitTest.ECS;
+
+[TestClass]
+[DoNotParallelize]
+public class WorldTests
+{
+    [TestMethod]
+    public void CreateWorld()
+    {
+        using var world = World.Create();
+        Assert.IsNotNull(world);
+    }
+
+    [TestMethod]
+    public void AddEntityThenClearWorld()
+    {
+        using var world =  World.Create();
+        Assert.IsNotNull(world);
+
+        world.EntityManager.CreateEntity();
+        Assert.AreEqual(1, world.EntityManager.EntityCount);
+
+        world.Clear(default);
+        Assert.AreEqual(0, world.EntityManager.EntityCount);
+    }
+}

src/Test/Ghost.UnitTest/MockingEnvironment/MockingContentProvider.cs

@@ -1,12 +1,11 @@
 using Ghost.Core;
 using Ghost.Core.Utilities;
-using Ghost.Engine;
+using Ghost.Engine.Streaming;
 using Ghost.Graphics.Core;
 using Ghost.Graphics.RHI;
 using Misaki.HighPerformance.Mathematics;
 using Misaki.HighPerformance.Mathematics.Geometry;
 using System.Collections.Concurrent;
-using System.Runtime.InteropServices;
 
 namespace Ghost.UnitTest.MockingEnvironment;
 
@@ -130,14 +129,14 @@ internal class MockingContentProvider : IContentProvider
         {
             magic = MeshContentHeader.MAGIC,
             version = MeshContentHeader.VERSION,
-            vertexCount = (uint)vertices.Length,
+            vertexCount = vertices.Length,
-            indexCount = (uint)indices.Length,
+            indexCount = indices.Length,
-            materialPartCount = (uint)materialParts.Length,
+            materialPartCount = materialParts.Length,
-            meshletCount = (uint)meshlets.Length,
+            meshletCount = meshlets.Length,
-            meshletGroupCount = (uint)groups.Length,
+            meshletGroupCount = groups.Length,
-            meshletHierarchyNodeCount = (uint)hierarchy.Length,
+            meshletHierarchyNodeCount = hierarchy.Length,
-            meshletVertexCount = (uint)meshletVertices.Length,
+            meshletVertexCount = meshletVertices.Length,
-            meshletTriangleCount = (uint)meshletTriangles.Length,
+            meshletTriangleCount = meshletTriangles.Length,
             materialSlotCount = 1,
             lodLevelCount = 1,
             boundsMin = new float3(0, 0, 0),