From e71851550b0a671b08f476c53b1c1f7caa8a2697 Mon Sep 17 00:00:00 2001 From: Misaki Date: Thu, 29 Jan 2026 14:03:24 +0900 Subject: [PATCH] Update asset database --- Ghost.Core/Result.cs | 21 + .../AssetHandle/AssetDatabase.FileOps.cs | 112 ++--- .../AssetHandle/AssetDatabase.Importer.cs | 56 ++- .../AssetHandle/AssetDatabase.Lookup.cs | 20 +- .../AssetHandle/AssetDatabase.Meta.cs | 143 +----- .../AssetHandle/AssetDatabase.SQLite.cs | 113 +---- .../AssetHandle/AssetDatabase.cs | 449 +++++++++++++++--- .../AssetHandle/AssetDatabase_Architecture.md | 115 +++++ .../AssetDatabase_Documentation.md | 131 +++++ .../AssetHandle/{ => Models}/Asset.cs | 0 .../AssetHandle/{ => Models}/TextureAsset.cs | 0 Ghost.Editor.Core/AssetHandle/README.md | 250 ---------- .../SceneGraph/SceneGraph Plan.md | 4 +- Ghost.Editor.Core/SceneGraph/SceneNode.cs | 11 +- Ghost.Editor.Core/Utilities/TypeCache.cs | 2 +- .../AssetDatabaseIntegrationTest.cs | 98 ++-- 16 files changed, 879 insertions(+), 646 deletions(-) create mode 100644 Ghost.Editor.Core/AssetHandle/AssetDatabase_Architecture.md create mode 100644 Ghost.Editor.Core/AssetHandle/AssetDatabase_Documentation.md rename Ghost.Editor.Core/AssetHandle/{ => Models}/Asset.cs (100%) rename Ghost.Editor.Core/AssetHandle/{ => Models}/TextureAsset.cs (100%) delete mode 100644 Ghost.Editor.Core/AssetHandle/README.md diff --git a/Ghost.Core/Result.cs b/Ghost.Core/Result.cs index b611c16..9b21761 100644 --- a/Ghost.Core/Result.cs +++ b/Ghost.Core/Result.cs @@ -383,4 +383,25 @@ public static class ResultExtensions return result; } + + public static Result Then(this Result result, Func> func) + { + if (result.IsFailure) + { + return Result.Failure(result.Message); + } + + return func(result.Value); + } + + public static Result Then(this Result result, Func> func) + where E : struct, Enum + { + if (result.IsFailure) + { + return Result.Failure(result.Error); + } + + return func(result.Value); + } } diff --git a/Ghost.Editor.Core/AssetHandle/AssetDatabase.FileOps.cs b/Ghost.Editor.Core/AssetHandle/AssetDatabase.FileOps.cs index 08f5150..258e079 100644 --- a/Ghost.Editor.Core/AssetHandle/AssetDatabase.FileOps.cs +++ b/Ghost.Editor.Core/AssetHandle/AssetDatabase.FileOps.cs @@ -11,7 +11,7 @@ public static partial class AssetDatabase /// Path to create the asset at. /// Content to write to the asset file. /// Result indicating success or failure. - public static async Task CreateAssetAsync(string assetPath, byte[] content) + public static async ValueTask CreateAssetAsync(string assetPath, ReadOnlyMemory content, CancellationToken token = default) { if (AssetsDirectory == null) { @@ -36,11 +36,12 @@ public static partial class AssetDatabase Directory.CreateDirectory(directory); } - await File.WriteAllBytesAsync(assetPath, content); + using var fs = File.Create(assetPath); + await fs.WriteAsync(content, token); // GenerateMetaFileAsync will be called automatically by the file watcher // But we'll call it directly to ensure it's created immediately - await GenerateMetaFileAsync(assetPath); + await GenerateMetaFileAsync(assetPath, token); return Result.Success(); } @@ -56,9 +57,9 @@ public static partial class AssetDatabase /// /// Path to create the asset at. /// Result indicating success or failure. - public static async Task CreateAssetAsync(string assetPath) + public static ValueTask CreateAssetAsync(string assetPath, CancellationToken token = default) { - return await CreateAssetAsync(assetPath, Array.Empty()); + return CreateAssetAsync(assetPath, ReadOnlyMemory.Empty, token); } /// @@ -66,7 +67,7 @@ public static partial class AssetDatabase /// /// GUID of the asset to delete. /// Result indicating success or failure. - public static async Task DeleteAssetAsync(Guid guid) + public static async ValueTask DeleteAssetAsync(Guid guid, CancellationToken token = default) { var pathResult = GuidToPath(guid); if (pathResult.IsFailure) @@ -98,7 +99,7 @@ public static partial class AssetDatabase } // Remove from database - await RemoveAssetFromDatabaseAsync(guid); + await RemoveAssetFromDatabaseAsync(guid, token); return Result.Success(); } @@ -113,15 +114,15 @@ public static partial class AssetDatabase /// /// Path to the asset to delete. /// Result indicating success or failure. - public static async Task DeleteAssetAsync(string assetPath) + public static ValueTask DeleteAssetAsync(string assetPath, CancellationToken token = default) { var guidResult = PathToGuid(assetPath); if (guidResult.IsFailure) { - return Result.Failure(guidResult.Message); + return new ValueTask(Task.FromResult(Result.Failure(guidResult.Message))); } - return await DeleteAssetAsync(guidResult.Value); + return DeleteAssetAsync(guidResult.Value, token); } /// @@ -130,7 +131,7 @@ public static partial class AssetDatabase /// GUID of the asset to move. /// New path for the asset (relative or absolute). /// Result indicating success or failure. - public static async Task MoveAssetAsync(Guid guid, string newPath) + public static async ValueTask MoveAssetAsync(Guid guid, string newPath, CancellationToken token = default) { var oldPathResult = GuidToPath(guid); if (oldPathResult.IsFailure) @@ -174,45 +175,27 @@ public static partial class AssetDatabase } // Read metadata and calculate hash before moving - var metaResult = await ReadMetaFileAsync(oldFullPathResult.Value); + var metaResult = await ReadMetaFileAsync(oldFullPathResult.Value, token); if (metaResult.IsFailure) { return Result.Failure(metaResult.Message); } - var fileHash = await CalculateFileHashAsync(oldFullPathResult.Value); + var fileHash = await CalculateFileHashAsync(oldFullPathResult.Value, token); - // Temporarily disable file watcher to prevent race conditions - var watcherWasEnabled = s_watcher?.EnableRaisingEvents ?? false; - if (s_watcher != null) + // Move the asset file + File.Move(oldFullPathResult.Value, newPath); + + // Move the .gmeta file + var oldMetaPath = oldFullPathResult.Value + Utilities.FileExtensions.META_FILE_EXTENSION; + var newMetaPath = newPath + Utilities.FileExtensions.META_FILE_EXTENSION; + if (File.Exists(oldMetaPath)) { - s_watcher.EnableRaisingEvents = false; + File.Move(oldMetaPath, newMetaPath); } - try - { - // Move the asset file - File.Move(oldFullPathResult.Value, newPath); - - // Move the .gmeta file - var oldMetaPath = oldFullPathResult.Value + Utilities.FileExtensions.META_FILE_EXTENSION; - var newMetaPath = newPath + Utilities.FileExtensions.META_FILE_EXTENSION; - if (File.Exists(oldMetaPath)) - { - File.Move(oldMetaPath, newMetaPath); - } - - // Update database with new path (hash remains the same since content didn't change) - await UpsertAssetAsync(newPath, metaResult.Value, fileHash); - } - finally - { - // Re-enable file watcher - if (s_watcher != null && watcherWasEnabled) - { - s_watcher.EnableRaisingEvents = true; - } - } + // Update database directly (bypassing file watcher) + await UpsertAssetAsync(newPath, metaResult.Value, fileHash, null, token); return Result.Success(); } @@ -228,15 +211,15 @@ public static partial class AssetDatabase /// Current path of the asset. /// New path for the asset (relative or absolute). /// Result indicating success or failure. - public static async Task MoveAssetAsync(string oldPath, string newPath) + public static ValueTask MoveAssetAsync(string oldPath, string newPath, CancellationToken token = default) { var guidResult = PathToGuid(oldPath); if (guidResult.IsFailure) { - return Result.Failure(guidResult.Message); + return ValueTask.FromResult(Result.Failure(guidResult.Message)); } - - return await MoveAssetAsync(guidResult.Value, newPath); + + return MoveAssetAsync(guidResult.Value, newPath, token); } /// @@ -245,7 +228,7 @@ public static partial class AssetDatabase /// GUID of the asset to copy. /// New path for the copied asset (relative or absolute). /// Result containing the new asset's GUID. - public static async Task> CopyAssetAsync(Guid guid, string newPath) + public static async ValueTask> CopyAssetAsync(Guid guid, string newPath, CancellationToken token = default) { var oldPathResult = GuidToPath(guid); if (oldPathResult.IsFailure) @@ -288,10 +271,12 @@ public static partial class AssetDatabase Directory.CreateDirectory(directory); } - File.Copy(oldFullPathResult.Value, newPath); + await using var oldFs = File.OpenRead(oldFullPathResult.Value); + await using var newFs = File.Create(newPath); + await oldFs.CopyToAsync(newFs, token); // Generate new metadata with new GUID - await GenerateMetaFileAsync(newPath); + await GenerateMetaFileAsync(newPath, token); // Get the new GUID var newGuidResult = PathToGuid(newPath); @@ -314,47 +299,54 @@ public static partial class AssetDatabase /// Path of the asset to copy. /// New path for the copied asset (relative or absolute). /// Result containing the new asset's GUID. - public static async Task> CopyAssetAsync(string sourcePath, string destPath) + public static ValueTask> CopyAssetAsync(string sourcePath, string destPath, CancellationToken token = default) { var guidResult = PathToGuid(sourcePath); if (guidResult.IsFailure) { - return Result.Failure(guidResult.Message); + return new ValueTask>(Task.FromResult(Result.Failure(guidResult.Message))); } - return await CopyAssetAsync(guidResult.Value, destPath); + return CopyAssetAsync(guidResult.Value, destPath, token); } /// - /// Mark an asset as dirty for re-importing. + /// Mark an asset as dirty for re-importing (in-memory only). /// /// GUID of the asset to mark dirty. /// Result indicating success or failure. - public static async Task MarkDirtyAsync(Guid guid) + public static Result MarkDirtyAsync(Guid guid, CancellationToken token = default) { - return await MarkAssetDirtyAsync(guid, true); + MarkDirty(guid); + return Result.Success(); } /// /// Import all dirty assets. /// /// Result indicating success or failure. - public static async Task ImportDirtyAssetsAsync() + public static async Task ImportDirtyAssetsAsync(CancellationToken token = default) { - var dirtyAssets = await GetDirtyAssetsAsync(); + var dirtyGuids = GetDirtyAssets(); - foreach (var (guid, path) in dirtyAssets) + foreach (var guid in dirtyGuids) { - var fullPathResult = GetFullPath(path); + var pathResult = GuidToPath(guid); + if (pathResult.IsFailure) + { + continue; + } + + var fullPathResult = GetFullPath(pathResult.Value); if (fullPathResult.IsFailure) { continue; } - var result = await ImportAssetAsync(fullPathResult.Value); + var result = await ImportAssetAsync(fullPathResult.Value, token); if (result.IsSuccess) { - await MarkAssetDirtyAsync(guid, false); + ClearDirty(guid); } } diff --git a/Ghost.Editor.Core/AssetHandle/AssetDatabase.Importer.cs b/Ghost.Editor.Core/AssetHandle/AssetDatabase.Importer.cs index 0b24451..780b754 100644 --- a/Ghost.Editor.Core/AssetHandle/AssetDatabase.Importer.cs +++ b/Ghost.Editor.Core/AssetHandle/AssetDatabase.Importer.cs @@ -12,7 +12,7 @@ public static partial class AssetDatabase /// /// Full path to the asset file. /// Result indicating success or failure. - private static async Task ImportAssetAsync(string assetPath) + private static async ValueTask ImportAssetAsync(string assetPath, CancellationToken token = default) { var extension = Path.GetExtension(assetPath); @@ -35,13 +35,15 @@ public static partial class AssetDatabase } // Read metadata - var metaResult = await ReadMetaFileAsync(assetPath); + var metaResult = await ReadMetaFileAsync(assetPath, token); if (metaResult.IsFailure) { return Result.Failure($"Failed to read asset metadata: {metaResult.Message}"); } - // Find and invoke the ImportAsync method + // TODO: Avoid reflection. + // Find and invoke the ImportAsync method. Support importers that accept (string, AssetMeta) + // or (string, AssetMeta, CancellationToken). var importMethod = importerType.GetMethod("ImportAsync", BindingFlags.Public | BindingFlags.Instance); if (importMethod == null) { @@ -50,8 +52,23 @@ public static partial class AssetDatabase try { - var task = importMethod.Invoke(importerInstance, new object[] { assetPath, metaResult.Value }) as Task; - if (task == null) + var parameters = importMethod.GetParameters(); + object? invokeResult; + + if (parameters.Length == 2) + { + invokeResult = importMethod.Invoke(importerInstance, new object[] { assetPath, metaResult.Value }); + } + else if (parameters.Length == 3 && parameters[2].ParameterType == typeof(CancellationToken)) + { + invokeResult = importMethod.Invoke(importerInstance, new object[] { assetPath, metaResult.Value, token }); + } + else + { + return Result.Failure($"Unsupported ImportAsync signature on importer {importerType.Name}"); + } + + if (invokeResult is not Task task) { return Result.Failure("Importer did not return a valid Task"); } @@ -93,7 +110,7 @@ public static partial class AssetDatabase /// Full path where the asset should be saved. /// In-memory asset data to export. /// Result with the GUID of the exported asset. - public static async Task> ExportAssetAsync(string assetPath, T assetData) where T : class + public static async ValueTask> ExportAssetAsync(string assetPath, T assetData, CancellationToken token = default) where T : class { var extension = Path.GetExtension(assetPath); @@ -124,16 +141,31 @@ public static partial class AssetDatabase try { // Generate metadata for the new asset - await GenerateMetaFileAsync(assetPath); + await GenerateMetaFileAsync(assetPath, token); - var metaResult = await ReadMetaFileAsync(assetPath); + var metaResult = await ReadMetaFileAsync(assetPath, token); if (metaResult.IsFailure) { return Result.Failure($"Failed to generate metadata: {metaResult.Message}"); } - var task = exportMethod.Invoke(importerInstance, new object[] { assetPath, assetData, metaResult.Value }) as Task; - if (task == null) + var parameters = exportMethod.GetParameters(); + object? invokeResult; + + if (parameters.Length == 3) + { + invokeResult = exportMethod.Invoke(importerInstance, new object[] { assetPath, assetData, metaResult.Value }); + } + else if (parameters.Length == 4 && parameters[3].ParameterType == typeof(CancellationToken)) + { + invokeResult = exportMethod.Invoke(importerInstance, new object[] { assetPath, assetData, metaResult.Value, token }); + } + else + { + return Result.Failure($"Unsupported ExportAsync signature on importer {importerType.Name}"); + } + + if (invokeResult is not Task task) { return Result.Failure("Exporter did not return a valid Task"); } @@ -145,8 +177,8 @@ public static partial class AssetDatabase } // Calculate file hash and update database - var fileHash = await CalculateFileHashAsync(assetPath); - await UpsertAssetAsync(assetPath, metaResult.Value, fileHash); + var fileHash = await CalculateFileHashAsync(assetPath, token); + await UpsertAssetAsync(assetPath, metaResult.Value, fileHash, null, token); return metaResult.Value.Guid; } diff --git a/Ghost.Editor.Core/AssetHandle/AssetDatabase.Lookup.cs b/Ghost.Editor.Core/AssetHandle/AssetDatabase.Lookup.cs index 85d9a6f..46f013f 100644 --- a/Ghost.Editor.Core/AssetHandle/AssetDatabase.Lookup.cs +++ b/Ghost.Editor.Core/AssetHandle/AssetDatabase.Lookup.cs @@ -119,7 +119,7 @@ public static partial class AssetDatabase return Result>.Failure(fullPathResult.Message); } - var metaResult = await ReadMetaFileAsync(fullPathResult.Value); + var metaResult = await ReadMetaFileAsync(fullPathResult.Value, token); if (metaResult.IsFailure) { return Result>.Failure(metaResult.Message); @@ -134,7 +134,7 @@ public static partial class AssetDatabase /// GUID of the asset. /// New tags for the asset. /// Result indicating success or failure. - public static async ValueTask SetAssetTagsAsync(Guid guid, List tags) + public static async ValueTask SetAssetTagsAsync(Guid guid, List tags, CancellationToken token = default) { var pathResult = GuidToPath(guid); if (pathResult.IsFailure) @@ -148,7 +148,7 @@ public static partial class AssetDatabase return Result.Failure(fullPathResult.Message); } - var metaResult = await ReadMetaFileAsync(fullPathResult.Value); + var metaResult = await ReadMetaFileAsync(fullPathResult.Value, token); if (metaResult.IsFailure) { return Result.Failure(metaResult.Message); @@ -157,15 +157,15 @@ public static partial class AssetDatabase metaResult.Value.Tags = tags; // Write updated metadata to .gmeta file - var writeResult = await WriteMetaFileAsync(fullPathResult.Value + Utilities.FileExtensions.META_FILE_EXTENSION, metaResult.Value); + var writeResult = await WriteMetaFileAsync(fullPathResult.Value + Utilities.FileExtensions.META_FILE_EXTENSION, metaResult.Value, token); if (writeResult.IsFailure) { return writeResult; } // Update database with new tags - var fileHash = await CalculateFileHashAsync(fullPathResult.Value); - return await UpsertAssetAsync(fullPathResult.Value, metaResult.Value, fileHash); + var fileHash = await CalculateFileHashAsync(fullPathResult.Value, token); + return await UpsertAssetAsync(fullPathResult.Value, metaResult.Value, fileHash, null, token); } /// @@ -174,9 +174,9 @@ public static partial class AssetDatabase /// /// Search pattern (e.g., "*.txt", "player?", "test*"). /// List of matching asset GUIDs. - public static async Task> FindAssetsByNameAsync(string namePattern) + public static async Task> FindAssetsByNameAsync(string namePattern, CancellationToken token = default) { - return await GetAssetsByNameAsync(namePattern); + return await GetAssetsByNameAsync(namePattern, token); } /// @@ -184,9 +184,9 @@ public static partial class AssetDatabase /// /// Tag to search for. /// List of asset GUIDs with the specified tag. - public static async Task> FindAssetsByTagAsync(string tag) + public static async Task> FindAssetsByTagAsync(string tag, CancellationToken token = default) { - return await GetAssetsByTagAsync(tag); + return await GetAssetsByTagAsync(tag, token); } /// diff --git a/Ghost.Editor.Core/AssetHandle/AssetDatabase.Meta.cs b/Ghost.Editor.Core/AssetHandle/AssetDatabase.Meta.cs index b501bfd..697ea5e 100644 --- a/Ghost.Editor.Core/AssetHandle/AssetDatabase.Meta.cs +++ b/Ghost.Editor.Core/AssetHandle/AssetDatabase.Meta.cs @@ -70,12 +70,12 @@ public static partial class AssetDatabase /// /// Calculate SHA256 hash of a file for change detection. /// - private static async Task CalculateFileHashAsync(string filePath) + private static async Task CalculateFileHashAsync(string filePath, CancellationToken token = default) { try { await using var stream = File.OpenRead(filePath); - var hash = await SHA256.HashDataAsync(stream); + var hash = await SHA256.HashDataAsync(stream, token); return Convert.ToHexString(hash); } catch @@ -84,12 +84,12 @@ public static partial class AssetDatabase } } - private static async Task WriteMetaFileAsync(string metaFilePath, AssetMeta metaData) + private static async Task WriteMetaFileAsync(string metaFilePath, AssetMeta metaData, CancellationToken token = default) { try { await using var fileStream = File.Create(metaFilePath); - await JsonSerializer.SerializeAsync(fileStream, metaData, s_defaultJsonOptions); + await JsonSerializer.SerializeAsync(fileStream, metaData, s_defaultJsonOptions, token); return Result.Success(); } catch (Exception ex) @@ -143,7 +143,7 @@ public static partial class AssetDatabase if (File.Exists(metaFileResult.Value)) { - var existingMetaResult = await ReadMetaFileAsync(assetPath); + var existingMetaResult = await ReadMetaFileAsync(assetPath, token); if (existingMetaResult.IsSuccess) { var existingMeta = existingMetaResult.Value; @@ -154,7 +154,7 @@ public static partial class AssetDatabase { // GUID conflict - regenerate existingMeta.Guid = Guid.NewGuid(); - r = await WriteMetaFileAsync(metaFileResult.Value, existingMeta); + r = await WriteMetaFileAsync(metaFileResult.Value, existingMeta, token); if (r.IsFailure) { return r; @@ -163,14 +163,14 @@ public static partial class AssetDatabase } // Calculate file hash and update database - var fileHash = await CalculateFileHashAsync(assetPath); - await UpsertAssetAsync(assetPath, existingMeta, fileHash); + var fileHash = await CalculateFileHashAsync(assetPath, token); + await UpsertAssetAsync(assetPath, existingMeta, fileHash, null, token); return Result.Success(); } } // Calculate initial file hash - var fileHash2 = await CalculateFileHashAsync(assetPath); + var fileHash2 = await CalculateFileHashAsync(assetPath, token); var defaultSettings = GetDefaultSettingsForAsset(assetPath); var metaData = new AssetMeta @@ -183,19 +183,19 @@ public static partial class AssetDatabase metaData.SetImporterSettings(defaultSettings.GetType().Name, defaultSettings); } - r = await WriteMetaFileAsync(metaFileResult.Value, metaData); + r = await WriteMetaFileAsync(metaFileResult.Value, metaData, token); if (r.IsFailure) { return r; } // Add to database - await UpsertAssetAsync(assetPath, metaData, fileHash2); + await UpsertAssetAsync(assetPath, metaData, fileHash2, null, token); return r; } - private static async void OnAssetCreated(object sender, FileSystemEventArgs e) + private static void OnAssetCreated(object sender, FileSystemEventArgs e) { // Skip meta files if (Path.GetExtension(e.FullPath).Equals(FileExtensions.META_FILE_EXTENSION, StringComparison.OrdinalIgnoreCase)) @@ -203,16 +203,10 @@ public static partial class AssetDatabase return; } - // Debounce to prevent duplicate events - if (!ShouldProcessFileOperation(e.FullPath)) - { - return; - } - - await GenerateMetaFileAsync(e.FullPath); + PostCommand(new AssetCommand(AssetCommandType.FileCreated, e.FullPath, Timestamp: DateTime.UtcNow)); } - private static async void OnAssetDeleted(object sender, FileSystemEventArgs e) + private static void OnAssetDeleted(object sender, FileSystemEventArgs e) { // Skip meta files if (Path.GetExtension(e.FullPath).Equals(FileExtensions.META_FILE_EXTENSION, StringComparison.OrdinalIgnoreCase)) @@ -220,39 +214,10 @@ public static partial class AssetDatabase return; } - // Debounce to prevent duplicate events - if (!ShouldProcessFileOperation(e.FullPath)) - { - return; - } - - var metaFileResult = GetMetaFilePath(e.FullPath); - if (metaFileResult.IsSuccess && File.Exists(metaFileResult.Value)) - { - try - { - var metaResult = await ReadMetaFileAsync(e.FullPath); - if (metaResult.IsSuccess) - { - var meta = metaResult.Value; - - // Remove from database - await RemoveAssetFromDatabaseAsync(meta.Guid); - - // Mark dependent assets as dirty - await MarkDependentAssetsDirtyAsync(meta.Guid); - } - - File.Delete(metaFileResult.Value); - } - catch (Exception ex) - { - Console.WriteLine($"Error deleting asset metadata: {ex.Message}"); - } - } + PostCommand(new AssetCommand(AssetCommandType.FileDeleted, e.FullPath, Timestamp: DateTime.UtcNow)); } - private static async void OnAssetRenamed(object sender, RenamedEventArgs e) + private static void OnAssetRenamed(object sender, RenamedEventArgs e) { // Skip meta files if (Path.GetExtension(e.FullPath).Equals(FileExtensions.META_FILE_EXTENSION, StringComparison.OrdinalIgnoreCase)) @@ -260,54 +225,10 @@ public static partial class AssetDatabase return; } - // Debounce to prevent duplicate events - if (!ShouldProcessFileOperation(e.FullPath)) - { - return; - } - - var oldMetaPath = e.OldFullPath + FileExtensions.META_FILE_EXTENSION; - var newMetaPath = e.FullPath + FileExtensions.META_FILE_EXTENSION; - - if (File.Exists(newMetaPath)) - { - // Validate and update - await GenerateMetaFileAsync(e.FullPath); - } - else if (File.Exists(oldMetaPath)) - { - // Move meta file - File.Move(oldMetaPath, newMetaPath); - - // Update database with new path and recalculated hash - var metaResult = await ReadMetaFileAsync(e.FullPath); - if (metaResult.IsSuccess) - { - var fileHash = await CalculateFileHashAsync(e.FullPath); - await UpsertAssetAsync(e.FullPath, metaResult.Value, fileHash); - } - } - else - { - // Generate new meta file - await GenerateMetaFileAsync(e.FullPath); - } - - // Delete old meta if it still exists - if (File.Exists(oldMetaPath) && oldMetaPath != newMetaPath) - { - try - { - File.Delete(oldMetaPath); - } - catch - { - // Ignore - } - } + PostCommand(new AssetCommand(AssetCommandType.FileRenamed, e.FullPath, e.OldFullPath, DateTime.UtcNow)); } - private static async void OnAssetChanged(object sender, FileSystemEventArgs e) + private static void OnAssetChanged(object sender, FileSystemEventArgs e) { // Skip meta files if (Path.GetExtension(e.FullPath).Equals(FileExtensions.META_FILE_EXTENSION, StringComparison.OrdinalIgnoreCase)) @@ -315,29 +236,7 @@ public static partial class AssetDatabase return; } - // Debounce to prevent duplicate events - if (!ShouldProcessFileOperation(e.FullPath)) - { - return; - } - - // Check if file hash changed - var metaResult = await ReadMetaFileAsync(e.FullPath); - if (metaResult.IsFailure) - { - return; - } - - // Calculate new hash and compare against database - var newHash = await CalculateFileHashAsync(e.FullPath); - var oldHash = await GetFileHashAsync(metaResult.Value.Guid); - - if (oldHash != newHash) - { - // File changed - update database and mark as dirty - await UpsertAssetAsync(e.FullPath, metaResult.Value, newHash); - await MarkAssetDirtyAsync(metaResult.Value.Guid, true); - } + PostCommand(new AssetCommand(AssetCommandType.FileModified, e.FullPath, Timestamp: DateTime.UtcNow)); } /// @@ -350,10 +249,10 @@ public static partial class AssetDatabase foreach (var kvp in allAssets) { - var dependencies = await GetDependenciesAsync(kvp.Key); + var dependencies = await GetDependenciesAsync(kvp.Key, CancellationToken.None); if (dependencies.Contains(assetGuid)) { - await MarkAssetDirtyAsync(kvp.Key, true); + MarkDirty(kvp.Key); } } } diff --git a/Ghost.Editor.Core/AssetHandle/AssetDatabase.SQLite.cs b/Ghost.Editor.Core/AssetHandle/AssetDatabase.SQLite.cs index 4c6adce..2b41cd4 100644 --- a/Ghost.Editor.Core/AssetHandle/AssetDatabase.SQLite.cs +++ b/Ghost.Editor.Core/AssetHandle/AssetDatabase.SQLite.cs @@ -12,7 +12,7 @@ public static partial class AssetDatabase /// /// Initialize the SQLite database for asset caching. /// - private static async Task InitializeDatabaseAsync() + private static async Task InitializeDatabaseAsync(CancellationToken token = default) { if (AssetsDirectory == null) { @@ -34,7 +34,7 @@ public static partial class AssetDatabase }.ToString(); s_dbConnection = new SqliteConnection(connectionString); - await s_dbConnection.OpenAsync(); + await s_dbConnection.OpenAsync(token); // Create tables await using var cmd = s_dbConnection.CreateCommand(); @@ -46,13 +46,11 @@ public static partial class AssetDatabase Tags TEXT, FileHash TEXT, DependencyGuids TEXT, - IsDirty INTEGER NOT NULL DEFAULT 0, LastModified INTEGER NOT NULL ); CREATE INDEX IF NOT EXISTS idx_path ON Assets(Path); - CREATE INDEX IF NOT EXISTS idx_dirty ON Assets(IsDirty); "; - await cmd.ExecuteNonQueryAsync(); + await cmd.ExecuteNonQueryAsync(token); } /// @@ -116,7 +114,7 @@ public static partial class AssetDatabase /// /// Remove an asset from the database. /// - private static async Task RemoveAssetFromDatabaseAsync(Guid guid) + private static async Task RemoveAssetFromDatabaseAsync(Guid guid, CancellationToken token = default) { if (s_dbConnection == null) { @@ -138,7 +136,7 @@ public static partial class AssetDatabase cmd.CommandText = "DELETE FROM Assets WHERE Guid = @guid"; cmd.Parameters.AddWithValue("@guid", guid.ToString()); - await cmd.ExecuteNonQueryAsync(); + await cmd.ExecuteNonQueryAsync(token); return Result.Success(); } catch (Exception ex) @@ -147,73 +145,12 @@ public static partial class AssetDatabase } } - /// - /// Mark an asset as dirty for re-importing. - /// - private static async Task MarkAssetDirtyAsync(Guid guid, bool isDirty = true) - { - if (s_dbConnection == null) - { - return Result.Failure("Database not initialized"); - } - try - { - await using var cmd = s_dbConnection.CreateCommand(); - cmd.CommandText = "UPDATE Assets SET IsDirty = @dirty WHERE Guid = @guid"; - cmd.Parameters.AddWithValue("@dirty", isDirty ? 1 : 0); - cmd.Parameters.AddWithValue("@guid", guid.ToString()); - - await cmd.ExecuteNonQueryAsync(); - return Result.Success(); - } - catch (Exception ex) - { - return Result.Failure($"Failed to mark asset dirty: {ex.Message}"); - } - } - - /// - /// Get all dirty assets that need re-importing. - /// - private static async Task> GetDirtyAssetsAsync() - { - var result = new List<(Guid guid, string path)>(); - - if (s_dbConnection == null) - { - return result; - } - - try - { - await using var cmd = s_dbConnection.CreateCommand(); - cmd.CommandText = "SELECT Guid, Path FROM Assets WHERE IsDirty = 1"; - - await using var reader = await cmd.ExecuteReaderAsync(); - while (await reader.ReadAsync()) - { - var guidStr = reader.GetString(0); - var path = reader.GetString(1); - - if (Guid.TryParse(guidStr, out var guid)) - { - result.Add((guid, path)); - } - } - } - catch - { - // Silently fail - we'll return empty list - } - - return result; - } /// /// Load all assets from the database into memory cache. /// - private static async Task LoadAssetCacheFromDatabaseAsync() + private static async Task LoadAssetCacheFromDatabaseAsync(CancellationToken token = default) { if (s_dbConnection == null) { @@ -225,8 +162,8 @@ public static partial class AssetDatabase await using var cmd = s_dbConnection.CreateCommand(); cmd.CommandText = "SELECT Guid, Path FROM Assets"; - await using var reader = await cmd.ExecuteReaderAsync(); - while (await reader.ReadAsync()) + await using var reader = await cmd.ExecuteReaderAsync(token); + while (await reader.ReadAsync(token)) { var guidStr = reader.GetString(0); var path = reader.GetString(1); @@ -250,7 +187,7 @@ public static partial class AssetDatabase /// /// Get assets by tag. /// - private static async Task> GetAssetsByTagAsync(string tag) + private static async Task> GetAssetsByTagAsync(string tag, CancellationToken token = default) { var result = new List(); @@ -264,8 +201,8 @@ public static partial class AssetDatabase await using var cmd = s_dbConnection.CreateCommand(); cmd.CommandText = "SELECT Guid, Tags FROM Assets"; - await using var reader = await cmd.ExecuteReaderAsync(); - while (await reader.ReadAsync()) + await using var reader = await cmd.ExecuteReaderAsync(token); + while (await reader.ReadAsync(token)) { var guidStr = reader.GetString(0); var tagsJson = reader.GetString(1); @@ -291,7 +228,7 @@ public static partial class AssetDatabase /// /// Get the file hash for an asset from the database. /// - private static async Task GetFileHashAsync(Guid guid) + private static async Task GetFileHashAsync(Guid guid, CancellationToken token = default) { if (s_dbConnection == null) { @@ -304,7 +241,7 @@ public static partial class AssetDatabase cmd.CommandText = "SELECT FileHash FROM Assets WHERE Guid = @guid"; cmd.Parameters.AddWithValue("@guid", guid.ToString()); - var result = await cmd.ExecuteScalarAsync(); + var result = await cmd.ExecuteScalarAsync(token); return result?.ToString(); } catch @@ -316,7 +253,7 @@ public static partial class AssetDatabase /// /// Get the dependencies for an asset from the database. /// - private static async Task> GetDependenciesAsync(Guid guid) + private static async Task> GetDependenciesAsync(Guid guid, CancellationToken token = default) { if (s_dbConnection == null) { @@ -329,7 +266,7 @@ public static partial class AssetDatabase cmd.CommandText = "SELECT DependencyGuids FROM Assets WHERE Guid = @guid"; cmd.Parameters.AddWithValue("@guid", guid.ToString()); - var result = await cmd.ExecuteScalarAsync(); + var result = await cmd.ExecuteScalarAsync(token); if (result != null) { var json = result.ToString(); @@ -348,7 +285,7 @@ public static partial class AssetDatabase /// Find assets by name pattern using database query with wildcards. /// /// Pattern supporting * (any chars) and ? (single char). - private static async Task> GetAssetsByNameAsync(string namePattern) + private static async Task> GetAssetsByNameAsync(string namePattern, CancellationToken token = default) { var results = new List(); @@ -373,8 +310,8 @@ public static partial class AssetDatabase "; cmd.Parameters.AddWithValue("@pattern", sqlPattern); - await using var reader = await cmd.ExecuteReaderAsync(); - while (await reader.ReadAsync()) + await using var reader = await cmd.ExecuteReaderAsync(token); + while (await reader.ReadAsync(token)) { var guidStr = reader.GetString(0); var path = reader.GetString(1); @@ -407,7 +344,7 @@ public static partial class AssetDatabase /// /// Remove orphaned entries from database (assets that no longer exist on disk). /// - private static async Task RemoveOrphanedEntriesAsync() + private static async Task RemoveOrphanedEntriesAsync(CancellationToken token = default) { if (s_dbConnection == null || AssetsDirectory == null) { @@ -421,8 +358,8 @@ public static partial class AssetDatabase await using var cmd = s_dbConnection.CreateCommand(); cmd.CommandText = "SELECT Guid, Path FROM Assets"; - await using var reader = await cmd.ExecuteReaderAsync(); - while (await reader.ReadAsync()) + await using var reader = await cmd.ExecuteReaderAsync(token); + while (await reader.ReadAsync(token)) { var guidStr = reader.GetString(0); var path = reader.GetString(1); @@ -439,10 +376,10 @@ public static partial class AssetDatabase } // Remove orphaned entries - foreach (var guid in orphanedGuids) - { - await RemoveAssetFromDatabaseAsync(guid); - } + foreach (var guid in orphanedGuids) + { + await RemoveAssetFromDatabaseAsync(guid, token); + } } catch { diff --git a/Ghost.Editor.Core/AssetHandle/AssetDatabase.cs b/Ghost.Editor.Core/AssetHandle/AssetDatabase.cs index 2c94012..dacb3cb 100644 --- a/Ghost.Editor.Core/AssetHandle/AssetDatabase.cs +++ b/Ghost.Editor.Core/AssetHandle/AssetDatabase.cs @@ -1,9 +1,33 @@ +using Ghost.Core; using Ghost.Data.Services; using System.Text.Json; using System.Text.Json.Serialization; +using System.Threading.Channels; namespace Ghost.Editor.Core.AssetHandle; +/// +/// Command types for asset database operations. +/// +internal enum AssetCommandType +{ + FileCreated, + FileModified, + FileDeleted, + FileRenamed, + ManualRefresh +} + +/// +/// Represents a command to process an asset operation. +/// +internal readonly record struct AssetCommand( + AssetCommandType Type, + string Path, + string? OldPath = null, + DateTime Timestamp = default +); + /// /// Centralized asset database that manages all assets in the project. /// Handles asset registration, lookup, importing, and dependency management. @@ -15,16 +39,24 @@ public static partial class AssetDatabase private static readonly Lock s_dbLock = new(); private static readonly Dictionary s_assetPathLookup = new(); private static readonly Dictionary s_pathAssetLookup = new(); - - // Debouncing for file system watcher to prevent duplicate events - private static readonly Dictionary s_pendingFileOperations = new(); - private static readonly Lock s_pendingOperationsLock = new(); - private static readonly TimeSpan s_debounceDelay = TimeSpan.FromMilliseconds(100); - + + // In-memory dirty asset tracking (for runtime modifications only) + private static readonly HashSet s_dirtyAssets = new(); + + // Command buffer pattern - Channel for file system event commands + private static Channel? s_commandChannel; + private static Timer? s_commandProcessorTimer; + private static readonly HashSet s_pendingCommandPaths = new(); // Track paths with pending commands + private static readonly Lock s_commandLock = new(); + private static bool s_autoRefreshEnabled = true; + private static readonly Queue s_waitingCommands = new(); // Commands waiting for manual refresh + // Initialization guard private static readonly Lock s_initializationLock = new(); private static bool s_initialized = false; + private static readonly TimeSpan s_debounceDelay = TimeSpan.FromMilliseconds(100); + private static readonly JsonSerializerOptions s_defaultJsonOptions = new() { WriteIndented = true, @@ -45,7 +77,7 @@ public static partial class AssetDatabase /// Initialize the asset database. /// Must be called after project is loaded. /// - internal static async void Initialize() + internal static async void Initialize(CancellationToken token = default) { lock (s_initializationLock) { @@ -63,11 +95,21 @@ public static partial class AssetDatabase AssetsDirectory = new DirectoryInfo(Path.Combine(Path.GetDirectoryName(ProjectService.CurrentProject.Path)!, ProjectService.ASSETS_FOLDER)); + // Initialize command channel (unbounded for simplicity) + s_commandChannel = Channel.CreateUnbounded(new UnboundedChannelOptions + { + SingleReader = false, // Timer callback reads + SingleWriter = false // Multiple FS events can write + }); + + // Initialize command processor timer (starts disabled, triggered by events) + s_commandProcessorTimer = new Timer(ProcessPendingCommands, null, Timeout.Infinite, Timeout.Infinite); + // Initialize database - await InitializeDatabaseAsync(); + await InitializeDatabaseAsync(token); // Load asset cache from database - await LoadAssetCacheFromDatabaseAsync(); + await LoadAssetCacheFromDatabaseAsync(token); // Initialize file system watcher s_watcher = new FileSystemWatcher @@ -82,26 +124,25 @@ public static partial class AssetDatabase InitializeMetaData(); // Validate and fix database on startup - await ValidateAndFixDatabaseAsync(); + await ValidateAndFixDatabaseAsync(token); } /// /// Validate the asset database and fix any inconsistencies. /// Checks for missing/corrupted assets and regenerates metadata as needed. /// - private static async Task ValidateAndFixDatabaseAsync() + private static async Task ValidateAndFixDatabaseAsync(CancellationToken token = default) { if (AssetsDirectory == null) { - return Ghost.Core.Result.Failure("AssetsDirectory not initialized"); + return Result.Failure("AssetsDirectory not initialized"); } try { // Scan all files in assets directory var allFiles = Directory.GetFiles(AssetsDirectory.FullName, "*.*", SearchOption.AllDirectories) - .Where(f => !f.EndsWith(Utilities.FileExtensions.META_FILE_EXTENSION, StringComparison.OrdinalIgnoreCase)) - .ToList(); + .Where(f => !f.EndsWith(Utilities.FileExtensions.META_FILE_EXTENSION, StringComparison.OrdinalIgnoreCase)); // Ensure all files have metadata foreach (var file in allFiles) @@ -109,91 +150,372 @@ public static partial class AssetDatabase var metaPath = file + Utilities.FileExtensions.META_FILE_EXTENSION; if (!File.Exists(metaPath)) { - await GenerateMetaFileAsync(file); + await GenerateMetaFileAsync(file, token); } else { // Validate and update database - var metaResult = await ReadMetaFileAsync(file); + var metaResult = await ReadMetaFileAsync(file, token); if (metaResult.IsSuccess) { - var fileHash = await CalculateFileHashAsync(file); - await UpsertAssetAsync(file, metaResult.Value, fileHash); + var fileHash = await CalculateFileHashAsync(file, token); + await UpsertAssetAsync(file, metaResult.Value, fileHash, null, token); } else { // Corrupted meta file - regenerate - await GenerateMetaFileAsync(file); + await GenerateMetaFileAsync(file, token); } } } // Remove orphaned entries from database (files that no longer exist) - await RemoveOrphanedEntriesAsync(); + await RemoveOrphanedEntriesAsync(token); - return Ghost.Core.Result.Success(); + return Result.Success(); } catch (Exception ex) { - return Ghost.Core.Result.Failure($"Failed to validate database: {ex.Message}"); + return Result.Failure($"Failed to validate database: {ex.Message}"); } } /// /// Refresh the asset database manually. - /// Scans the project directory for changes. + /// Scans the project directory for changes and processes any queued file system events. /// - public static async Task RefreshAsync() + public static async Task RefreshAsync(CancellationToken token = default) { - return await ValidateAndFixDatabaseAsync(); + // Flush waiting commands to channel + lock (s_commandLock) + { + while (s_waitingCommands.TryDequeue(out var cmd)) + { + s_commandChannel?.Writer.TryWrite(cmd); + } + } + + // Post manual refresh command + s_commandChannel?.Writer.TryWrite(new AssetCommand(AssetCommandType.ManualRefresh, string.Empty)); + + // Trigger timer immediately + s_commandProcessorTimer?.Change(TimeSpan.Zero, Timeout.InfiniteTimeSpan); + + // Wait a bit for processing to complete (this is best-effort) + await Task.Delay(200, token); + + return Result.Success(); } /// - /// Check if a file operation should be processed or debounced. - /// Returns true if the operation should proceed. + /// Mark an asset as dirty (modified in memory but not yet saved). + /// This state is NOT persisted and will be lost on application restart. /// - private static bool ShouldProcessFileOperation(string filePath) + public static void MarkDirty(Guid assetGuid) { - lock (s_pendingOperationsLock) + lock (s_dbLock) { - var now = DateTime.UtcNow; - - // Clean up old entries - var toRemove = s_pendingFileOperations - .Where(kvp => now - kvp.Value > s_debounceDelay * 2) - .Select(kvp => kvp.Key) - .ToList(); - - foreach (var key in toRemove) - { - s_pendingFileOperations.Remove(key); - } - - // Check if this operation was recently processed - if (s_pendingFileOperations.TryGetValue(filePath, out var lastTime)) - { - if (now - lastTime < s_debounceDelay) - { - // Too soon, skip this event - return false; - } - } - - // Update timestamp and allow processing - s_pendingFileOperations[filePath] = now; - return true; + s_dirtyAssets.Add(assetGuid); } } /// - /// Register a file operation to prevent the file watcher from processing it. - /// Used by file operations (move, copy, etc.) to prevent duplicate processing. + /// Check if an asset is marked as dirty. /// - private static void RegisterFileOperation(string filePath) + public static bool IsDirty(Guid assetGuid) { - lock (s_pendingOperationsLock) + lock (s_dbLock) { - s_pendingFileOperations[filePath] = DateTime.UtcNow; + return s_dirtyAssets.Contains(assetGuid); + } + } + + /// + /// Get all dirty assets. + /// + public static Guid[] GetDirtyAssets() + { + lock (s_dbLock) + { + return s_dirtyAssets.ToArray(); + } + } + + /// + /// Clear dirty flag for an asset (typically after saving). + /// + public static void ClearDirty(Guid assetGuid) + { + lock (s_dbLock) + { + s_dirtyAssets.Remove(assetGuid); + } + } + + /// + /// Clear all dirty flags. + /// + public static void ClearAllDirty() + { + lock (s_dbLock) + { + s_dirtyAssets.Clear(); + } + } + + /// + /// Enable or disable automatic asset database refresh. + /// When disabled, file system events are queued and processed only when RefreshAsync() is called. + /// + public static void SetAutoRefresh(bool enabled) + { + s_autoRefreshEnabled = enabled; + } + + /// + /// Process all pending commands immediately (synchronous, for testing). + /// + internal static void FlushPendingCommands() + { + // Stop timer temporarily + s_commandProcessorTimer?.Change(Timeout.Infinite, Timeout.Infinite); + + // Give a tiny bit of time for any in-flight file watcher events to post to channel + Thread.Sleep(50); + + // Process all commands now + ProcessPendingCommands(null); + } + + /// + /// Post a command to the command channel for processing. + /// + private static void PostCommand(AssetCommand command) + { + if (s_commandChannel == null) + { + return; + } + + if (s_autoRefreshEnabled) + { + // Add to pending paths for temp file tracking + lock (s_commandLock) + { + s_pendingCommandPaths.Add(command.Path); + if (command.OldPath != null) + { + s_pendingCommandPaths.Add(command.OldPath); + } + } + + // Write to channel (lock-free, async-safe) + s_commandChannel.Writer.TryWrite(command); + + // Start or reset timer (safe even if not initialized yet) + s_commandProcessorTimer?.Change(s_debounceDelay, Timeout.InfiniteTimeSpan); + } + else + { + // Queue for manual refresh + lock (s_commandLock) + { + s_waitingCommands.Enqueue(command); + } + } + } + + /// + /// Timer callback to process pending commands. + /// + private static void ProcessPendingCommands(object? state) + { + try + { + // Collect all pending commands + var commands = new List(); + + while (s_commandChannel?.Reader.TryRead(out var cmd) == true) + { + commands.Add(cmd); + } + + // Group commands by path (last command wins) + var commandsByPath = new Dictionary(); + foreach (var cmd in commands) + { + commandsByPath[cmd.Path] = cmd; + } + + // Filter out temp files (files that were created then deleted) + lock (s_commandLock) + { + var pathsToProcess = commandsByPath.Keys.ToList(); + foreach (var path in pathsToProcess) + { + // If file was created/modified but doesn't exist anymore, skip + if (!File.Exists(path) && commandsByPath[path].Type != AssetCommandType.FileDeleted) + { + commandsByPath.Remove(path); + } + } + + // Clear pending paths + s_pendingCommandPaths.Clear(); + } + + // Execute commands + foreach (var cmd in commandsByPath.Values) + { + ExecuteCommandAsync(cmd).GetAwaiter().GetResult(); + } + } + catch (Exception ex) + { + Console.WriteLine($"Error processing commands: {ex.Message}"); + } + } + + /// + /// Execute a single asset command. + /// + private static async Task ExecuteCommandAsync(AssetCommand command) + { + switch (command.Type) + { + case AssetCommandType.FileCreated: + await HandleFileCreatedAsync(command.Path); + break; + + case AssetCommandType.FileModified: + await HandleFileModifiedAsync(command.Path); + break; + + case AssetCommandType.FileDeleted: + await HandleFileDeletedAsync(command.Path); + break; + + case AssetCommandType.FileRenamed: + if (command.OldPath != null) + { + await HandleFileRenamedAsync(command.OldPath, command.Path); + } + break; + + case AssetCommandType.ManualRefresh: + await ValidateAndFixDatabaseAsync(CancellationToken.None); + break; + } + } + + /// + /// Handle file created event. + /// + private static async Task HandleFileCreatedAsync(string path) + { + await GenerateMetaFileAsync(path, CancellationToken.None); + } + + /// + /// Handle file modified event. + /// + private static async Task HandleFileModifiedAsync(string path) + { + // Check if file hash changed + var metaResult = await ReadMetaFileAsync(path, CancellationToken.None); + if (metaResult.IsFailure) + { + // No .gmeta file - treat this as a new file creation + await HandleFileCreatedAsync(path); + return; + } + + // Calculate new hash and compare against database + var newHash = await CalculateFileHashAsync(path, CancellationToken.None); + var oldHash = await GetFileHashAsync(metaResult.Value.Guid, CancellationToken.None); + + if (oldHash != newHash) + { + // File changed - update database and mark as dirty + await UpsertAssetAsync(path, metaResult.Value, newHash, null, CancellationToken.None); + MarkDirty(metaResult.Value.Guid); + } + } + + /// + /// Handle file deleted event. + /// + private static async Task HandleFileDeletedAsync(string path) + { + var metaFileResult = GetMetaFilePath(path); + if (metaFileResult.IsSuccess && File.Exists(metaFileResult.Value)) + { + try + { + var metaResult = await ReadMetaFileAsync(path, CancellationToken.None); + if (metaResult.IsSuccess) + { + var meta = metaResult.Value; + + // Remove from database + await RemoveAssetFromDatabaseAsync(meta.Guid, CancellationToken.None); + + // Mark dependent assets as dirty + await MarkDependentAssetsDirtyAsync(meta.Guid); + } + + File.Delete(metaFileResult.Value); + } + catch (Exception ex) + { + Console.WriteLine($"Error deleting asset metadata: {ex.Message}"); + } + } + } + + /// + /// Handle file renamed event. + /// + private static async Task HandleFileRenamedAsync(string oldPath, string newPath) + { + var oldMetaPath = oldPath + Utilities.FileExtensions.META_FILE_EXTENSION; + var newMetaPath = newPath + Utilities.FileExtensions.META_FILE_EXTENSION; + + if (File.Exists(newMetaPath)) + { + // Validate and update + await GenerateMetaFileAsync(newPath, CancellationToken.None); + } + else if (File.Exists(oldMetaPath)) + { + // Move meta file + File.Move(oldMetaPath, newMetaPath); + + // Update database with new path and recalculated hash + var metaResult = await ReadMetaFileAsync(newPath, CancellationToken.None); + if (metaResult.IsSuccess) + { + var fileHash = await CalculateFileHashAsync(newPath, CancellationToken.None); + await UpsertAssetAsync(newPath, metaResult.Value, fileHash, null, CancellationToken.None); + } + } + else + { + // Generate new meta file + await GenerateMetaFileAsync(newPath, CancellationToken.None); + } + + // Delete old meta if it still exists + if (File.Exists(oldMetaPath) && oldMetaPath != newMetaPath) + { + try + { + File.Delete(oldMetaPath); + } + catch + { + // Ignore + } } } @@ -213,15 +535,20 @@ public static partial class AssetDatabase s_watcher?.Dispose(); s_watcher = null; + s_commandProcessorTimer?.Dispose(); + s_commandProcessorTimer = null; + s_dbConnection?.Close(); s_dbConnection?.Dispose(); s_dbConnection = null; s_assetPathLookup.Clear(); s_pathAssetLookup.Clear(); + s_dirtyAssets.Clear(); + s_pendingCommandPaths.Clear(); + s_waitingCommands.Clear(); s_importerInstances.Clear(); s_importerTypeLookup.Clear(); - s_pendingFileOperations.Clear(); s_initialized = false; } diff --git a/Ghost.Editor.Core/AssetHandle/AssetDatabase_Architecture.md b/Ghost.Editor.Core/AssetHandle/AssetDatabase_Architecture.md new file mode 100644 index 0000000..93d7710 --- /dev/null +++ b/Ghost.Editor.Core/AssetHandle/AssetDatabase_Architecture.md @@ -0,0 +1,115 @@ +# Asset Database Architecture + +This document details the architectural design and data flow of the `AssetHandle` module in Ghost Editor. + +## System Overview + +The Asset Database acts as the bridge between the raw file system (Source Assets) and the runtime engine (Imported Assets). It maintains a consistent state using a dual-storage approach: +1. **File System**: The source of truth. Contains source files (e.g., `.png`, `.fbx`) and metadata files (`.gmeta`). +2. **SQLite Database**: An acceleration layer (cache) for fast lookups, dependency tracking, and searching. + +## Data Flow + +### 1. Asset Discovery & Registration +When the editor starts or a file changes: +1. **FileSystemWatcher** detects the change (Create/Delete/Modify/Rename). +2. **Event Handler** queues an `AssetCommand` (debounce mechanism prevents event storms). +3. **Command Processor** executes the command: + * **New File**: Generates a `.gmeta` file with a new GUID and default settings. Adds to SQLite. + * **Modified File**: Checks hash. If changed, marks asset as "Dirty" and updates SQLite. + * **Deleted File**: Removes from SQLite and marks dependents as "Dirty". + +### 2. Import Pipeline +The import process converts source formats into engine-ready data. + +**Flow:** +1. `AssetDatabase.ImportDirtyAssetsAsync()` or direct `ImportAssetAsync` is called. +2. System looks up the registered `AssetImporter` for the file extension. +3. `AssetImporter.ImportAsync` is invoked with the source path and metadata. +4. Importer reads source file and settings from metadata. +5. Importer processes data (e.g., compiles shaders, compresses textures). +6. Importer calls `AssetDatabase.SaveImportedAsset(guid, data)`. +7. Data is serialized to JSON (or binary) in the `Cache/ImportedAssets` directory as `{GUID}.asset`. + +### 3. Loading Pipeline +When the engine requests an asset: + +**Flow:** +1. `AssetDatabase.LoadAsset(guid)` is called. +2. **Memory Cache Check**: + * Checks `s_assetCache` (ConcurrentDictionary). + * If found: Updates LRU timestamp and returns object. + * If not found: Proceeds to disk load. +3. **Disk Load**: + * Locates `{GUID}.asset` in `Cache/ImportedAssets`. + * Deserializes the data into the target runtime type (e.g., `TextureAsset`). +4. **Cache Update**: + * Adds new object to `s_assetCache`. + * If cache size > `MAX_CACHED_ASSETS` (1000), evicts oldest 20% based on access time. + +## Key Components Diagram + +```mermaid +graph TD + User[Editor / User] -->|File Ops| API[AssetDatabase API] + FS[File System] -->|Events| Watcher[FileSystemWatcher] + + subgraph AssetDatabase + API --> DB[SQLite Database] + API --> Meta[Meta Handler] + API --> Loader[Asset Loader] + API --> Importer[Import System] + + Watcher -->|Queue| Cmd[Command Processor] + Cmd --> Meta + Cmd --> DB + + Importer -->|Read| FS + Importer -->|Write| Cache[Imported Assets Cache] + + Loader -->|Read| Cache + Loader -->|Check| MemCache[Memory LRU Cache] + end + + Meta -->|Read/Write| FS + DB -->|Index| FS +``` + +## Database Schema (SQLite) + +The `AssetDatabase.db` contains a single `Assets` table: + +| Column | Type | Description | +|--------|------|-------------| +| **Guid** | TEXT (PK) | The unique identifier of the asset. | +| **Path** | TEXT | Relative path from `Assets/` folder. Indexed for fast lookup. | +| **Version** | INTEGER | Importer version for migration support. | +| **Tags** | TEXT | JSON array of string tags. | +| **FileHash** | TEXT | SHA256 hash of the source file content. | +| **DependencyGuids** | TEXT | JSON array of GUIDs this asset depends on. | +| **LastModified** | INTEGER | Unix timestamp of last modification. | + +## Detailed Subsystems + +### Metadata System (`.gmeta`) +* **Format**: JSON. +* **Content**: GUID, Version, Tags, ImporterSettings (per importer type). +* **Strategy**: The `.gmeta` file is the *only* place the persistent GUID lives. If the database is corrupted, it can be rebuilt entirely by scanning the file system and reading `.gmeta` files. + +### Threading & Safety +* **Locks**: + * `s_dbLock`: Protects in-memory dictionaries (`s_assetPathLookup`) and dirty tracking. + * `s_commandLock`: Protects the command queue for file events. +* **Async**: Heavy I/O operations (DB access, File I/O) are async. +* **Channels**: Uses `System.Threading.Channels` to decouple high-frequency file system events from database processing. + +### Importer Registry +* Uses `TypeCache` and reflection to find classes with `[AssetImporter]`. +* Mappings are stored in `s_importerTypeLookup` (Extension -> Type). +* Importers are stateless (instantiated on demand or cached as singletons depending on implementation, currently cached in `s_importerInstances`). + +## Future Improvements / Known Limitations + +1. **Binary Formats**: Currently, imported assets are stored as JSON. For large assets (textures, models), a binary format is required for performance. +2. **Dependency Graph**: While dependencies are stored, a full graph traversal for complex invalidation (e.g., if A changes, re-import B which depends on A) is partial. +3. **Cross-Process Locking**: SQLite is file-based; concurrent access from multiple editor instances needs careful file locking mode configuration. diff --git a/Ghost.Editor.Core/AssetHandle/AssetDatabase_Documentation.md b/Ghost.Editor.Core/AssetHandle/AssetDatabase_Documentation.md new file mode 100644 index 0000000..5e759c1 --- /dev/null +++ b/Ghost.Editor.Core/AssetHandle/AssetDatabase_Documentation.md @@ -0,0 +1,131 @@ +# Asset Database Documentation + +The Asset Database is a core component of the Ghost Editor responsible for managing the lifecycle, storage, import, and retrieval of project assets. It provides a unified API for interacting with assets, ensuring that metadata (GUIDs, tags, settings) stays synchronized with files on disk. + +## Key Features + +- **GUID-based Asset Identification**: Every asset is uniquely identified by a stable GUID, stored in a sidecar `.gmeta` file. +- **Automatic Importing**: Monitors the file system for changes and automatically imports assets using registered importers. +- **Dependency Tracking**: Tracks dependencies between assets to ensure validity and trigger re-imports when dependencies change. +- **Caching**: Implements an LRU (Least Recently Used) cache for loaded assets to optimize performance. +- **SQLite Backed**: Uses a local SQLite database for fast lookups (Path <-> GUID) and metadata queries. +- **Metadata Management**: Handles `.gmeta` files automatically, including generation, validation, and cleanup. + +## usage + +### Initialization +The Asset Database must be initialized after the project is loaded. +```csharp +await AssetDatabase.Initialize(cancellationToken); +``` + +### Loading Assets +Assets can be loaded by GUID or by Path. + +```csharp +// Load by Path +var result = AssetDatabase.LoadAssetAtPath("Assets/Textures/my_texture.png"); +if (result.IsSuccess) +{ + var texture = result.Value; +} + +// Load by GUID +var guid = ...; +var result = AssetDatabase.LoadAsset(guid); +``` + +### File Operations +Always use the `AssetDatabase` API for file operations to ensure metadata is preserved. + +```csharp +// Create +await AssetDatabase.CreateAssetAsync("Assets/Data/config.json", dataBytes); + +// Move +await AssetDatabase.MoveAssetAsync("Assets/Old/file.txt", "Assets/New/file.txt"); + +// Copy +await AssetDatabase.CopyAssetAsync("Assets/template.txt", "Assets/instance.txt"); + +// Delete +await AssetDatabase.DeleteAssetAsync("Assets/garbage.tmp"); +``` + +### Searching +Find assets using wildcards or tags. + +```csharp +// Find all PNGs +var guids = await AssetDatabase.FindAssetsByNameAsync("*.png"); + +// Find assets with a specific tag +var enemyAssets = await AssetDatabase.FindAssetsByTagAsync("Enemy"); +``` + +### Tags +Manage asset tags for organization. + +```csharp +// Get tags +var tagsResult = await AssetDatabase.GetAssetTagsAsync(guid); + +// Set tags +await AssetDatabase.SetAssetTagsAsync(guid, new List { "Level1", "Prop" }); +``` + +### Opening Assets +Open an asset using its registered handler or the system default. +```csharp +AssetDatabase.OpenAsset("Assets/Docs/readme.txt"); +``` + +## Extending the Asset Database + +### Creating a New Importer +To support a new file type, create a class that inherits from `AssetImporter` and decorate it with the `[AssetImporter]` attribute. + +```csharp +[AssetImporter(".myfmt")] +internal class MyFormatImporter : AssetImporter +{ + public override async Task ImportAsync(string assetPath, AssetMeta meta) + { + var settings = GetSettings(meta); + + // 1. Read source file + // 2. Process data + // 3. Save imported data using AssetDatabase.SaveImportedAsset + + var myAsset = new MyAsset(meta.Guid) { ... }; + return AssetDatabase.SaveImportedAsset(meta.Guid, myAsset); + } +} + +internal class MyFormatSettings : ImporterSettings +{ + public float Scale { get; set; } = 1.0f; +} +``` + +### Creating an Open Handler +To define custom behavior when an asset is opened (e.g., double-clicked in the editor), use the `[AssetOpenHandler]` attribute. + +```csharp +internal static class MyHandlers +{ + [AssetOpenHandler(".myfmt")] + private static void OpenMyFormat(string path) + { + // Open custom editor window + } +} +``` + +## Internal Architecture + +- **AssetDatabase.cs**: Core initialization and event coordination. +- **AssetDatabase.SQLite.cs**: Database table management and queries. +- **AssetDatabase.Meta.cs**: `.gmeta` file handling and file system watcher events. +- **AssetDatabase.Importer.cs**: Importer discovery and execution. +- **AssetDatabase.Loader.cs**: Asset loading and caching logic. diff --git a/Ghost.Editor.Core/AssetHandle/Asset.cs b/Ghost.Editor.Core/AssetHandle/Models/Asset.cs similarity index 100% rename from Ghost.Editor.Core/AssetHandle/Asset.cs rename to Ghost.Editor.Core/AssetHandle/Models/Asset.cs diff --git a/Ghost.Editor.Core/AssetHandle/TextureAsset.cs b/Ghost.Editor.Core/AssetHandle/Models/TextureAsset.cs similarity index 100% rename from Ghost.Editor.Core/AssetHandle/TextureAsset.cs rename to Ghost.Editor.Core/AssetHandle/Models/TextureAsset.cs diff --git a/Ghost.Editor.Core/AssetHandle/README.md b/Ghost.Editor.Core/AssetHandle/README.md deleted file mode 100644 index e1a856a..0000000 --- a/Ghost.Editor.Core/AssetHandle/README.md +++ /dev/null @@ -1,250 +0,0 @@ -# Asset Database Implementation - -This is the complete implementation of the GhostEngine Asset Database system based on the plan in `AssetDBPlan.md`. - -## Structure - -The asset database is implemented as a partial class split across multiple files: - -### Core Files - -- **AssetDatabase.cs** - Main entry point with initialization and shutdown logic -- **AssetDatabase.Meta.cs** - Metadata file management and file system watching -- **AssetDatabase.SQLite.cs** - SQLite database operations for caching -- **AssetDatabase.Lookup.cs** - GUID/Path lookup and search operations -- **AssetDatabase.FileOps.cs** - File operations (create, delete, move, copy) -- **AssetDatabase.Importer.cs** - Asset importing framework -- **AssetDatabase.Open.cs** - Asset opening handlers (existing file) - -### Supporting Files - -- **Asset.cs** - Base class for all assets -- **AssetMeta.cs** - Metadata structure (stored in .gmeta files) -- **AssetImporter.cs** - Base class for all asset importers -- **AssetImporterAttribute.cs** - Attribute to mark importer classes -- **AssetOpenHandlerAttribute.cs** - Attribute for custom open handlers -- **ImporterSettings.cs** - Base class for importer settings - -### Example Importer - -- **Importers/TextImporter.cs** - Example importer for .txt and .md files - -## Features Implemented - -### Core API (AssetDatabase.Lookup.cs) - -- ✅ `PathToGuid(string assetPath)` - Find GUID by path -- ✅ `GuidToPath(Guid guid)` - Find path by GUID -- ✅ `LoadAsset(Guid guid)` - Load asset by GUID (TODO: needs asset loader) -- ✅ `GetAssetTagsAsync(Guid guid)` - Get asset tags -- ✅ `SetAssetTagsAsync(Guid guid, List tags)` - Set asset tags -- ✅ `FindAssetsByName(string namePattern)` - Search by name -- ✅ `FindAssetsByTagAsync(string tag)` - Search by tag -- ✅ `GetAllAssets()` - Get all assets in database - -### File Operations (AssetDatabase.FileOps.cs) - -- ✅ `CreateAssetAsync(string assetPath, byte[] content)` - Create new asset -- ✅ `DeleteAssetAsync(Guid guid)` - Delete asset -- ✅ `MoveAssetAsync(Guid guid, string newPath)` - Move/rename asset -- ✅ `CopyAssetAsync(Guid guid, string newPath)` - Copy asset with new GUID -- ✅ `RefreshAsync()` - Refresh database manually -- ✅ `MarkDirtyAsync(Guid guid)` - Mark asset for re-import -- ✅ `ImportDirtyAssetsAsync()` - Import all dirty assets - -### Background Services (AssetDatabase.Meta.cs) - -- ✅ File system watcher for automatic change detection -- ✅ Automatic metadata generation on file creation -- ✅ Automatic metadata cleanup on file deletion -- ✅ Automatic metadata movement on file rename -- ✅ File hash comparison for change detection -- ✅ Automatic dirty marking on file modification -- ✅ Dependent asset tracking and dirty propagation - -### Database (AssetDatabase.SQLite.cs) - -- ✅ SQLite for persistent storage and efficient querying -- ✅ In-memory cache for fast lookups -- ✅ Automatic database creation and schema management -- ✅ Asset indexing by GUID and path -- ✅ Dirty flag tracking for re-import -- ✅ Tag-based search support - -### Validation (AssetDatabase.cs) - -- ✅ Validate and fix database on project load -- ✅ Check for missing/corrupted metadata files -- ✅ Regenerate metadata when necessary -- ✅ Database consistency checks - -## Metadata File Format - -Assets have associated `.gmeta` files stored alongside them: - -```json -{ - "Guid": "123e4567-e89b-12d3-a456-426614174000", - "Version": 1, - "Tags": ["Environment", "Texture"], - "FileHash": "ABC123...", - "Dependencies": [ - "456e7890-e89b-12d3-a456-426614174001" - ], - "ImporterSettings": { - "TextureImporter": { - "MaxSize": 2048, - "MipLevels": 1 - } - } -} -``` - -## Usage Examples - -### Finding Assets - -```csharp -// Find by path -var guidResult = AssetDatabase.PathToGuid("Assets/Textures/logo.png"); -if (guidResult.IsSuccess) -{ - var guid = guidResult.Value; - // Use guid... -} - -// Find by GUID -var pathResult = AssetDatabase.GuidToPath(myGuid); -if (pathResult.IsSuccess) -{ - var path = pathResult.Value; - // Use path... -} - -// Search by name -var results = AssetDatabase.FindAssetsByName("logo"); - -// Search by tag -var textureAssets = await AssetDatabase.FindAssetsByTagAsync("Texture"); -``` - -### Creating and Managing Assets - -```csharp -// Create new asset -var content = Encoding.UTF8.GetBytes("Hello, World!"); -await AssetDatabase.CreateAssetAsync("Assets/test.txt", content); - -// Move asset -await AssetDatabase.MoveAssetAsync(guid, "Assets/NewFolder/test.txt"); - -// Copy asset -var newGuid = await AssetDatabase.CopyAssetAsync(guid, "Assets/test_copy.txt"); - -// Delete asset -await AssetDatabase.DeleteAssetAsync(guid); -``` - -### Working with Tags - -```csharp -// Get tags -var tagsResult = await AssetDatabase.GetAssetTagsAsync(guid); -if (tagsResult.IsSuccess) -{ - var tags = tagsResult.Value; -} - -// Set tags -await AssetDatabase.SetAssetTagsAsync(guid, new List { "UI", "Icon" }); -``` - -### Asset Importing - -```csharp -// Mark asset dirty for re-import -await AssetDatabase.MarkDirtyAsync(guid); - -// Import all dirty assets -await AssetDatabase.ImportDirtyAssetsAsync(); -``` - -## Creating Custom Importers - -To create a custom asset importer: - -1. Create a settings class inheriting from `ImporterSettings` -2. Create an importer class inheriting from `AssetImporter` -3. Add the `[AssetImporter]` attribute with supported extensions - -Example: - -```csharp -public class MyImporterSettings : ImporterSettings -{ - public bool SomeOption { get; set; } = true; -} - -[AssetImporter(".myext")] -public class MyImporter : AssetImporter -{ - public override async Task ImportAsync(string assetPath, AssetMeta meta) - { - var settings = GetSettings(meta); - - // Validate dependencies - var depResult = await ValidateDependenciesAsync(meta); - if (depResult.IsFailure) - { - return depResult; - } - - // Import logic here... - - return Result.Success(); - } -} -``` - -## Architecture Notes - -### Source of Truth - -The `.gmeta` files are the **source of truth** for asset information. The SQLite database is used only for: -- Caching for fast lookups -- Efficient querying and search operations -- Tracking dirty state - -If the database becomes inconsistent, it can be regenerated from the `.gmeta` files by calling `RefreshAsync()`. - -### Thread Safety - -All database operations use locks to ensure thread safety. File system watcher events are handled asynchronously to avoid blocking the main thread. - -### Error Handling - -The system uses the `Result` pattern for railway-oriented programming. All operations return `Result` or `Result` to indicate success or failure without throwing exceptions for expected failures. - -## Testing - -Unit tests should be added to verify: -- Metadata file generation and parsing -- Database consistency -- File operations (create, delete, move, copy) -- Asset importing -- Dependency tracking -- Tag management -- Search functionality - -## Future Improvements - -- Asset loader implementation (`LoadAsset`) -- Asset browser UI -- More sophisticated dependency resolution -- Asset preview generation -- Asset versioning and migration -- Orphaned entry cleanup in database -- Better error reporting and logging -- Asset import progress tracking -- Parallel asset importing -- Asset thumbnail generation diff --git a/Ghost.Editor.Core/SceneGraph/SceneGraph Plan.md b/Ghost.Editor.Core/SceneGraph/SceneGraph Plan.md index 34f0aea..d13fe61 100644 --- a/Ghost.Editor.Core/SceneGraph/SceneGraph Plan.md +++ b/Ghost.Editor.Core/SceneGraph/SceneGraph Plan.md @@ -63,7 +63,7 @@ When loading a scene, we need to reconstruct the entities and their relationship ### Data format -The scene data should be stored in a structured format (e.g., JSON or binary) that includes: +The scene data should be stored in a structured format (JSON and binary) that includes: - List of entities with their components and properties (Entities must in the order that file local id directly maps to the index in the list) - References between entities using file local IDs @@ -75,7 +75,7 @@ Binary format should be used in the runtime for better performance. The runtime Currently we strict the IComponent to must be unmanaged and blittable types. However, we also support ManagedEntity and ManagedEntityRef with ScriptComponent to allow OOP like logic for common gameplay logic that DOD pattern is not suitable for. -Serializing/deserializing with those components will be tricky. We can use MemoryPack for binary serialization/deserialization because it supports both unmanaged and managed types. +Serializing/deserializing with those components will be tricky. We can use MemoryPack (already installed) for binary serialization/deserialization because it supports both unmanaged and managed types. ## What need to implement diff --git a/Ghost.Editor.Core/SceneGraph/SceneNode.cs b/Ghost.Editor.Core/SceneGraph/SceneNode.cs index 23e8944..ce78ca7 100644 --- a/Ghost.Editor.Core/SceneGraph/SceneNode.cs +++ b/Ghost.Editor.Core/SceneGraph/SceneNode.cs @@ -1,5 +1,14 @@ +using Ghost.Editor.Core.Inspector; +using Microsoft.UI.Xaml; +using Microsoft.UI.Xaml.Controls; + namespace Ghost.Editor.Core.SceneGraph; -public sealed partial class SceneNode : SceneGraphNode +public sealed partial class SceneNode : SceneGraphNode, IInspectable { + public IconSource? Icon => throw new NotImplementedException(); + + public UIElement? HeaderContent => throw new NotImplementedException(); + + public UIElement? InspectorContent => throw new NotImplementedException(); } diff --git a/Ghost.Editor.Core/Utilities/TypeCache.cs b/Ghost.Editor.Core/Utilities/TypeCache.cs index 7ac65b5..c3e8890 100644 --- a/Ghost.Editor.Core/Utilities/TypeCache.cs +++ b/Ghost.Editor.Core/Utilities/TypeCache.cs @@ -9,7 +9,7 @@ public static class TypeCache static TypeCache() { - var loadableTypes = new List(); + var loadableTypes = new List(512); var assembliesToScan = AppDomain.CurrentDomain.GetAssemblies() .Where(a => a.GetCustomAttribute() != null); diff --git a/Ghost.UnitTest/AssetDatabaseIntegrationTest.cs b/Ghost.UnitTest/AssetDatabaseIntegrationTest.cs index a40ab9c..051b985 100644 --- a/Ghost.UnitTest/AssetDatabaseIntegrationTest.cs +++ b/Ghost.UnitTest/AssetDatabaseIntegrationTest.cs @@ -22,7 +22,7 @@ public class AssetDatabaseIntegrationTest // Create temporary test project structure _testProjectDir = Path.Combine(Path.GetTempPath(), "GhostAssetDBIntegration_" + Guid.NewGuid().ToString()); _testAssetsDir = Path.Combine(_testProjectDir, ProjectService.ASSETS_FOLDER); - + Directory.CreateDirectory(_testProjectDir); Directory.CreateDirectory(_testAssetsDir); Directory.CreateDirectory(Path.Combine(_testProjectDir, ProjectService.CACHE_FOLDER)); @@ -33,22 +33,22 @@ public class AssetDatabaseIntegrationTest // Create a minimal project file with required metadata var projectPath = Path.Combine(_testProjectDir, "TestProject.gproj"); - + // Create a proper ProjectMetadata instance var metadata = new Ghost.Data.Models.ProjectMetadata("TestProject", new Version(1, 0, 0)); - + await using var fileStream = File.Create(projectPath); await System.Text.Json.JsonSerializer.SerializeAsync(fileStream, metadata, Ghost.Data.JsonContext.Default.ProjectMetadata, TestContext.CancellationToken); await fileStream.FlushAsync(TestContext.CancellationToken); fileStream.Close(); // Set CurrentProject directly - var projectMetadataInfo = new Ghost.Data.Models.ProjectMetadataInfo(projectPath, metadata); + var projectMetadataInfo = new Data.Models.ProjectMetadataInfo(projectPath, metadata); ProjectService.CurrentProject = projectMetadataInfo; - + // Initialize AssetDatabase - AssetDatabase.Initialize(); - + AssetDatabase.Initialize(TestContext.CancellationToken); + // Give the file system watcher time to start await Task.Delay(100, TestContext.CancellationToken); } @@ -72,7 +72,7 @@ public class AssetDatabaseIntegrationTest try { // Add delay to allow file handles to be released - System.Threading.Thread.Sleep(100); + Thread.Sleep(100); Directory.Delete(_testProjectDir, true); } catch @@ -82,6 +82,18 @@ public class AssetDatabaseIntegrationTest } } + /// + /// Helper to wait for file system events to be processed. + /// + private async Task WaitForFileSystemEvents(int delayMs = 300) + { + await Task.Delay(delayMs, TestContext.CancellationToken); + AssetDatabase.FlushPendingCommands(); + + // Give a bit more time after flush for any final processing + await Task.Delay(50, TestContext.CancellationToken); + } + [TestMethod] public async Task TestAutoMetaGeneration_WhenFileCreated() { @@ -89,8 +101,8 @@ public class AssetDatabaseIntegrationTest var testFile = Path.Combine(_testAssetsDir, "test.txt"); await File.WriteAllTextAsync(testFile, "Hello World", TestContext.CancellationToken); - // Wait a bit for file system watcher to react - await Task.Delay(200, TestContext.CancellationToken); + // Wait for file system watcher to react and process commands + await WaitForFileSystemEvents(); // Check if meta file was auto-generated var metaFile = testFile + ".gmeta"; @@ -111,18 +123,18 @@ public class AssetDatabaseIntegrationTest await File.WriteAllTextAsync(Path.Combine(_testAssetsDir, "enemy.txt"), "data", TestContext.CancellationToken); // Wait for database to update - await Task.Delay(200, TestContext.CancellationToken); + await WaitForFileSystemEvents(); // Test wildcard search: player* - var results = await AssetDatabase.FindAssetsByNameAsync("player*"); + var results = await AssetDatabase.FindAssetsByNameAsync("player*", TestContext.CancellationToken); Assert.HasCount(3, results, "Should find 3 files matching 'player*'"); // Test single character wildcard: player? - results = await AssetDatabase.FindAssetsByNameAsync("player?.txt"); + results = await AssetDatabase.FindAssetsByNameAsync("player?.txt", TestContext.CancellationToken); Assert.HasCount(2, results, "Should find 2 files matching 'player?.txt'"); // Test exact match - results = await AssetDatabase.FindAssetsByNameAsync("enemy.txt"); + results = await AssetDatabase.FindAssetsByNameAsync("enemy.txt", TestContext.CancellationToken); Assert.HasCount(1, results, "Should find 1 file matching 'enemy.txt'"); } @@ -132,7 +144,7 @@ public class AssetDatabaseIntegrationTest // Create a file var originalPath = Path.Combine(_testAssetsDir, "original.txt"); await File.WriteAllTextAsync(originalPath, "data", TestContext.CancellationToken); - await Task.Delay(200, TestContext.CancellationToken); + await WaitForFileSystemEvents(); // Get the GUID before rename var guidResult = AssetDatabase.PathToGuid(originalPath); @@ -142,7 +154,7 @@ public class AssetDatabaseIntegrationTest // Rename via file system var newPath = Path.Combine(_testAssetsDir, "renamed.txt"); File.Move(originalPath, newPath); - await Task.Delay(200, TestContext.CancellationToken); + await WaitForFileSystemEvents(); // Check if meta file was also moved var newMetaPath = newPath + ".gmeta"; @@ -160,7 +172,7 @@ public class AssetDatabaseIntegrationTest // Create a file var filePath = Path.Combine(_testAssetsDir, "todelete.txt"); await File.WriteAllTextAsync(filePath, "data", TestContext.CancellationToken); - await Task.Delay(200, TestContext.CancellationToken); + await WaitForFileSystemEvents(); var guidResult = AssetDatabase.PathToGuid(filePath); Assert.IsTrue(guidResult.IsSuccess); @@ -168,7 +180,7 @@ public class AssetDatabaseIntegrationTest // Delete via file system File.Delete(filePath); - await Task.Delay(200, TestContext.CancellationToken); + await WaitForFileSystemEvents(); // Meta file should also be deleted var metaPath = filePath + ".gmeta"; @@ -183,9 +195,9 @@ public class AssetDatabaseIntegrationTest public async Task TestFileCreate_ViaAPI() { var filePath = Path.Combine(_testAssetsDir, "apiCreated.txt"); - + // Create via API - var result = await AssetDatabase.CreateAssetAsync(filePath); + var result = await AssetDatabase.CreateAssetAsync(filePath, TestContext.CancellationToken); Assert.IsTrue(result.IsSuccess, "Should create asset successfully"); // File and meta should exist @@ -203,7 +215,7 @@ public class AssetDatabaseIntegrationTest // Create initial file var sourcePath = Path.Combine(_testAssetsDir, "source.txt"); await File.WriteAllTextAsync(sourcePath, "data", TestContext.CancellationToken); - await Task.Delay(200, TestContext.CancellationToken); + await WaitForFileSystemEvents(); var guid = AssetDatabase.PathToGuid(sourcePath).Value; @@ -214,7 +226,7 @@ public class AssetDatabaseIntegrationTest var destPath = Path.Combine(subDir, "source.txt"); // Move via API - var result = await AssetDatabase.MoveAssetAsync(sourcePath, destPath); + var result = await AssetDatabase.MoveAssetAsync(sourcePath, destPath, TestContext.CancellationToken); Assert.IsTrue(result.IsSuccess, $"Should move asset successfully. Error: {result.Message}"); // Old file should not exist @@ -236,13 +248,13 @@ public class AssetDatabaseIntegrationTest // Create initial file var sourcePath = Path.Combine(_testAssetsDir, "tocopy.txt"); await File.WriteAllTextAsync(sourcePath, "data", TestContext.CancellationToken); - await Task.Delay(200, TestContext.CancellationToken); + await WaitForFileSystemEvents(); var sourceGuid = AssetDatabase.PathToGuid(sourcePath).Value; var destPath = Path.Combine(_testAssetsDir, "copied.txt"); // Copy via API - var result = await AssetDatabase.CopyAssetAsync(sourcePath, destPath); + var result = await AssetDatabase.CopyAssetAsync(sourcePath, destPath, TestContext.CancellationToken); Assert.IsTrue(result.IsSuccess, "Should copy asset successfully"); // Both files should exist @@ -260,12 +272,12 @@ public class AssetDatabaseIntegrationTest // Create initial file var filePath = Path.Combine(_testAssetsDir, "todelete2.txt"); await File.WriteAllTextAsync(filePath, "data", TestContext.CancellationToken); - await Task.Delay(200, TestContext.CancellationToken); + await WaitForFileSystemEvents(); var guid = AssetDatabase.PathToGuid(filePath).Value; // Delete via API - var result = await AssetDatabase.DeleteAssetAsync(filePath); + var result = await AssetDatabase.DeleteAssetAsync(filePath, TestContext.CancellationToken); Assert.IsTrue(result.IsSuccess, "Should delete asset successfully"); // File and meta should not exist @@ -289,7 +301,7 @@ public class AssetDatabaseIntegrationTest var fileName = $"race{i}.txt"; fileNames.Add(fileName); var filePath = Path.Combine(_testAssetsDir, fileName); - + tasks.Add(Task.Run(async () => { await File.WriteAllTextAsync(filePath, $"data{i}", TestContext.CancellationToken); @@ -297,16 +309,16 @@ public class AssetDatabaseIntegrationTest } await Task.WhenAll(tasks); - await Task.Delay(500, TestContext.CancellationToken); // Wait for all file system events + await WaitForFileSystemEvents(500); // Wait for all file system events // All files should have exactly one meta file foreach (var fileName in fileNames) { var filePath = Path.Combine(_testAssetsDir, fileName); var metaPath = filePath + ".gmeta"; - + Assert.IsTrue(File.Exists(metaPath), $"Meta file should exist for {fileName}"); - + // Read meta and verify it's valid JSON var metaContent = await File.ReadAllTextAsync(metaPath, TestContext.CancellationToken); Assert.Contains("Guid", metaContent, $"Meta file should be valid for {fileName}"); @@ -324,20 +336,20 @@ public class AssetDatabaseIntegrationTest await File.WriteAllTextAsync(file1, "data", TestContext.CancellationToken); await File.WriteAllTextAsync(file2, "data", TestContext.CancellationToken); await File.WriteAllTextAsync(file3, "data", TestContext.CancellationToken); - await Task.Delay(200, TestContext.CancellationToken); + await WaitForFileSystemEvents(); var guid1 = AssetDatabase.PathToGuid(file1).Value; var guid2 = AssetDatabase.PathToGuid(file2).Value; // Add tags - await AssetDatabase.SetAssetTagsAsync(guid1, new List { "Test", "Player" }); - await AssetDatabase.SetAssetTagsAsync(guid2, new List { "Test", "Enemy" }); + await AssetDatabase.SetAssetTagsAsync(guid1, new List { "Test", "Player" }, TestContext.CancellationToken); + await AssetDatabase.SetAssetTagsAsync(guid2, new List { "Test", "Enemy" }, TestContext.CancellationToken); // Search by tag - var testAssets = await AssetDatabase.FindAssetsByTagAsync("Test"); + var testAssets = await AssetDatabase.FindAssetsByTagAsync("Test", TestContext.CancellationToken); Assert.HasCount(2, testAssets, "Should find 2 assets with 'Test' tag"); - var playerAssets = await AssetDatabase.FindAssetsByTagAsync("Player"); + var playerAssets = await AssetDatabase.FindAssetsByTagAsync("Player", TestContext.CancellationToken); Assert.HasCount(1, playerAssets, "Should find 1 asset with 'Player' tag"); } @@ -347,14 +359,14 @@ public class AssetDatabaseIntegrationTest // Create a file var filePath = Path.Combine(_testAssetsDir, "refresh.txt"); await File.WriteAllTextAsync(filePath, "data", TestContext.CancellationToken); - await Task.Delay(200, TestContext.CancellationToken); + await WaitForFileSystemEvents(); var guid1 = AssetDatabase.PathToGuid(filePath).Value; // Call RefreshAsync multiple times - await AssetDatabase.RefreshAsync(); - await AssetDatabase.RefreshAsync(); - await AssetDatabase.RefreshAsync(); + await AssetDatabase.RefreshAsync(TestContext.CancellationToken); + await AssetDatabase.RefreshAsync(TestContext.CancellationToken); + await AssetDatabase.RefreshAsync(TestContext.CancellationToken); // GUID should remain the same var guid2 = AssetDatabase.PathToGuid(filePath).Value; @@ -364,4 +376,12 @@ public class AssetDatabaseIntegrationTest var metaFiles = Directory.GetFiles(_testAssetsDir, "refresh.txt.gmeta"); Assert.HasCount(1, metaFiles, "Should have exactly one meta file"); } + + [TestMethod] + public async Task ThreadSafetyTest() + { + var testFile = Path.Combine(_testAssetsDir, "test.txt"); + await File.WriteAllTextAsync(testFile, "Hello World", TestContext.CancellationToken); + await AssetDatabase.RefreshAsync(TestContext.CancellationToken); // This will cause race conditions if not handle properly because both AssetDatabase and FileSystemWatcher are involved + } }