using Ghost.Core; using Ghost.Data.Services; using System.Text.Json; using System.Text.Json.Serialization; using System.Threading.Channels; namespace Ghost.Editor.Core.AssetHandle; /// /// Command types for asset database operations. /// internal enum AssetCommandType { FileCreated, FileModified, FileDeleted, FileRenamed, ManualRefresh } /// /// Represents a command to process an asset operation. /// internal readonly record struct AssetCommand( AssetCommandType Type, string Path, string? OldPath = null, DateTime Timestamp = default ); /// /// Centralized asset database that manages all assets in the project. /// Handles asset registration, lookup, importing, and dependency management. /// Uses SQLite for persistent storage and efficient querying. /// public static partial class AssetDatabase { private static FileSystemWatcher? s_watcher; private static readonly Lock s_dbLock = new(); private static readonly Dictionary s_assetPathLookup = new(); private static readonly Dictionary s_pathAssetLookup = new(); // In-memory dirty asset tracking (for runtime modifications only) private static readonly HashSet s_dirtyAssets = new(); // Command buffer pattern - Channel for file system event commands private static Channel? s_commandChannel; private static Timer? s_commandProcessorTimer; private static readonly HashSet s_pendingCommandPaths = new(); // Track paths with pending commands private static readonly Lock s_commandLock = new(); private static bool s_autoRefreshEnabled = true; private static readonly Queue s_waitingCommands = new(); // Commands waiting for manual refresh private static TaskCompletionSource? s_refreshTcs; // Initialization guard private static readonly Lock s_initializationLock = new(); private static bool s_initialized = false; private static readonly TimeSpan s_debounceDelay = TimeSpan.FromMilliseconds(100); private static readonly JsonSerializerOptions s_defaultJsonOptions = new() { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) } }; public static DirectoryInfo? AssetsDirectory { get; private set; } /// /// Initialize the asset database. /// Must be called after project is loaded. /// internal static async Task Initialize(CancellationToken token = default) { lock (s_initializationLock) { if (s_initialized) { return; // Already initialized, skip } s_initialized = true; } if (ProjectService.CurrentProject.Metadata == null) { throw new InvalidOperationException("Project metadata is not initialized. Ensure that the project is loaded before accessing the AssetDatabase."); } AssetsDirectory = new DirectoryInfo(Path.Combine(Path.GetDirectoryName(ProjectService.CurrentProject.Path)!, ProjectService.ASSETS_FOLDER)); s_commandChannel = Channel.CreateUnbounded(new UnboundedChannelOptions { SingleReader = false, // Timer callback reads SingleWriter = false // Multiple FS events can write }); // Initialize command processor timer (starts disabled, triggered by events) s_commandProcessorTimer = new Timer(ProcessPendingCommands, null, Timeout.Infinite, Timeout.Infinite); await InitializeDatabaseAsync(token); await LoadAssetCacheFromDatabaseAsync(token); s_watcher = new FileSystemWatcher { Path = AssetsDirectory.FullName, IncludeSubdirectories = true, EnableRaisingEvents = true, NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.LastWrite }; InitializeAssetHandle(); InitializeMetaData(); await ValidateAndFixDatabaseAsync(token); } /// /// Validate the asset database and fix any inconsistencies. /// Checks for missing/corrupted assets and regenerates metadata as needed. /// private static async Task ValidateAndFixDatabaseAsync(CancellationToken token = default) { if (AssetsDirectory == null) { return Result.Failure("AssetsDirectory not initialized"); } try { // Scan all files in assets directory var allFiles = Directory.GetFiles(AssetsDirectory.FullName, "*.*", SearchOption.AllDirectories) .Where(f => !f.EndsWith(Utilities.FileExtensions.META_FILE_EXTENSION, StringComparison.OrdinalIgnoreCase)); // Ensure all files have metadata foreach (var file in allFiles) { var metaPath = file + Utilities.FileExtensions.META_FILE_EXTENSION; if (!File.Exists(metaPath)) { await GenerateMetaFileAsync(file, token); } else { // Validate and update database var metaResult = await ReadMetaFileAsync(file, token); if (metaResult.IsSuccess) { var fileHash = await CalculateFileHashAsync(file, token); await UpsertAssetAsync(file, metaResult.Value, fileHash, null, token); } else { // Corrupted meta file - regenerate await GenerateMetaFileAsync(file, token); } } } // Remove orphaned entries from database (files that no longer exist) await RemoveOrphanedEntriesAsync(token); return Result.Success(); } catch (Exception ex) { return Result.Failure($"Failed to validate database: {ex.Message}"); } } /// /// Refresh the asset database manually. /// Scans the project directory for changes and processes any queued file system events. /// public static async Task RefreshAsync(CancellationToken token = default) { // Flush waiting commands to channel lock (s_commandLock) { while (s_waitingCommands.TryDequeue(out var cmd)) { s_commandChannel?.Writer.TryWrite(cmd); } s_refreshTcs = new TaskCompletionSource(); } s_commandChannel?.Writer.TryWrite(new AssetCommand(AssetCommandType.ManualRefresh, string.Empty)); s_commandProcessorTimer?.Change(TimeSpan.Zero, Timeout.InfiniteTimeSpan); if (!await s_refreshTcs.Task.WaitAsync(token)) { return Result.Failure("Asset database refresh failed"); } return Result.Success(); } /// /// Mark an asset as dirty (modified in memory but not yet saved). /// This state is NOT persisted and will be lost on application restart. /// public static void MarkDirty(Guid assetGuid) { lock (s_dbLock) { s_dirtyAssets.Add(assetGuid); } } /// /// Check if an asset is marked as dirty. /// public static bool IsDirty(Guid assetGuid) { lock (s_dbLock) { return s_dirtyAssets.Contains(assetGuid); } } /// /// Get all dirty assets. /// public static Guid[] GetDirtyAssets() { lock (s_dbLock) { return s_dirtyAssets.ToArray(); } } /// /// Clear dirty flag for an asset (typically after saving). /// public static void ClearDirty(Guid assetGuid) { lock (s_dbLock) { s_dirtyAssets.Remove(assetGuid); } } /// /// Clear all dirty flags. /// public static void ClearAllDirty() { lock (s_dbLock) { s_dirtyAssets.Clear(); } } /// /// Enable or disable automatic asset database refresh. /// When disabled, file system events are queued and processed only when RefreshAsync() is called. /// public static void SetAutoRefresh(bool enabled) { s_autoRefreshEnabled = enabled; } /// /// Process all pending commands immediately (synchronous, for testing). /// internal static void FlushPendingCommands() { // Stop timer temporarily s_commandProcessorTimer?.Change(Timeout.Infinite, Timeout.Infinite); // Give a tiny bit of time for any in-flight file watcher events to post to channel Thread.Sleep(50); // Process all commands now ProcessPendingCommands(null); } /// /// Post a command to the command channel for processing. /// private static void PostCommand(AssetCommand command) { if (s_commandChannel == null) { return; } if (s_autoRefreshEnabled) { // Add to pending paths for temp file tracking lock (s_commandLock) { s_pendingCommandPaths.Add(command.Path); if (command.OldPath != null) { s_pendingCommandPaths.Add(command.OldPath); } } // Write to channel (lock-free, async-safe) s_commandChannel.Writer.TryWrite(command); // Start or reset timer (safe even if not initialized yet) s_commandProcessorTimer?.Change(s_debounceDelay, Timeout.InfiniteTimeSpan); } else { // Queue for manual refresh lock (s_commandLock) { s_waitingCommands.Enqueue(command); } } } /// /// Timer callback to process pending commands. /// private static async void ProcessPendingCommands(object? state) { try { // Collect all pending commands var commands = new List(); while (s_commandChannel?.Reader.TryRead(out var cmd) == true) { commands.Add(cmd); } // Group commands by path (last command wins) var commandsByPath = new Dictionary(); foreach (var cmd in commands) { commandsByPath[cmd.Path] = cmd; } // NOTE: We handle the temp file filtering in each command handler now // We should able to remove this allocation heavy code // Filter out temp files (files that were created then deleted) // lock (s_commandLock) // { // var pathsToProcess = commandsByPath.Keys.ToList(); // foreach (var path in pathsToProcess) // { // // If file was created/modified but doesn't exist anymore, skip // if (!File.Exists(path) && commandsByPath[path].Type != AssetCommandType.FileDeleted) // { // commandsByPath.Remove(path); // } // } // // // Clear pending paths // s_pendingCommandPaths.Clear(); // } // Execute commands foreach (var cmd in commandsByPath.Values) { await ExecuteCommandAsync(cmd); } s_refreshTcs?.SetResult(true); } catch (Exception ex) { Logger.LogError($"Error processing commands: {ex.Message}"); s_refreshTcs?.SetResult(false); } } /// /// Execute a single asset command. /// private static async Task ExecuteCommandAsync(AssetCommand command) { switch (command.Type) { case AssetCommandType.FileCreated: await HandleFileCreatedAsync(command.Path); break; case AssetCommandType.FileModified: await HandleFileModifiedAsync(command.Path); break; case AssetCommandType.FileDeleted: await HandleFileDeletedAsync(command.Path); break; case AssetCommandType.FileRenamed: if (command.OldPath != null) { await HandleFileRenamedAsync(command.OldPath, command.Path); } break; case AssetCommandType.ManualRefresh: await ValidateAndFixDatabaseAsync(CancellationToken.None); break; } } /// /// Handle file created event. /// private static async Task HandleFileCreatedAsync(string path) { if (!File.Exists(path)) { return; } await GenerateMetaFileAsync(path, CancellationToken.None); } /// /// Handle file modified event. /// private static async Task HandleFileModifiedAsync(string path) { if (!File.Exists(path)) { return; } // Check if file hash changed var metaResult = await ReadMetaFileAsync(path, CancellationToken.None); if (metaResult.IsFailure) { // No .gmeta file - treat this as a new file creation await HandleFileCreatedAsync(path); return; } // Calculate new hash and compare against database var newHash = await CalculateFileHashAsync(path, CancellationToken.None); var oldHash = await GetFileHashAsync(metaResult.Value.Guid, CancellationToken.None); if (oldHash != newHash) { // File changed - update database and mark as dirty await UpsertAssetAsync(path, metaResult.Value, newHash, null, CancellationToken.None); MarkDirty(metaResult.Value.Guid); } } /// /// Handle file deleted event. /// private static async Task HandleFileDeletedAsync(string path) { if (!File.Exists(path)) { return; } var metaFileResult = GetMetaFilePath(path); if (metaFileResult.IsSuccess && File.Exists(metaFileResult.Value)) { try { var metaResult = await ReadMetaFileAsync(path, CancellationToken.None); if (metaResult.IsSuccess) { var meta = metaResult.Value; // Remove from database await RemoveAssetFromDatabaseAsync(meta.Guid, CancellationToken.None); // Mark dependent assets as dirty await MarkDependentAssetsDirtyAsync(meta.Guid); } File.Delete(metaFileResult.Value); } catch (Exception ex) { Logger.LogError($"Error deleting asset metadata: {ex.Message}"); } } } /// /// Handle file renamed event. /// private static async Task HandleFileRenamedAsync(string oldPath, string newPath) { if (!File.Exists(oldPath)) { return; } if (File.Exists(newPath)) { Logger.LogWarning($"Cannot rename asset from '{oldPath}' to '{newPath}': target file already exists."); return; } var oldMetaPath = oldPath + Utilities.FileExtensions.META_FILE_EXTENSION; var newMetaPath = newPath + Utilities.FileExtensions.META_FILE_EXTENSION; if (File.Exists(newMetaPath)) { // Validate and update await GenerateMetaFileAsync(newPath, CancellationToken.None); } else if (File.Exists(oldMetaPath)) { // Move meta file File.Move(oldMetaPath, newMetaPath); // Update database with new path and recalculated hash var metaResult = await ReadMetaFileAsync(newPath, CancellationToken.None); if (metaResult.IsSuccess) { var fileHash = await CalculateFileHashAsync(newPath, CancellationToken.None); await UpsertAssetAsync(newPath, metaResult.Value, fileHash, null, CancellationToken.None); } } else { // Generate new meta file await GenerateMetaFileAsync(newPath, CancellationToken.None); } // Delete old meta if it still exists if (File.Exists(oldMetaPath) && oldMetaPath != newMetaPath) { try { File.Delete(oldMetaPath); } catch { // Ignore } } } /// /// Shutdown the asset database. /// Disposes resources and closes database connections. /// internal static void Shutdown() { lock (s_initializationLock) { if (!s_initialized) { return; // Not initialized, nothing to shutdown } s_watcher?.Dispose(); s_watcher = null; s_commandProcessorTimer?.Dispose(); s_commandProcessorTimer = null; s_dbConnection?.Close(); s_dbConnection?.Dispose(); s_dbConnection = null; s_assetPathLookup.Clear(); s_pathAssetLookup.Clear(); s_dirtyAssets.Clear(); s_pendingCommandPaths.Clear(); s_waitingCommands.Clear(); s_importerInstances.Clear(); s_importerTypeLookup.Clear(); s_initialized = false; } } }