using Ghost.Core; using Ghost.Data.Services; using System.Collections.Concurrent; using System.Text.Json; using System.Text.Json.Serialization; using System.Threading.Channels; namespace Ghost.Editor.Core.AssetHandle; /// /// Command types for asset database operations. /// internal enum AssetCommandType { FileCreated, FileModified, FileDeleted, FileRenamed, ManualRefresh } /// /// Represents a command to process an asset operation. /// internal readonly record struct AssetCommand( AssetCommandType Type, string Path, string? OldPath = null, DateTime Timestamp = default ); /// /// Centralized asset database that manages all assets in the project. /// Handles asset registration, lookup, importing, and dependency management. /// Uses SQLite for persistent storage and efficient querying. /// public static partial class AssetDatabase { private static FileSystemWatcher? s_watcher; private static readonly Lock s_dbLock = new(); private static readonly Dictionary s_assetPathLookup = new(); private static readonly Dictionary s_pathAssetLookup = new(); // In-memory dirty asset tracking (for runtime modifications only) // TODO: We do not handle the reimporting of dirty assets yet private static readonly HashSet s_dirtyAssets = new(); // Command buffer pattern - Channel for file system event commands private static Channel? s_commandChannel; private static Timer? s_commandProcessorTimer; private static readonly Lock s_commandLock = new(); private static readonly ConcurrentQueue s_waitingCommands = new(); // Commands waiting for manual refresh private static bool s_autoRefreshEnabled = true; // Initialization guard private static readonly Lock s_initializationLock = new(); private static bool s_initialized = false; private static readonly TimeSpan s_debounceDelay = TimeSpan.FromMilliseconds(100); private static ManualResetEventSlim s_resetEventSlim = new(false); private static readonly JsonSerializerOptions s_defaultJsonOptions = new() { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) } }; public static DirectoryInfo? AssetsDirectory { get; private set; } /// /// Initialize the asset database. /// Must be called after project is loaded. /// internal static async Task Initialize(CancellationToken token = default) { lock (s_initializationLock) { if (s_initialized) { return; } s_initialized = true; } if (ProjectService.CurrentProject.Metadata == null) { throw new InvalidOperationException("Project metadata is not initialized. Ensure that the project is loaded before accessing the AssetDatabase."); } AssetsDirectory = new DirectoryInfo(Path.Combine(Path.GetDirectoryName(ProjectService.CurrentProject.Path)!, ProjectService.ASSETS_FOLDER)); s_commandChannel = Channel.CreateUnbounded(new UnboundedChannelOptions { SingleReader = false, SingleWriter = false }); // Initialize command processor timer (starts disabled, triggered by events) s_commandProcessorTimer = new Timer(ProcessPendingCommands, null, Timeout.Infinite, Timeout.Infinite); await InitializeDatabaseAsync(token); await LoadAssetCacheFromDatabaseAsync(token); s_watcher = new FileSystemWatcher { Path = AssetsDirectory.FullName, IncludeSubdirectories = true, EnableRaisingEvents = true, NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.LastWrite }; InitializeAssetHandle(); InitializeMetaData(); // TODO: Timestamp fake instead of full scan. await ValidateAndFixDatabaseAsync(token); } /// /// Validate the asset database and fix any inconsistencies. /// Checks for missing/corrupted assets and regenerates metadata as needed. /// private static async Task ValidateAndFixDatabaseAsync(CancellationToken token = default) { if (AssetsDirectory == null) { return Result.Failure("AssetsDirectory not initialized"); } try { // Scan all files in assets directory var allFiles = Directory.EnumerateFiles(AssetsDirectory.FullName, "*.*", SearchOption.AllDirectories) .Where(f => !f.EndsWith(Utilities.FileExtensions.META_FILE_EXTENSION, StringComparison.OrdinalIgnoreCase)); // Ensure all files have metadata foreach (var file in allFiles) { var metaPath = file + Utilities.FileExtensions.META_FILE_EXTENSION; if (!File.Exists(metaPath)) { await GenerateMetaFileAsync(file, token); } else { // Validate and update database var metaResult = await ReadMetaFileAsync(file, token); if (metaResult.IsSuccess) { var fileHash = await CalculateFileHashAsync(file, token); await UpsertAssetAsync(file, metaResult.Value, fileHash, null, token); } else { // Corrupted meta file - regenerate await GenerateMetaFileAsync(file, token); } } } // Remove orphaned entries from database (files that no longer exist) await RemoveOrphanedEntriesAsync(token); return Result.Success(); } catch (Exception ex) { return Result.Failure($"Failed to validate database: {ex.Message}"); } } /// /// Refresh the asset database manually. /// Scans the project directory for changes and processes any queued file system events. /// public static async Task RefreshAsync(CancellationToken token = default) { // Flush waiting commands to channel while (s_waitingCommands.TryDequeue(out var cmd)) { s_commandChannel?.Writer.TryWrite(cmd); } s_resetEventSlim.Reset(); s_commandChannel?.Writer.TryWrite(new AssetCommand(AssetCommandType.ManualRefresh, string.Empty)); s_commandProcessorTimer?.Change(TimeSpan.Zero, Timeout.InfiniteTimeSpan); await Task.Run(s_resetEventSlim.Wait, token); return Result.Success(); } /// /// Mark an asset as dirty (modified in memory but not yet saved). /// This state is NOT persisted and will be lost on application restart. /// public static void MarkDirty(Guid assetGuid) { lock (s_dbLock) { s_dirtyAssets.Add(assetGuid); } } /// /// Check if an asset is marked as dirty. /// public static bool IsDirty(Guid assetGuid) { lock (s_dbLock) { return s_dirtyAssets.Contains(assetGuid); } } /// /// Get all dirty assets. /// public static Guid[] GetDirtyAssets() { lock (s_dbLock) { return s_dirtyAssets.ToArray(); } } /// /// Clear dirty flag for an asset (typically after saving). /// public static void ClearDirty(Guid assetGuid) { lock (s_dbLock) { s_dirtyAssets.Remove(assetGuid); } } /// /// Clear all dirty flags. /// public static void ClearAllDirty() { lock (s_dbLock) { s_dirtyAssets.Clear(); } } /// /// Enable or disable automatic asset database refresh. /// When disabled, file system events are queued and processed only when RefreshAsync() is called. /// public static void SetAutoRefresh(bool enabled) { s_autoRefreshEnabled = enabled; } internal static void FlushPendingCommands() { // Stop timer temporarily s_commandProcessorTimer?.Change(Timeout.Infinite, Timeout.Infinite); // Give a tiny bit of time for any in-flight file watcher events to post to channel Thread.Sleep(50); // Process all commands now ProcessPendingCommands(null); } private static async ValueTask PostCommandAsync(AssetCommand command, CancellationToken token = default) { if (s_commandChannel == null) { return; } if (s_autoRefreshEnabled) { await s_commandChannel.Writer.WriteAsync(command, token); s_commandProcessorTimer?.Change(s_debounceDelay, Timeout.InfiniteTimeSpan); } else { s_waitingCommands.Enqueue(command); } } private static async void ProcessPendingCommands(object? state) { if (s_commandChannel == null) { return; } try { // // Collect all pending commands // var commands = new List(); // // while (s_commandChannel.Reader.TryRead(out var cmd)) // { // commands.Add(cmd); // } // // Group commands by path (last command wins) // var commandsByPath = new Dictionary(); // foreach (var cmd in commands) // { // commandsByPath[cmd.Path] = cmd; // } // NOTE: We handle the temp file filtering in each command handler now // We should able to remove this allocation heavy code // Filter out temp files (files that were created then deleted) // lock (s_commandLock) // { // var pathsToProcess = commandsByPath.Keys.ToList(); // foreach (var path in pathsToProcess) // { // // If file was created/modified but doesn't exist anymore, skip // if (!File.Exists(path) && commandsByPath[path].Type != AssetCommandType.FileDeleted) // { // commandsByPath.Remove(path); // } // } // // // Clear pending paths // s_pendingCommandPaths.Clear(); // } // Execute commands // NOTE: We many don't need to collect all commands first, just process as we read. // Channel in c# is thread-safe for multiple readers/writers. //await foreach (var cmd in s_commandChannel.Reader.ReadAllAsync()) //{ // await ExecuteCommandAsync(cmd); //} while (s_commandChannel.Reader.TryRead(out var cmd)) { await ExecuteCommandAsync(cmd); } await ImportDirtyAssetsAsync(); } catch (Exception ex) { Logger.LogError($"Error processing commands: {ex.Message}"); } finally { s_resetEventSlim.Set(); } } private static async ValueTask ExecuteCommandAsync(AssetCommand command) { switch (command.Type) { case AssetCommandType.FileCreated: await HandleFileCreatedAsync(command.Path); break; case AssetCommandType.FileModified: await HandleFileModifiedAsync(command.Path); break; case AssetCommandType.FileDeleted: await HandleFileDeletedAsync(command.Path); break; case AssetCommandType.FileRenamed: if (command.OldPath != null) { await HandleFileRenamedAsync(command.OldPath, command.Path); } break; case AssetCommandType.ManualRefresh: await ValidateAndFixDatabaseAsync(CancellationToken.None); break; } } private static async ValueTask HandleFileCreatedAsync(string path) { if (!File.Exists(path)) { return; } await GenerateMetaFileAsync(path, CancellationToken.None); } private static async ValueTask HandleFileModifiedAsync(string path) { if (!File.Exists(path)) { return; } // Check if file hash changed var metaResult = await ReadMetaFileAsync(path, CancellationToken.None); if (metaResult.IsFailure) { // No .gmeta file - treat this as a new file creation await HandleFileCreatedAsync(path); return; } var newHash = await CalculateFileHashAsync(path, CancellationToken.None); var oldHash = await GetFileHashAsync(metaResult.Value.Guid, CancellationToken.None); if (oldHash != newHash) { // File changed - update database and mark as dirty await UpsertAssetAsync(path, metaResult.Value, newHash, null, CancellationToken.None); MarkDirty(metaResult.Value.Guid); } } private static async ValueTask HandleFileDeletedAsync(string path) { var metaFileResult = GetMetaFilePath(path); if (metaFileResult.IsSuccess && File.Exists(metaFileResult.Value)) { try { var metaResult = await ReadMetaFileAsync(path, CancellationToken.None); if (metaResult.IsSuccess) { var meta = metaResult.Value; // Remove from database await RemoveAssetFromDatabaseAsync(meta.Guid, CancellationToken.None); // Mark dependent assets as dirty await MarkDependentAssetsDirtyAsync(meta.Guid); } File.Delete(metaFileResult.Value); } catch (Exception ex) { Logger.LogError($"Error deleting asset metadata: {ex.Message}"); } } } private static async ValueTask HandleFileRenamedAsync(string oldPath, string newPath) { var oldMetaPath = oldPath + Utilities.FileExtensions.META_FILE_EXTENSION; var newMetaPath = newPath + Utilities.FileExtensions.META_FILE_EXTENSION; if (File.Exists(newMetaPath)) { // Validate and update await GenerateMetaFileAsync(newPath, CancellationToken.None); } else if (File.Exists(oldMetaPath)) { // Move meta file File.Move(oldMetaPath, newMetaPath); // Update database with new path and recalculated hash var metaResult = await ReadMetaFileAsync(newPath, CancellationToken.None); if (metaResult.IsSuccess) { var fileHash = await CalculateFileHashAsync(newPath, CancellationToken.None); await UpsertAssetAsync(newPath, metaResult.Value, fileHash, null, CancellationToken.None); } } else { // Generate new meta file await GenerateMetaFileAsync(newPath, CancellationToken.None); } // Delete old meta if it still exists if (File.Exists(oldMetaPath) && oldMetaPath != newMetaPath) { try { File.Delete(oldMetaPath); } catch { } } } internal static void Shutdown() { lock (s_initializationLock) { if (!s_initialized) { return; } s_watcher?.Dispose(); s_watcher = null; s_commandProcessorTimer?.Dispose(); s_commandProcessorTimer = null; s_dbConnection?.Close(); s_dbConnection?.Dispose(); s_dbConnection = null; s_assetPathLookup.Clear(); s_pathAssetLookup.Clear(); s_dirtyAssets.Clear(); s_waitingCommands.Clear(); s_importerInstances.Clear(); s_importerTypeLookup.Clear(); s_initialized = false; } } }