Misaki/GhostEngine
2
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update asset database

Browse Source
This commit is contained in:
Misaki
2026-01-29 14:03:24 +09:00
parent 8a5795069f
commit e71851550b
16 changed files with 879 additions and 646 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
Ghost.Editor.Core/AssetHandle/AssetDatabase.FileOps.cs
View File

@@ -11,7 +11,7 @@ public static partial class AssetDatabase
/// <param name="assetPath">Path to create the asset at.</param>
/// <param name="content">Content to write to the asset file.</param>
/// <returns>Result indicating success or failure.</returns>
public static async Task<Result> CreateAssetAsync(string assetPath, byte[] content)
public static async ValueTask<Result> CreateAssetAsync(string assetPath, ReadOnlyMemory<byte> content, CancellationToken token = default)
{
if (AssetsDirectory == null)
{
@@ -36,11 +36,12 @@ public static partial class AssetDatabase
Directory.CreateDirectory(directory);
}
await File.WriteAllBytesAsync(assetPath, content);
using var fs = File.Create(assetPath);
await fs.WriteAsync(content, token);
// GenerateMetaFileAsync will be called automatically by the file watcher
// But we'll call it directly to ensure it's created immediately
await GenerateMetaFileAsync(assetPath);
await GenerateMetaFileAsync(assetPath, token);
return Result.Success();
}
@@ -56,9 +57,9 @@ public static partial class AssetDatabase
/// </summary>
/// <param name="assetPath">Path to create the asset at.</param>
/// <returns>Result indicating success or failure.</returns>
public static async Task<Result> CreateAssetAsync(string assetPath)
public static ValueTask<Result> CreateAssetAsync(string assetPath, CancellationToken token = default)
{
return await CreateAssetAsync(assetPath, Array.Empty<byte>());
return CreateAssetAsync(assetPath, ReadOnlyMemory<byte>.Empty, token);
}
/// <summary>
@@ -66,7 +67,7 @@ public static partial class AssetDatabase
/// </summary>
/// <param name="guid">GUID of the asset to delete.</param>
/// <returns>Result indicating success or failure.</returns>
public static async Task<Result> DeleteAssetAsync(Guid guid)
public static async ValueTask<Result> DeleteAssetAsync(Guid guid, CancellationToken token = default)
{
var pathResult = GuidToPath(guid);
if (pathResult.IsFailure)
@@ -98,7 +99,7 @@ public static partial class AssetDatabase
}
// Remove from database
await RemoveAssetFromDatabaseAsync(guid);
await RemoveAssetFromDatabaseAsync(guid, token);
return Result.Success();
}
@@ -113,15 +114,15 @@ public static partial class AssetDatabase
/// </summary>
/// <param name="assetPath">Path to the asset to delete.</param>
/// <returns>Result indicating success or failure.</returns>
public static async Task<Result> DeleteAssetAsync(string assetPath)
public static ValueTask<Result> DeleteAssetAsync(string assetPath, CancellationToken token = default)
{
var guidResult = PathToGuid(assetPath);
if (guidResult.IsFailure)
{
return Result.Failure(guidResult.Message);
return new ValueTask<Result>(Task.FromResult(Result.Failure(guidResult.Message)));
}
return await DeleteAssetAsync(guidResult.Value);
return DeleteAssetAsync(guidResult.Value, token);
}
/// <summary>
@@ -130,7 +131,7 @@ public static partial class AssetDatabase
/// <param name="guid">GUID of the asset to move.</param>
/// <param name="newPath">New path for the asset (relative or absolute).</param>
/// <returns>Result indicating success or failure.</returns>
public static async Task<Result> MoveAssetAsync(Guid guid, string newPath)
public static async ValueTask<Result> MoveAssetAsync(Guid guid, string newPath, CancellationToken token = default)
{
var oldPathResult = GuidToPath(guid);
if (oldPathResult.IsFailure)
@@ -174,45 +175,27 @@ public static partial class AssetDatabase
}
// Read metadata and calculate hash before moving
var metaResult = await ReadMetaFileAsync(oldFullPathResult.Value);
var metaResult = await ReadMetaFileAsync(oldFullPathResult.Value, token);
if (metaResult.IsFailure)
{
return Result.Failure(metaResult.Message);
}
var fileHash = await CalculateFileHashAsync(oldFullPathResult.Value);
var fileHash = await CalculateFileHashAsync(oldFullPathResult.Value, token);
// Temporarily disable file watcher to prevent race conditions
var watcherWasEnabled = s_watcher?.EnableRaisingEvents ?? false;
if (s_watcher != null)
// Move the asset file
File.Move(oldFullPathResult.Value, newPath);
// Move the .gmeta file
var oldMetaPath = oldFullPathResult.Value + Utilities.FileExtensions.META_FILE_EXTENSION;
var newMetaPath = newPath + Utilities.FileExtensions.META_FILE_EXTENSION;
if (File.Exists(oldMetaPath))
{
s_watcher.EnableRaisingEvents = false;
File.Move(oldMetaPath, newMetaPath);
}
try
{
// Move the asset file
File.Move(oldFullPathResult.Value, newPath);
// Move the .gmeta file
var oldMetaPath = oldFullPathResult.Value + Utilities.FileExtensions.META_FILE_EXTENSION;
var newMetaPath = newPath + Utilities.FileExtensions.META_FILE_EXTENSION;
if (File.Exists(oldMetaPath))
{
File.Move(oldMetaPath, newMetaPath);
}
// Update database with new path (hash remains the same since content didn't change)
await UpsertAssetAsync(newPath, metaResult.Value, fileHash);
}
finally
{
// Re-enable file watcher
if (s_watcher != null && watcherWasEnabled)
{
s_watcher.EnableRaisingEvents = true;
}
}
// Update database directly (bypassing file watcher)
await UpsertAssetAsync(newPath, metaResult.Value, fileHash, null, token);
return Result.Success();
}
@@ -228,15 +211,15 @@ public static partial class AssetDatabase
/// <param name="oldPath">Current path of the asset.</param>
/// <param name="newPath">New path for the asset (relative or absolute).</param>
/// <returns>Result indicating success or failure.</returns>
public static async Task<Result> MoveAssetAsync(string oldPath, string newPath)
public static ValueTask<Result> MoveAssetAsync(string oldPath, string newPath, CancellationToken token = default)
{
var guidResult = PathToGuid(oldPath);
if (guidResult.IsFailure)
{
return Result.Failure(guidResult.Message);
return ValueTask.FromResult(Result.Failure(guidResult.Message));
}
return await MoveAssetAsync(guidResult.Value, newPath);
return MoveAssetAsync(guidResult.Value, newPath, token);
}
/// <summary>
@@ -245,7 +228,7 @@ public static partial class AssetDatabase
/// <param name="guid">GUID of the asset to copy.</param>
/// <param name="newPath">New path for the copied asset (relative or absolute).</param>
/// <returns>Result containing the new asset's GUID.</returns>
public static async Task<Result<Guid>> CopyAssetAsync(Guid guid, string newPath)
public static async ValueTask<Result<Guid>> CopyAssetAsync(Guid guid, string newPath, CancellationToken token = default)
{
var oldPathResult = GuidToPath(guid);
if (oldPathResult.IsFailure)
@@ -288,10 +271,12 @@ public static partial class AssetDatabase
Directory.CreateDirectory(directory);
}
File.Copy(oldFullPathResult.Value, newPath);
await using var oldFs = File.OpenRead(oldFullPathResult.Value);
await using var newFs = File.Create(newPath);
await oldFs.CopyToAsync(newFs, token);
// Generate new metadata with new GUID
await GenerateMetaFileAsync(newPath);
await GenerateMetaFileAsync(newPath, token);
// Get the new GUID
var newGuidResult = PathToGuid(newPath);
@@ -314,47 +299,54 @@ public static partial class AssetDatabase
/// <param name="sourcePath">Path of the asset to copy.</param>
/// <param name="destPath">New path for the copied asset (relative or absolute).</param>
/// <returns>Result containing the new asset's GUID.</returns>
public static async Task<Result<Guid>> CopyAssetAsync(string sourcePath, string destPath)
public static ValueTask<Result<Guid>> CopyAssetAsync(string sourcePath, string destPath, CancellationToken token = default)
{
var guidResult = PathToGuid(sourcePath);
if (guidResult.IsFailure)
{
return Result<Guid>.Failure(guidResult.Message);
return new ValueTask<Result<Guid>>(Task.FromResult(Result<Guid>.Failure(guidResult.Message)));
}
return await CopyAssetAsync(guidResult.Value, destPath);
return CopyAssetAsync(guidResult.Value, destPath, token);
}
/// <summary>
/// Mark an asset as dirty for re-importing.
/// Mark an asset as dirty for re-importing (in-memory only).
/// </summary>
/// <param name="guid">GUID of the asset to mark dirty.</param>
/// <returns>Result indicating success or failure.</returns>
public static async Task<Result> MarkDirtyAsync(Guid guid)
public static Result MarkDirtyAsync(Guid guid, CancellationToken token = default)
{
return await MarkAssetDirtyAsync(guid, true);
MarkDirty(guid);
return Result.Success();
}
/// <summary>
/// Import all dirty assets.
/// </summary>
/// <returns>Result indicating success or failure.</returns>
public static async Task<Result> ImportDirtyAssetsAsync()
public static async Task<Result> ImportDirtyAssetsAsync(CancellationToken token = default)
{
var dirtyAssets = await GetDirtyAssetsAsync();
var dirtyGuids = GetDirtyAssets();
foreach (var (guid, path) in dirtyAssets)
foreach (var guid in dirtyGuids)
{
var fullPathResult = GetFullPath(path);
var pathResult = GuidToPath(guid);
if (pathResult.IsFailure)
{
continue;
}
var fullPathResult = GetFullPath(pathResult.Value);
if (fullPathResult.IsFailure)
{
continue;
}
var result = await ImportAssetAsync(fullPathResult.Value);
var result = await ImportAssetAsync(fullPathResult.Value, token);
if (result.IsSuccess)
{
await MarkAssetDirtyAsync(guid, false);
ClearDirty(guid);
}
}

56
Ghost.Editor.Core/AssetHandle/AssetDatabase.Importer.cs
View File

@@ -12,7 +12,7 @@ public static partial class AssetDatabase
/// </summary>
/// <param name="assetPath">Full path to the asset file.</param>
/// <returns>Result indicating success or failure.</returns>
private static async Task<Result> ImportAssetAsync(string assetPath)
private static async ValueTask<Result> ImportAssetAsync(string assetPath, CancellationToken token = default)
{
var extension = Path.GetExtension(assetPath);
@@ -35,13 +35,15 @@ public static partial class AssetDatabase
}
// Read metadata
var metaResult = await ReadMetaFileAsync(assetPath);
var metaResult = await ReadMetaFileAsync(assetPath, token);
if (metaResult.IsFailure)
{
return Result.Failure($"Failed to read asset metadata: {metaResult.Message}");
}
// Find and invoke the ImportAsync method
// TODO: Avoid reflection.
// Find and invoke the ImportAsync method. Support importers that accept (string, AssetMeta)
// or (string, AssetMeta, CancellationToken).
var importMethod = importerType.GetMethod("ImportAsync", BindingFlags.Public | BindingFlags.Instance);
if (importMethod == null)
{
@@ -50,8 +52,23 @@ public static partial class AssetDatabase
try
{
var task = importMethod.Invoke(importerInstance, new object[] { assetPath, metaResult.Value }) as Task<Result>;
if (task == null)
var parameters = importMethod.GetParameters();
object? invokeResult;
if (parameters.Length == 2)
{
invokeResult = importMethod.Invoke(importerInstance, new object[] { assetPath, metaResult.Value });
}
else if (parameters.Length == 3 && parameters[2].ParameterType == typeof(CancellationToken))
{
invokeResult = importMethod.Invoke(importerInstance, new object[] { assetPath, metaResult.Value, token });
}
else
{
return Result.Failure($"Unsupported ImportAsync signature on importer {importerType.Name}");
}
if (invokeResult is not Task<Result> task)
{
return Result.Failure("Importer did not return a valid Task<Result>");
}
@@ -93,7 +110,7 @@ public static partial class AssetDatabase
/// <param name="assetPath">Full path where the asset should be saved.</param>
/// <param name="assetData">In-memory asset data to export.</param>
/// <returns>Result with the GUID of the exported asset.</returns>
public static async Task<Result<Guid>> ExportAssetAsync<T>(string assetPath, T assetData) where T : class
public static async ValueTask<Result<Guid>> ExportAssetAsync<T>(string assetPath, T assetData, CancellationToken token = default) where T : class
{
var extension = Path.GetExtension(assetPath);
@@ -124,16 +141,31 @@ public static partial class AssetDatabase
try
{
// Generate metadata for the new asset
await GenerateMetaFileAsync(assetPath);
await GenerateMetaFileAsync(assetPath, token);
var metaResult = await ReadMetaFileAsync(assetPath);
var metaResult = await ReadMetaFileAsync(assetPath, token);
if (metaResult.IsFailure)
{
return Result<Guid>.Failure($"Failed to generate metadata: {metaResult.Message}");
}
var task = exportMethod.Invoke(importerInstance, new object[] { assetPath, assetData, metaResult.Value }) as Task<Result>;
if (task == null)
var parameters = exportMethod.GetParameters();
object? invokeResult;
if (parameters.Length == 3)
{
invokeResult = exportMethod.Invoke(importerInstance, new object[] { assetPath, assetData, metaResult.Value });
}
else if (parameters.Length == 4 && parameters[3].ParameterType == typeof(CancellationToken))
{
invokeResult = exportMethod.Invoke(importerInstance, new object[] { assetPath, assetData, metaResult.Value, token });
}
else
{
return Result<Guid>.Failure($"Unsupported ExportAsync signature on importer {importerType.Name}");
}
if (invokeResult is not Task<Result> task)
{
return Result<Guid>.Failure("Exporter did not return a valid Task<Result>");
}
@@ -145,8 +177,8 @@ public static partial class AssetDatabase
}
// Calculate file hash and update database
var fileHash = await CalculateFileHashAsync(assetPath);
await UpsertAssetAsync(assetPath, metaResult.Value, fileHash);
var fileHash = await CalculateFileHashAsync(assetPath, token);
await UpsertAssetAsync(assetPath, metaResult.Value, fileHash, null, token);
return metaResult.Value.Guid;
}

20
Ghost.Editor.Core/AssetHandle/AssetDatabase.Lookup.cs
View File

@@ -119,7 +119,7 @@ public static partial class AssetDatabase
return Result<List<string>>.Failure(fullPathResult.Message);
}
var metaResult = await ReadMetaFileAsync(fullPathResult.Value);
var metaResult = await ReadMetaFileAsync(fullPathResult.Value, token);
if (metaResult.IsFailure)
{
return Result<List<string>>.Failure(metaResult.Message);
@@ -134,7 +134,7 @@ public static partial class AssetDatabase
/// <param name="guid">GUID of the asset.</param>
/// <param name="tags">New tags for the asset.</param>
/// <returns>Result indicating success or failure.</returns>
public static async ValueTask<Result> SetAssetTagsAsync(Guid guid, List<string> tags)
public static async ValueTask<Result> SetAssetTagsAsync(Guid guid, List<string> tags, CancellationToken token = default)
{
var pathResult = GuidToPath(guid);
if (pathResult.IsFailure)
@@ -148,7 +148,7 @@ public static partial class AssetDatabase
return Result.Failure(fullPathResult.Message);
}
var metaResult = await ReadMetaFileAsync(fullPathResult.Value);
var metaResult = await ReadMetaFileAsync(fullPathResult.Value, token);
if (metaResult.IsFailure)
{
return Result.Failure(metaResult.Message);
@@ -157,15 +157,15 @@ public static partial class AssetDatabase
metaResult.Value.Tags = tags;
// Write updated metadata to .gmeta file
var writeResult = await WriteMetaFileAsync(fullPathResult.Value + Utilities.FileExtensions.META_FILE_EXTENSION, metaResult.Value);
var writeResult = await WriteMetaFileAsync(fullPathResult.Value + Utilities.FileExtensions.META_FILE_EXTENSION, metaResult.Value, token);
if (writeResult.IsFailure)
{
return writeResult;
}
// Update database with new tags
var fileHash = await CalculateFileHashAsync(fullPathResult.Value);
return await UpsertAssetAsync(fullPathResult.Value, metaResult.Value, fileHash);
var fileHash = await CalculateFileHashAsync(fullPathResult.Value, token);
return await UpsertAssetAsync(fullPathResult.Value, metaResult.Value, fileHash, null, token);
}
/// <summary>
@@ -174,9 +174,9 @@ public static partial class AssetDatabase
/// </summary>
/// <param name="namePattern">Search pattern (e.g., "*.txt", "player?", "test*").</param>
/// <returns>List of matching asset GUIDs.</returns>
public static async Task<List<Guid>> FindAssetsByNameAsync(string namePattern)
public static async Task<List<Guid>> FindAssetsByNameAsync(string namePattern, CancellationToken token = default)
{
return await GetAssetsByNameAsync(namePattern);
return await GetAssetsByNameAsync(namePattern, token);
}
/// <summary>
@@ -184,9 +184,9 @@ public static partial class AssetDatabase
/// </summary>
/// <param name="tag">Tag to search for.</param>
/// <returns>List of asset GUIDs with the specified tag.</returns>
public static async Task<List<Guid>> FindAssetsByTagAsync(string tag)
public static async Task<List<Guid>> FindAssetsByTagAsync(string tag, CancellationToken token = default)
{
return await GetAssetsByTagAsync(tag);
return await GetAssetsByTagAsync(tag, token);
}
/// <summary>

143
Ghost.Editor.Core/AssetHandle/AssetDatabase.Meta.cs
View File

@@ -70,12 +70,12 @@ public static partial class AssetDatabase
/// <summary>
/// Calculate SHA256 hash of a file for change detection.
/// </summary>
private static async Task<string> CalculateFileHashAsync(string filePath)
private static async Task<string> CalculateFileHashAsync(string filePath, CancellationToken token = default)
{
try
{
await using var stream = File.OpenRead(filePath);
var hash = await SHA256.HashDataAsync(stream);
var hash = await SHA256.HashDataAsync(stream, token);
return Convert.ToHexString(hash);
}
catch
@@ -84,12 +84,12 @@ public static partial class AssetDatabase
}
}
private static async Task<Result> WriteMetaFileAsync(string metaFilePath, AssetMeta metaData)
private static async Task<Result> WriteMetaFileAsync(string metaFilePath, AssetMeta metaData, CancellationToken token = default)
{
try
{
await using var fileStream = File.Create(metaFilePath);
await JsonSerializer.SerializeAsync(fileStream, metaData, s_defaultJsonOptions);
await JsonSerializer.SerializeAsync(fileStream, metaData, s_defaultJsonOptions, token);
return Result.Success();
}
catch (Exception ex)
@@ -143,7 +143,7 @@ public static partial class AssetDatabase
if (File.Exists(metaFileResult.Value))
{
var existingMetaResult = await ReadMetaFileAsync(assetPath);
var existingMetaResult = await ReadMetaFileAsync(assetPath, token);
if (existingMetaResult.IsSuccess)
{
var existingMeta = existingMetaResult.Value;
@@ -154,7 +154,7 @@ public static partial class AssetDatabase
{
// GUID conflict - regenerate
existingMeta.Guid = Guid.NewGuid();
r = await WriteMetaFileAsync(metaFileResult.Value, existingMeta);
r = await WriteMetaFileAsync(metaFileResult.Value, existingMeta, token);
if (r.IsFailure)
{
return r;
@@ -163,14 +163,14 @@ public static partial class AssetDatabase
}
// Calculate file hash and update database
var fileHash = await CalculateFileHashAsync(assetPath);
await UpsertAssetAsync(assetPath, existingMeta, fileHash);
var fileHash = await CalculateFileHashAsync(assetPath, token);
await UpsertAssetAsync(assetPath, existingMeta, fileHash, null, token);
return Result.Success();
}
}
// Calculate initial file hash
var fileHash2 = await CalculateFileHashAsync(assetPath);
var fileHash2 = await CalculateFileHashAsync(assetPath, token);
var defaultSettings = GetDefaultSettingsForAsset(assetPath);
var metaData = new AssetMeta
@@ -183,19 +183,19 @@ public static partial class AssetDatabase
metaData.SetImporterSettings(defaultSettings.GetType().Name, defaultSettings);
}
r = await WriteMetaFileAsync(metaFileResult.Value, metaData);
r = await WriteMetaFileAsync(metaFileResult.Value, metaData, token);
if (r.IsFailure)
{
return r;
}
// Add to database
await UpsertAssetAsync(assetPath, metaData, fileHash2);
await UpsertAssetAsync(assetPath, metaData, fileHash2, null, token);
return r;
}
private static async void OnAssetCreated(object sender, FileSystemEventArgs e)
private static void OnAssetCreated(object sender, FileSystemEventArgs e)
{
// Skip meta files
if (Path.GetExtension(e.FullPath).Equals(FileExtensions.META_FILE_EXTENSION, StringComparison.OrdinalIgnoreCase))
@@ -203,16 +203,10 @@ public static partial class AssetDatabase
return;
}
// Debounce to prevent duplicate events
if (!ShouldProcessFileOperation(e.FullPath))
{
return;
}
await GenerateMetaFileAsync(e.FullPath);
PostCommand(new AssetCommand(AssetCommandType.FileCreated, e.FullPath, Timestamp: DateTime.UtcNow));
}
private static async void OnAssetDeleted(object sender, FileSystemEventArgs e)
private static void OnAssetDeleted(object sender, FileSystemEventArgs e)
{
// Skip meta files
if (Path.GetExtension(e.FullPath).Equals(FileExtensions.META_FILE_EXTENSION, StringComparison.OrdinalIgnoreCase))
@@ -220,39 +214,10 @@ public static partial class AssetDatabase
return;
}
// Debounce to prevent duplicate events
if (!ShouldProcessFileOperation(e.FullPath))
{
return;
}
var metaFileResult = GetMetaFilePath(e.FullPath);
if (metaFileResult.IsSuccess && File.Exists(metaFileResult.Value))
{
try
{
var metaResult = await ReadMetaFileAsync(e.FullPath);
if (metaResult.IsSuccess)
{
var meta = metaResult.Value;
// Remove from database
await RemoveAssetFromDatabaseAsync(meta.Guid);
// Mark dependent assets as dirty
await MarkDependentAssetsDirtyAsync(meta.Guid);
}
File.Delete(metaFileResult.Value);
}
catch (Exception ex)
{
Console.WriteLine($"Error deleting asset metadata: {ex.Message}");
}
}
PostCommand(new AssetCommand(AssetCommandType.FileDeleted, e.FullPath, Timestamp: DateTime.UtcNow));
}
private static async void OnAssetRenamed(object sender, RenamedEventArgs e)
private static void OnAssetRenamed(object sender, RenamedEventArgs e)
{
// Skip meta files
if (Path.GetExtension(e.FullPath).Equals(FileExtensions.META_FILE_EXTENSION, StringComparison.OrdinalIgnoreCase))
@@ -260,54 +225,10 @@ public static partial class AssetDatabase
return;
}
// Debounce to prevent duplicate events
if (!ShouldProcessFileOperation(e.FullPath))
{
return;
}
var oldMetaPath = e.OldFullPath + FileExtensions.META_FILE_EXTENSION;
var newMetaPath = e.FullPath + FileExtensions.META_FILE_EXTENSION;
if (File.Exists(newMetaPath))
{
// Validate and update
await GenerateMetaFileAsync(e.FullPath);
}
else if (File.Exists(oldMetaPath))
{
// Move meta file
File.Move(oldMetaPath, newMetaPath);
// Update database with new path and recalculated hash
var metaResult = await ReadMetaFileAsync(e.FullPath);
if (metaResult.IsSuccess)
{
var fileHash = await CalculateFileHashAsync(e.FullPath);
await UpsertAssetAsync(e.FullPath, metaResult.Value, fileHash);
}
}
else
{
// Generate new meta file
await GenerateMetaFileAsync(e.FullPath);
}
// Delete old meta if it still exists
if (File.Exists(oldMetaPath) && oldMetaPath != newMetaPath)
{
try
{
File.Delete(oldMetaPath);
}
catch
{
// Ignore
}
}
PostCommand(new AssetCommand(AssetCommandType.FileRenamed, e.FullPath, e.OldFullPath, DateTime.UtcNow));
}
private static async void OnAssetChanged(object sender, FileSystemEventArgs e)
private static void OnAssetChanged(object sender, FileSystemEventArgs e)
{
// Skip meta files
if (Path.GetExtension(e.FullPath).Equals(FileExtensions.META_FILE_EXTENSION, StringComparison.OrdinalIgnoreCase))
@@ -315,29 +236,7 @@ public static partial class AssetDatabase
return;
}
// Debounce to prevent duplicate events
if (!ShouldProcessFileOperation(e.FullPath))
{
return;
}
// Check if file hash changed
var metaResult = await ReadMetaFileAsync(e.FullPath);
if (metaResult.IsFailure)
{
return;
}
// Calculate new hash and compare against database
var newHash = await CalculateFileHashAsync(e.FullPath);
var oldHash = await GetFileHashAsync(metaResult.Value.Guid);
if (oldHash != newHash)
{
// File changed - update database and mark as dirty
await UpsertAssetAsync(e.FullPath, metaResult.Value, newHash);
await MarkAssetDirtyAsync(metaResult.Value.Guid, true);
}
PostCommand(new AssetCommand(AssetCommandType.FileModified, e.FullPath, Timestamp: DateTime.UtcNow));
}
/// <summary>
@@ -350,10 +249,10 @@ public static partial class AssetDatabase
foreach (var kvp in allAssets)
{
var dependencies = await GetDependenciesAsync(kvp.Key);
var dependencies = await GetDependenciesAsync(kvp.Key, CancellationToken.None);
if (dependencies.Contains(assetGuid))
{
await MarkAssetDirtyAsync(kvp.Key, true);
MarkDirty(kvp.Key);
}
}
}

113
Ghost.Editor.Core/AssetHandle/AssetDatabase.SQLite.cs
View File

@@ -12,7 +12,7 @@ public static partial class AssetDatabase
/// <summary>
/// Initialize the SQLite database for asset caching.
/// </summary>
private static async Task InitializeDatabaseAsync()
private static async Task InitializeDatabaseAsync(CancellationToken token = default)
{
if (AssetsDirectory == null)
{
@@ -34,7 +34,7 @@ public static partial class AssetDatabase
}.ToString();
s_dbConnection = new SqliteConnection(connectionString);
await s_dbConnection.OpenAsync();
await s_dbConnection.OpenAsync(token);
// Create tables
await using var cmd = s_dbConnection.CreateCommand();
@@ -46,13 +46,11 @@ public static partial class AssetDatabase
Tags TEXT,
FileHash TEXT,
DependencyGuids TEXT,
IsDirty INTEGER NOT NULL DEFAULT 0,
LastModified INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_path ON Assets(Path);
CREATE INDEX IF NOT EXISTS idx_dirty ON Assets(IsDirty);
";
await cmd.ExecuteNonQueryAsync();
await cmd.ExecuteNonQueryAsync(token);
}
/// <summary>
@@ -116,7 +114,7 @@ public static partial class AssetDatabase
/// <summary>
/// Remove an asset from the database.
/// </summary>
private static async Task<Result> RemoveAssetFromDatabaseAsync(Guid guid)
private static async Task<Result> RemoveAssetFromDatabaseAsync(Guid guid, CancellationToken token = default)
{
if (s_dbConnection == null)
{
@@ -138,7 +136,7 @@ public static partial class AssetDatabase
cmd.CommandText = "DELETE FROM Assets WHERE Guid = @guid";
cmd.Parameters.AddWithValue("@guid", guid.ToString());
await cmd.ExecuteNonQueryAsync();
await cmd.ExecuteNonQueryAsync(token);
return Result.Success();
}
catch (Exception ex)
@@ -147,73 +145,12 @@ public static partial class AssetDatabase
}
}
/// <summary>
/// Mark an asset as dirty for re-importing.
/// </summary>
private static async Task<Result> MarkAssetDirtyAsync(Guid guid, bool isDirty = true)
{
if (s_dbConnection == null)
{
return Result.Failure("Database not initialized");
}
try
{
await using var cmd = s_dbConnection.CreateCommand();
cmd.CommandText = "UPDATE Assets SET IsDirty = @dirty WHERE Guid = @guid";
cmd.Parameters.AddWithValue("@dirty", isDirty ? 1 : 0);
cmd.Parameters.AddWithValue("@guid", guid.ToString());
await cmd.ExecuteNonQueryAsync();
return Result.Success();
}
catch (Exception ex)
{
return Result.Failure($"Failed to mark asset dirty: {ex.Message}");
}
}
/// <summary>
/// Get all dirty assets that need re-importing.
/// </summary>
private static async Task<List<(Guid guid, string path)>> GetDirtyAssetsAsync()
{
var result = new List<(Guid guid, string path)>();
if (s_dbConnection == null)
{
return result;
}
try
{
await using var cmd = s_dbConnection.CreateCommand();
cmd.CommandText = "SELECT Guid, Path FROM Assets WHERE IsDirty = 1";
await using var reader = await cmd.ExecuteReaderAsync();
while (await reader.ReadAsync())
{
var guidStr = reader.GetString(0);
var path = reader.GetString(1);
if (Guid.TryParse(guidStr, out var guid))
{
result.Add((guid, path));
}
}
}
catch
{
// Silently fail - we'll return empty list
}
return result;
}
/// <summary>
/// Load all assets from the database into memory cache.
/// </summary>
private static async Task LoadAssetCacheFromDatabaseAsync()
private static async Task LoadAssetCacheFromDatabaseAsync(CancellationToken token = default)
{
if (s_dbConnection == null)
{
@@ -225,8 +162,8 @@ public static partial class AssetDatabase
await using var cmd = s_dbConnection.CreateCommand();
cmd.CommandText = "SELECT Guid, Path FROM Assets";
await using var reader = await cmd.ExecuteReaderAsync();
while (await reader.ReadAsync())
await using var reader = await cmd.ExecuteReaderAsync(token);
while (await reader.ReadAsync(token))
{
var guidStr = reader.GetString(0);
var path = reader.GetString(1);
@@ -250,7 +187,7 @@ public static partial class AssetDatabase
/// <summary>
/// Get assets by tag.
/// </summary>
private static async Task<List<Guid>> GetAssetsByTagAsync(string tag)
private static async Task<List<Guid>> GetAssetsByTagAsync(string tag, CancellationToken token = default)
{
var result = new List<Guid>();
@@ -264,8 +201,8 @@ public static partial class AssetDatabase
await using var cmd = s_dbConnection.CreateCommand();
cmd.CommandText = "SELECT Guid, Tags FROM Assets";
await using var reader = await cmd.ExecuteReaderAsync();
while (await reader.ReadAsync())
await using var reader = await cmd.ExecuteReaderAsync(token);
while (await reader.ReadAsync(token))
{
var guidStr = reader.GetString(0);
var tagsJson = reader.GetString(1);
@@ -291,7 +228,7 @@ public static partial class AssetDatabase
/// <summary>
/// Get the file hash for an asset from the database.
/// </summary>
private static async Task<string?> GetFileHashAsync(Guid guid)
private static async Task<string?> GetFileHashAsync(Guid guid, CancellationToken token = default)
{
if (s_dbConnection == null)
{
@@ -304,7 +241,7 @@ public static partial class AssetDatabase
cmd.CommandText = "SELECT FileHash FROM Assets WHERE Guid = @guid";
cmd.Parameters.AddWithValue("@guid", guid.ToString());
var result = await cmd.ExecuteScalarAsync();
var result = await cmd.ExecuteScalarAsync(token);
return result?.ToString();
}
catch
@@ -316,7 +253,7 @@ public static partial class AssetDatabase
/// <summary>
/// Get the dependencies for an asset from the database.
/// </summary>
private static async Task<List<Guid>> GetDependenciesAsync(Guid guid)
private static async Task<List<Guid>> GetDependenciesAsync(Guid guid, CancellationToken token = default)
{
if (s_dbConnection == null)
{
@@ -329,7 +266,7 @@ public static partial class AssetDatabase
cmd.CommandText = "SELECT DependencyGuids FROM Assets WHERE Guid = @guid";
cmd.Parameters.AddWithValue("@guid", guid.ToString());
var result = await cmd.ExecuteScalarAsync();
var result = await cmd.ExecuteScalarAsync(token);
if (result != null)
{
var json = result.ToString();
@@ -348,7 +285,7 @@ public static partial class AssetDatabase
/// Find assets by name pattern using database query with wildcards.
/// </summary>
/// <param name="namePattern">Pattern supporting * (any chars) and ? (single char).</param>
private static async Task<List<Guid>> GetAssetsByNameAsync(string namePattern)
private static async Task<List<Guid>> GetAssetsByNameAsync(string namePattern, CancellationToken token = default)
{
var results = new List<Guid>();
@@ -373,8 +310,8 @@ public static partial class AssetDatabase
";
cmd.Parameters.AddWithValue("@pattern", sqlPattern);
await using var reader = await cmd.ExecuteReaderAsync();
while (await reader.ReadAsync())
await using var reader = await cmd.ExecuteReaderAsync(token);
while (await reader.ReadAsync(token))
{
var guidStr = reader.GetString(0);
var path = reader.GetString(1);
@@ -407,7 +344,7 @@ public static partial class AssetDatabase
/// <summary>
/// Remove orphaned entries from database (assets that no longer exist on disk).
/// </summary>
private static async Task RemoveOrphanedEntriesAsync()
private static async Task RemoveOrphanedEntriesAsync(CancellationToken token = default)
{
if (s_dbConnection == null || AssetsDirectory == null)
{
@@ -421,8 +358,8 @@ public static partial class AssetDatabase
await using var cmd = s_dbConnection.CreateCommand();
cmd.CommandText = "SELECT Guid, Path FROM Assets";
await using var reader = await cmd.ExecuteReaderAsync();
while (await reader.ReadAsync())
await using var reader = await cmd.ExecuteReaderAsync(token);
while (await reader.ReadAsync(token))
{
var guidStr = reader.GetString(0);
var path = reader.GetString(1);
@@ -439,10 +376,10 @@ public static partial class AssetDatabase
}
// Remove orphaned entries
foreach (var guid in orphanedGuids)
{
await RemoveAssetFromDatabaseAsync(guid);
}
foreach (var guid in orphanedGuids)
{
await RemoveAssetFromDatabaseAsync(guid, token);
}
}
catch
{

449
Ghost.Editor.Core/AssetHandle/AssetDatabase.cs
View File

@@ -1,9 +1,33 @@
using Ghost.Core;
using Ghost.Data.Services;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading.Channels;
namespace Ghost.Editor.Core.AssetHandle;
/// <summary>
/// Command types for asset database operations.
/// </summary>
internal enum AssetCommandType
{
FileCreated,
FileModified,
FileDeleted,
FileRenamed,
ManualRefresh
}
/// <summary>
/// Represents a command to process an asset operation.
/// </summary>
internal readonly record struct AssetCommand(
AssetCommandType Type,
string Path,
string? OldPath = null,
DateTime Timestamp = default
);
/// <summary>
/// Centralized asset database that manages all assets in the project.
/// Handles asset registration, lookup, importing, and dependency management.
@@ -15,16 +39,24 @@ public static partial class AssetDatabase
private static readonly Lock s_dbLock = new();
private static readonly Dictionary<Guid, string> s_assetPathLookup = new();
private static readonly Dictionary<string, Guid> s_pathAssetLookup = new();
// Debouncing for file system watcher to prevent duplicate events
private static readonly Dictionary<string, DateTime> s_pendingFileOperations = new();
private static readonly Lock s_pendingOperationsLock = new();
private static readonly TimeSpan s_debounceDelay = TimeSpan.FromMilliseconds(100);
// In-memory dirty asset tracking (for runtime modifications only)
private static readonly HashSet<Guid> s_dirtyAssets = new();
// Command buffer pattern - Channel for file system event commands
private static Channel<AssetCommand>? s_commandChannel;
private static Timer? s_commandProcessorTimer;
private static readonly HashSet<string> s_pendingCommandPaths = new(); // Track paths with pending commands
private static readonly Lock s_commandLock = new();
private static bool s_autoRefreshEnabled = true;
private static readonly Queue<AssetCommand> s_waitingCommands = new(); // Commands waiting for manual refresh
// Initialization guard
private static readonly Lock s_initializationLock = new();
private static bool s_initialized = false;
private static readonly TimeSpan s_debounceDelay = TimeSpan.FromMilliseconds(100);
private static readonly JsonSerializerOptions s_defaultJsonOptions = new()
{
WriteIndented = true,
@@ -45,7 +77,7 @@ public static partial class AssetDatabase
/// Initialize the asset database.
/// Must be called after project is loaded.
/// </summary>
internal static async void Initialize()
internal static async void Initialize(CancellationToken token = default)
{
lock (s_initializationLock)
{
@@ -63,11 +95,21 @@ public static partial class AssetDatabase
AssetsDirectory = new DirectoryInfo(Path.Combine(Path.GetDirectoryName(ProjectService.CurrentProject.Path)!, ProjectService.ASSETS_FOLDER));
// Initialize command channel (unbounded for simplicity)
s_commandChannel = Channel.CreateUnbounded<AssetCommand>(new UnboundedChannelOptions
{
SingleReader = false, // Timer callback reads
SingleWriter = false // Multiple FS events can write
});
// Initialize command processor timer (starts disabled, triggered by events)
s_commandProcessorTimer = new Timer(ProcessPendingCommands, null, Timeout.Infinite, Timeout.Infinite);
// Initialize database
await InitializeDatabaseAsync();
await InitializeDatabaseAsync(token);
// Load asset cache from database
await LoadAssetCacheFromDatabaseAsync();
await LoadAssetCacheFromDatabaseAsync(token);
// Initialize file system watcher
s_watcher = new FileSystemWatcher
@@ -82,26 +124,25 @@ public static partial class AssetDatabase
InitializeMetaData();
// Validate and fix database on startup
await ValidateAndFixDatabaseAsync();
await ValidateAndFixDatabaseAsync(token);
}
/// <summary>
/// Validate the asset database and fix any inconsistencies.
/// Checks for missing/corrupted assets and regenerates metadata as needed.
/// </summary>
private static async Task<Ghost.Core.Result> ValidateAndFixDatabaseAsync()
private static async Task<Result> ValidateAndFixDatabaseAsync(CancellationToken token = default)
{
if (AssetsDirectory == null)
{
return Ghost.Core.Result.Failure("AssetsDirectory not initialized");
return Result.Failure("AssetsDirectory not initialized");
}
try
{
// Scan all files in assets directory
var allFiles = Directory.GetFiles(AssetsDirectory.FullName, "*.*", SearchOption.AllDirectories)
.Where(f => !f.EndsWith(Utilities.FileExtensions.META_FILE_EXTENSION, StringComparison.OrdinalIgnoreCase))
.ToList();
.Where(f => !f.EndsWith(Utilities.FileExtensions.META_FILE_EXTENSION, StringComparison.OrdinalIgnoreCase));
// Ensure all files have metadata
foreach (var file in allFiles)
@@ -109,91 +150,372 @@ public static partial class AssetDatabase
var metaPath = file + Utilities.FileExtensions.META_FILE_EXTENSION;
if (!File.Exists(metaPath))
{
await GenerateMetaFileAsync(file);
await GenerateMetaFileAsync(file, token);
}
else
{
// Validate and update database
var metaResult = await ReadMetaFileAsync(file);
var metaResult = await ReadMetaFileAsync(file, token);
if (metaResult.IsSuccess)
{
var fileHash = await CalculateFileHashAsync(file);
await UpsertAssetAsync(file, metaResult.Value, fileHash);
var fileHash = await CalculateFileHashAsync(file, token);
await UpsertAssetAsync(file, metaResult.Value, fileHash, null, token);
}
else
{
// Corrupted meta file - regenerate
await GenerateMetaFileAsync(file);
await GenerateMetaFileAsync(file, token);
}
}
}
// Remove orphaned entries from database (files that no longer exist)
await RemoveOrphanedEntriesAsync();
await RemoveOrphanedEntriesAsync(token);
return Ghost.Core.Result.Success();
return Result.Success();
}
catch (Exception ex)
{
return Ghost.Core.Result.Failure($"Failed to validate database: {ex.Message}");
return Result.Failure($"Failed to validate database: {ex.Message}");
}
}
/// <summary>
/// Refresh the asset database manually.
/// Scans the project directory for changes.
/// Scans the project directory for changes and processes any queued file system events.
/// </summary>
public static async Task<Ghost.Core.Result> RefreshAsync()
public static async Task<Result> RefreshAsync(CancellationToken token = default)
{
return await ValidateAndFixDatabaseAsync();
// Flush waiting commands to channel
lock (s_commandLock)
{
while (s_waitingCommands.TryDequeue(out var cmd))
{
s_commandChannel?.Writer.TryWrite(cmd);
}
}
// Post manual refresh command
s_commandChannel?.Writer.TryWrite(new AssetCommand(AssetCommandType.ManualRefresh, string.Empty));
// Trigger timer immediately
s_commandProcessorTimer?.Change(TimeSpan.Zero, Timeout.InfiniteTimeSpan);
// Wait a bit for processing to complete (this is best-effort)
await Task.Delay(200, token);
return Result.Success();
}
/// <summary>
/// Check if a file operation should be processed or debounced.
/// Returns true if the operation should proceed.
/// Mark an asset as dirty (modified in memory but not yet saved).
/// This state is NOT persisted and will be lost on application restart.
/// </summary>
private static bool ShouldProcessFileOperation(string filePath)
public static void MarkDirty(Guid assetGuid)
{
lock (s_pendingOperationsLock)
lock (s_dbLock)
{
var now = DateTime.UtcNow;
// Clean up old entries
var toRemove = s_pendingFileOperations
.Where(kvp => now - kvp.Value > s_debounceDelay * 2)
.Select(kvp => kvp.Key)
.ToList();
foreach (var key in toRemove)
{
s_pendingFileOperations.Remove(key);
}
// Check if this operation was recently processed
if (s_pendingFileOperations.TryGetValue(filePath, out var lastTime))
{
if (now - lastTime < s_debounceDelay)
{
// Too soon, skip this event
return false;
}
}
// Update timestamp and allow processing
s_pendingFileOperations[filePath] = now;
return true;
s_dirtyAssets.Add(assetGuid);
}
}
/// <summary>
/// Register a file operation to prevent the file watcher from processing it.
/// Used by file operations (move, copy, etc.) to prevent duplicate processing.
/// Check if an asset is marked as dirty.
/// </summary>
private static void RegisterFileOperation(string filePath)
public static bool IsDirty(Guid assetGuid)
{
lock (s_pendingOperationsLock)
lock (s_dbLock)
{
s_pendingFileOperations[filePath] = DateTime.UtcNow;
return s_dirtyAssets.Contains(assetGuid);
}
}
/// <summary>
/// Get all dirty assets.
/// </summary>
public static Guid[] GetDirtyAssets()
{
lock (s_dbLock)
{
return s_dirtyAssets.ToArray();
}
}
/// <summary>
/// Clear dirty flag for an asset (typically after saving).
/// </summary>
public static void ClearDirty(Guid assetGuid)
{
lock (s_dbLock)
{
s_dirtyAssets.Remove(assetGuid);
}
}
/// <summary>
/// Clear all dirty flags.
/// </summary>
public static void ClearAllDirty()
{
lock (s_dbLock)
{
s_dirtyAssets.Clear();
}
}
/// <summary>
/// Enable or disable automatic asset database refresh.
/// When disabled, file system events are queued and processed only when RefreshAsync() is called.
/// </summary>
public static void SetAutoRefresh(bool enabled)
{
s_autoRefreshEnabled = enabled;
}
/// <summary>
/// Process all pending commands immediately (synchronous, for testing).
/// </summary>
internal static void FlushPendingCommands()
{
// Stop timer temporarily
s_commandProcessorTimer?.Change(Timeout.Infinite, Timeout.Infinite);
// Give a tiny bit of time for any in-flight file watcher events to post to channel
Thread.Sleep(50);
// Process all commands now
ProcessPendingCommands(null);
}
/// <summary>
/// Post a command to the command channel for processing.
/// </summary>
private static void PostCommand(AssetCommand command)
{
if (s_commandChannel == null)
{
return;
}
if (s_autoRefreshEnabled)
{
// Add to pending paths for temp file tracking
lock (s_commandLock)
{
s_pendingCommandPaths.Add(command.Path);
if (command.OldPath != null)
{
s_pendingCommandPaths.Add(command.OldPath);
}
}
// Write to channel (lock-free, async-safe)
s_commandChannel.Writer.TryWrite(command);
// Start or reset timer (safe even if not initialized yet)
s_commandProcessorTimer?.Change(s_debounceDelay, Timeout.InfiniteTimeSpan);
}
else
{
// Queue for manual refresh
lock (s_commandLock)
{
s_waitingCommands.Enqueue(command);
}
}
}
/// <summary>
/// Timer callback to process pending commands.
/// </summary>
private static void ProcessPendingCommands(object? state)
{
try
{
// Collect all pending commands
var commands = new List<AssetCommand>();
while (s_commandChannel?.Reader.TryRead(out var cmd) == true)
{
commands.Add(cmd);
}
// Group commands by path (last command wins)
var commandsByPath = new Dictionary<string, AssetCommand>();
foreach (var cmd in commands)
{
commandsByPath[cmd.Path] = cmd;
}
// Filter out temp files (files that were created then deleted)
lock (s_commandLock)
{
var pathsToProcess = commandsByPath.Keys.ToList();
foreach (var path in pathsToProcess)
{
// If file was created/modified but doesn't exist anymore, skip
if (!File.Exists(path) && commandsByPath[path].Type != AssetCommandType.FileDeleted)
{
commandsByPath.Remove(path);
}
}
// Clear pending paths
s_pendingCommandPaths.Clear();
}
// Execute commands
foreach (var cmd in commandsByPath.Values)
{
ExecuteCommandAsync(cmd).GetAwaiter().GetResult();
}
}
catch (Exception ex)
{
Console.WriteLine($"Error processing commands: {ex.Message}");
}
}
/// <summary>
/// Execute a single asset command.
/// </summary>
private static async Task ExecuteCommandAsync(AssetCommand command)
{
switch (command.Type)
{
case AssetCommandType.FileCreated:
await HandleFileCreatedAsync(command.Path);
break;
case AssetCommandType.FileModified:
await HandleFileModifiedAsync(command.Path);
break;
case AssetCommandType.FileDeleted:
await HandleFileDeletedAsync(command.Path);
break;
case AssetCommandType.FileRenamed:
if (command.OldPath != null)
{
await HandleFileRenamedAsync(command.OldPath, command.Path);
}
break;
case AssetCommandType.ManualRefresh:
await ValidateAndFixDatabaseAsync(CancellationToken.None);
break;
}
}
/// <summary>
/// Handle file created event.
/// </summary>
private static async Task HandleFileCreatedAsync(string path)
{
await GenerateMetaFileAsync(path, CancellationToken.None);
}
/// <summary>
/// Handle file modified event.
/// </summary>
private static async Task HandleFileModifiedAsync(string path)
{
// Check if file hash changed
var metaResult = await ReadMetaFileAsync(path, CancellationToken.None);
if (metaResult.IsFailure)
{
// No .gmeta file - treat this as a new file creation
await HandleFileCreatedAsync(path);
return;
}
// Calculate new hash and compare against database
var newHash = await CalculateFileHashAsync(path, CancellationToken.None);
var oldHash = await GetFileHashAsync(metaResult.Value.Guid, CancellationToken.None);
if (oldHash != newHash)
{
// File changed - update database and mark as dirty
await UpsertAssetAsync(path, metaResult.Value, newHash, null, CancellationToken.None);
MarkDirty(metaResult.Value.Guid);
}
}
/// <summary>
/// Handle file deleted event.
/// </summary>
private static async Task HandleFileDeletedAsync(string path)
{
var metaFileResult = GetMetaFilePath(path);
if (metaFileResult.IsSuccess && File.Exists(metaFileResult.Value))
{
try
{
var metaResult = await ReadMetaFileAsync(path, CancellationToken.None);
if (metaResult.IsSuccess)
{
var meta = metaResult.Value;
// Remove from database
await RemoveAssetFromDatabaseAsync(meta.Guid, CancellationToken.None);
// Mark dependent assets as dirty
await MarkDependentAssetsDirtyAsync(meta.Guid);
}
File.Delete(metaFileResult.Value);
}
catch (Exception ex)
{
Console.WriteLine($"Error deleting asset metadata: {ex.Message}");
}
}
}
/// <summary>
/// Handle file renamed event.
/// </summary>
private static async Task HandleFileRenamedAsync(string oldPath, string newPath)
{
var oldMetaPath = oldPath + Utilities.FileExtensions.META_FILE_EXTENSION;
var newMetaPath = newPath + Utilities.FileExtensions.META_FILE_EXTENSION;
if (File.Exists(newMetaPath))
{
// Validate and update
await GenerateMetaFileAsync(newPath, CancellationToken.None);
}
else if (File.Exists(oldMetaPath))
{
// Move meta file
File.Move(oldMetaPath, newMetaPath);
// Update database with new path and recalculated hash
var metaResult = await ReadMetaFileAsync(newPath, CancellationToken.None);
if (metaResult.IsSuccess)
{
var fileHash = await CalculateFileHashAsync(newPath, CancellationToken.None);
await UpsertAssetAsync(newPath, metaResult.Value, fileHash, null, CancellationToken.None);
}
}
else
{
// Generate new meta file
await GenerateMetaFileAsync(newPath, CancellationToken.None);
}
// Delete old meta if it still exists
if (File.Exists(oldMetaPath) && oldMetaPath != newMetaPath)
{
try
{
File.Delete(oldMetaPath);
}
catch
{
// Ignore
}
}
}
@@ -213,15 +535,20 @@ public static partial class AssetDatabase
s_watcher?.Dispose();
s_watcher = null;
s_commandProcessorTimer?.Dispose();
s_commandProcessorTimer = null;
s_dbConnection?.Close();
s_dbConnection?.Dispose();
s_dbConnection = null;
s_assetPathLookup.Clear();
s_pathAssetLookup.Clear();
s_dirtyAssets.Clear();
s_pendingCommandPaths.Clear();
s_waitingCommands.Clear();
s_importerInstances.Clear();
s_importerTypeLookup.Clear();
s_pendingFileOperations.Clear();
s_initialized = false;
}

115
Ghost.Editor.Core/AssetHandle/AssetDatabase_Architecture.md Normal file
View File

@@ -0,0 +1,115 @@
# Asset Database Architecture
This document details the architectural design and data flow of the `AssetHandle` module in Ghost Editor.
## System Overview
The Asset Database acts as the bridge between the raw file system (Source Assets) and the runtime engine (Imported Assets). It maintains a consistent state using a dual-storage approach:
1. **File System**: The source of truth. Contains source files (e.g., `.png`, `.fbx`) and metadata files (`.gmeta`).
2. **SQLite Database**: An acceleration layer (cache) for fast lookups, dependency tracking, and searching.
## Data Flow
### 1. Asset Discovery & Registration
When the editor starts or a file changes:
1. **FileSystemWatcher** detects the change (Create/Delete/Modify/Rename).
2. **Event Handler** queues an `AssetCommand` (debounce mechanism prevents event storms).
3. **Command Processor** executes the command:
* **New File**: Generates a `.gmeta` file with a new GUID and default settings. Adds to SQLite.
* **Modified File**: Checks hash. If changed, marks asset as "Dirty" and updates SQLite.
* **Deleted File**: Removes from SQLite and marks dependents as "Dirty".
### 2. Import Pipeline
The import process converts source formats into engine-ready data.
**Flow:**
1. `AssetDatabase.ImportDirtyAssetsAsync()` or direct `ImportAssetAsync` is called.
2. System looks up the registered `AssetImporter` for the file extension.
3. `AssetImporter.ImportAsync` is invoked with the source path and metadata.
4. Importer reads source file and settings from metadata.
5. Importer processes data (e.g., compiles shaders, compresses textures).
6. Importer calls `AssetDatabase.SaveImportedAsset(guid, data)`.
7. Data is serialized to JSON (or binary) in the `Cache/ImportedAssets` directory as `{GUID}.asset`.
### 3. Loading Pipeline
When the engine requests an asset:
**Flow:**
1. `AssetDatabase.LoadAsset<T>(guid)` is called.
2. **Memory Cache Check**:
* Checks `s_assetCache` (ConcurrentDictionary).
* If found: Updates LRU timestamp and returns object.
* If not found: Proceeds to disk load.
3. **Disk Load**:
* Locates `{GUID}.asset` in `Cache/ImportedAssets`.
* Deserializes the data into the target runtime type (e.g., `TextureAsset`).
4. **Cache Update**:
* Adds new object to `s_assetCache`.
* If cache size > `MAX_CACHED_ASSETS` (1000), evicts oldest 20% based on access time.
## Key Components Diagram
```mermaid
graph TD
User[Editor / User] -->|File Ops| API[AssetDatabase API]
FS[File System] -->|Events| Watcher[FileSystemWatcher]
subgraph AssetDatabase
API --> DB[SQLite Database]
API --> Meta[Meta Handler]
API --> Loader[Asset Loader]
API --> Importer[Import System]
Watcher -->|Queue| Cmd[Command Processor]
Cmd --> Meta
Cmd --> DB
Importer -->|Read| FS
Importer -->|Write| Cache[Imported Assets Cache]
Loader -->|Read| Cache
Loader -->|Check| MemCache[Memory LRU Cache]
end
Meta -->|Read/Write| FS
DB -->|Index| FS
```
## Database Schema (SQLite)
The `AssetDatabase.db` contains a single `Assets` table:
| Column | Type | Description |
|--------|------|-------------|
| **Guid** | TEXT (PK) | The unique identifier of the asset. |
| **Path** | TEXT | Relative path from `Assets/` folder. Indexed for fast lookup. |
| **Version** | INTEGER | Importer version for migration support. |
| **Tags** | TEXT | JSON array of string tags. |
| **FileHash** | TEXT | SHA256 hash of the source file content. |
| **DependencyGuids** | TEXT | JSON array of GUIDs this asset depends on. |
| **LastModified** | INTEGER | Unix timestamp of last modification. |
## Detailed Subsystems
### Metadata System (`.gmeta`)
* **Format**: JSON.
* **Content**: GUID, Version, Tags, ImporterSettings (per importer type).
* **Strategy**: The `.gmeta` file is the *only* place the persistent GUID lives. If the database is corrupted, it can be rebuilt entirely by scanning the file system and reading `.gmeta` files.
### Threading & Safety
* **Locks**:
* `s_dbLock`: Protects in-memory dictionaries (`s_assetPathLookup`) and dirty tracking.
* `s_commandLock`: Protects the command queue for file events.
* **Async**: Heavy I/O operations (DB access, File I/O) are async.
* **Channels**: Uses `System.Threading.Channels` to decouple high-frequency file system events from database processing.
### Importer Registry
* Uses `TypeCache` and reflection to find classes with `[AssetImporter]`.
* Mappings are stored in `s_importerTypeLookup` (Extension -> Type).
* Importers are stateless (instantiated on demand or cached as singletons depending on implementation, currently cached in `s_importerInstances`).
## Future Improvements / Known Limitations
1. **Binary Formats**: Currently, imported assets are stored as JSON. For large assets (textures, models), a binary format is required for performance.
2. **Dependency Graph**: While dependencies are stored, a full graph traversal for complex invalidation (e.g., if A changes, re-import B which depends on A) is partial.
3. **Cross-Process Locking**: SQLite is file-based; concurrent access from multiple editor instances needs careful file locking mode configuration.

131
Ghost.Editor.Core/AssetHandle/AssetDatabase_Documentation.md Normal file
View File

@@ -0,0 +1,131 @@
# Asset Database Documentation
The Asset Database is a core component of the Ghost Editor responsible for managing the lifecycle, storage, import, and retrieval of project assets. It provides a unified API for interacting with assets, ensuring that metadata (GUIDs, tags, settings) stays synchronized with files on disk.
## Key Features
- **GUID-based Asset Identification**: Every asset is uniquely identified by a stable GUID, stored in a sidecar `.gmeta` file.
- **Automatic Importing**: Monitors the file system for changes and automatically imports assets using registered importers.
- **Dependency Tracking**: Tracks dependencies between assets to ensure validity and trigger re-imports when dependencies change.
- **Caching**: Implements an LRU (Least Recently Used) cache for loaded assets to optimize performance.
- **SQLite Backed**: Uses a local SQLite database for fast lookups (Path <-> GUID) and metadata queries.
- **Metadata Management**: Handles `.gmeta` files automatically, including generation, validation, and cleanup.
## usage
### Initialization
The Asset Database must be initialized after the project is loaded.
```csharp
await AssetDatabase.Initialize(cancellationToken);
```
### Loading Assets
Assets can be loaded by GUID or by Path.
```csharp
// Load by Path
var result = AssetDatabase.LoadAssetAtPath<TextureAsset>("Assets/Textures/my_texture.png");
if (result.IsSuccess)
{
var texture = result.Value;
}
// Load by GUID
var guid = ...;
var result = AssetDatabase.LoadAsset<TextureAsset>(guid);
```
### File Operations
Always use the `AssetDatabase` API for file operations to ensure metadata is preserved.
```csharp
// Create
await AssetDatabase.CreateAssetAsync("Assets/Data/config.json", dataBytes);
// Move
await AssetDatabase.MoveAssetAsync("Assets/Old/file.txt", "Assets/New/file.txt");
// Copy
await AssetDatabase.CopyAssetAsync("Assets/template.txt", "Assets/instance.txt");
// Delete
await AssetDatabase.DeleteAssetAsync("Assets/garbage.tmp");
```
### Searching
Find assets using wildcards or tags.
```csharp
// Find all PNGs
var guids = await AssetDatabase.FindAssetsByNameAsync("*.png");
// Find assets with a specific tag
var enemyAssets = await AssetDatabase.FindAssetsByTagAsync("Enemy");
```
### Tags
Manage asset tags for organization.
```csharp
// Get tags
var tagsResult = await AssetDatabase.GetAssetTagsAsync(guid);
// Set tags
await AssetDatabase.SetAssetTagsAsync(guid, new List<string> { "Level1", "Prop" });
```
### Opening Assets
Open an asset using its registered handler or the system default.
```csharp
AssetDatabase.OpenAsset("Assets/Docs/readme.txt");
```
## Extending the Asset Database
### Creating a New Importer
To support a new file type, create a class that inherits from `AssetImporter<T>` and decorate it with the `[AssetImporter]` attribute.
```csharp
[AssetImporter(".myfmt")]
internal class MyFormatImporter : AssetImporter<MyFormatSettings>
{
public override async Task<Result> ImportAsync(string assetPath, AssetMeta meta)
{
var settings = GetSettings(meta);
// 1. Read source file
// 2. Process data
// 3. Save imported data using AssetDatabase.SaveImportedAsset
var myAsset = new MyAsset(meta.Guid) { ... };
return AssetDatabase.SaveImportedAsset(meta.Guid, myAsset);
}
}
internal class MyFormatSettings : ImporterSettings
{
public float Scale { get; set; } = 1.0f;
}
```
### Creating an Open Handler
To define custom behavior when an asset is opened (e.g., double-clicked in the editor), use the `[AssetOpenHandler]` attribute.
```csharp
internal static class MyHandlers
{
[AssetOpenHandler(".myfmt")]
private static void OpenMyFormat(string path)
{
// Open custom editor window
}
}
```
## Internal Architecture
- **AssetDatabase.cs**: Core initialization and event coordination.
- **AssetDatabase.SQLite.cs**: Database table management and queries.
- **AssetDatabase.Meta.cs**: `.gmeta` file handling and file system watcher events.
- **AssetDatabase.Importer.cs**: Importer discovery and execution.
- **AssetDatabase.Loader.cs**: Asset loading and caching logic.

0
Ghost.Editor.Core/AssetHandle/Asset.cs → Ghost.Editor.Core/AssetHandle/Models/Asset.cs
View File

0
Ghost.Editor.Core/AssetHandle/TextureAsset.cs → Ghost.Editor.Core/AssetHandle/Models/TextureAsset.cs
View File

250
Ghost.Editor.Core/AssetHandle/README.md
View File

@@ -1,250 +0,0 @@
# Asset Database Implementation
This is the complete implementation of the GhostEngine Asset Database system based on the plan in `AssetDBPlan.md`.
## Structure
The asset database is implemented as a partial class split across multiple files:
### Core Files
- **AssetDatabase.cs** - Main entry point with initialization and shutdown logic
- **AssetDatabase.Meta.cs** - Metadata file management and file system watching
- **AssetDatabase.SQLite.cs** - SQLite database operations for caching
- **AssetDatabase.Lookup.cs** - GUID/Path lookup and search operations
- **AssetDatabase.FileOps.cs** - File operations (create, delete, move, copy)
- **AssetDatabase.Importer.cs** - Asset importing framework
- **AssetDatabase.Open.cs** - Asset opening handlers (existing file)
### Supporting Files
- **Asset.cs** - Base class for all assets
- **AssetMeta.cs** - Metadata structure (stored in .gmeta files)
- **AssetImporter.cs** - Base class for all asset importers
- **AssetImporterAttribute.cs** - Attribute to mark importer classes
- **AssetOpenHandlerAttribute.cs** - Attribute for custom open handlers
- **ImporterSettings.cs** - Base class for importer settings
### Example Importer
- **Importers/TextImporter.cs** - Example importer for .txt and .md files
## Features Implemented
### Core API (AssetDatabase.Lookup.cs)
- ✅ `PathToGuid(string assetPath)` - Find GUID by path
- ✅ `GuidToPath(Guid guid)` - Find path by GUID
- ✅ `LoadAsset<T>(Guid guid)` - Load asset by GUID (TODO: needs asset loader)
- ✅ `GetAssetTagsAsync(Guid guid)` - Get asset tags
- ✅ `SetAssetTagsAsync(Guid guid, List<string> tags)` - Set asset tags
- ✅ `FindAssetsByName(string namePattern)` - Search by name
- ✅ `FindAssetsByTagAsync(string tag)` - Search by tag
- ✅ `GetAllAssets()` - Get all assets in database
### File Operations (AssetDatabase.FileOps.cs)
- ✅ `CreateAssetAsync(string assetPath, byte[] content)` - Create new asset
- ✅ `DeleteAssetAsync(Guid guid)` - Delete asset
- ✅ `MoveAssetAsync(Guid guid, string newPath)` - Move/rename asset
- ✅ `CopyAssetAsync(Guid guid, string newPath)` - Copy asset with new GUID
- ✅ `RefreshAsync()` - Refresh database manually
- ✅ `MarkDirtyAsync(Guid guid)` - Mark asset for re-import
- ✅ `ImportDirtyAssetsAsync()` - Import all dirty assets
### Background Services (AssetDatabase.Meta.cs)
- ✅ File system watcher for automatic change detection
- ✅ Automatic metadata generation on file creation
- ✅ Automatic metadata cleanup on file deletion
- ✅ Automatic metadata movement on file rename
- ✅ File hash comparison for change detection
- ✅ Automatic dirty marking on file modification
- ✅ Dependent asset tracking and dirty propagation
### Database (AssetDatabase.SQLite.cs)
- ✅ SQLite for persistent storage and efficient querying
- ✅ In-memory cache for fast lookups
- ✅ Automatic database creation and schema management
- ✅ Asset indexing by GUID and path
- ✅ Dirty flag tracking for re-import
- ✅ Tag-based search support
### Validation (AssetDatabase.cs)
- ✅ Validate and fix database on project load
- ✅ Check for missing/corrupted metadata files
- ✅ Regenerate metadata when necessary
- ✅ Database consistency checks
## Metadata File Format
Assets have associated `.gmeta` files stored alongside them:
```json
{
"Guid": "123e4567-e89b-12d3-a456-426614174000",
"Version": 1,
"Tags": ["Environment", "Texture"],
"FileHash": "ABC123...",
"Dependencies": [
"456e7890-e89b-12d3-a456-426614174001"
],
"ImporterSettings": {
"TextureImporter": {
"MaxSize": 2048,
"MipLevels": 1
}
}
}
```
## Usage Examples
### Finding Assets
```csharp
// Find by path
var guidResult = AssetDatabase.PathToGuid("Assets/Textures/logo.png");
if (guidResult.IsSuccess)
{
var guid = guidResult.Value;
// Use guid...
}
// Find by GUID
var pathResult = AssetDatabase.GuidToPath(myGuid);
if (pathResult.IsSuccess)
{
var path = pathResult.Value;
// Use path...
}
// Search by name
var results = AssetDatabase.FindAssetsByName("logo");
// Search by tag
var textureAssets = await AssetDatabase.FindAssetsByTagAsync("Texture");
```
### Creating and Managing Assets
```csharp
// Create new asset
var content = Encoding.UTF8.GetBytes("Hello, World!");
await AssetDatabase.CreateAssetAsync("Assets/test.txt", content);
// Move asset
await AssetDatabase.MoveAssetAsync(guid, "Assets/NewFolder/test.txt");
// Copy asset
var newGuid = await AssetDatabase.CopyAssetAsync(guid, "Assets/test_copy.txt");
// Delete asset
await AssetDatabase.DeleteAssetAsync(guid);
```
### Working with Tags
```csharp
// Get tags
var tagsResult = await AssetDatabase.GetAssetTagsAsync(guid);
if (tagsResult.IsSuccess)
{
var tags = tagsResult.Value;
}
// Set tags
await AssetDatabase.SetAssetTagsAsync(guid, new List<string> { "UI", "Icon" });
```
### Asset Importing
```csharp
// Mark asset dirty for re-import
await AssetDatabase.MarkDirtyAsync(guid);
// Import all dirty assets
await AssetDatabase.ImportDirtyAssetsAsync();
```
## Creating Custom Importers
To create a custom asset importer:
1. Create a settings class inheriting from `ImporterSettings`
2. Create an importer class inheriting from `AssetImporter<TSettings>`
3. Add the `[AssetImporter]` attribute with supported extensions
Example:
```csharp
public class MyImporterSettings : ImporterSettings
{
public bool SomeOption { get; set; } = true;
}
[AssetImporter(".myext")]
public class MyImporter : AssetImporter<MyImporterSettings>
{
public override async Task<Result> ImportAsync(string assetPath, AssetMeta meta)
{
var settings = GetSettings(meta);
// Validate dependencies
var depResult = await ValidateDependenciesAsync(meta);
if (depResult.IsFailure)
{
return depResult;
}
// Import logic here...
return Result.Success();
}
}
```
## Architecture Notes
### Source of Truth
The `.gmeta` files are the **source of truth** for asset information. The SQLite database is used only for:
- Caching for fast lookups
- Efficient querying and search operations
- Tracking dirty state
If the database becomes inconsistent, it can be regenerated from the `.gmeta` files by calling `RefreshAsync()`.
### Thread Safety
All database operations use locks to ensure thread safety. File system watcher events are handled asynchronously to avoid blocking the main thread.
### Error Handling
The system uses the `Result` pattern for railway-oriented programming. All operations return `Result` or `Result<T>` to indicate success or failure without throwing exceptions for expected failures.
## Testing
Unit tests should be added to verify:
- Metadata file generation and parsing
- Database consistency
- File operations (create, delete, move, copy)
- Asset importing
- Dependency tracking
- Tag management
- Search functionality
## Future Improvements
- Asset loader implementation (`LoadAsset<T>`)
- Asset browser UI
- More sophisticated dependency resolution
- Asset preview generation
- Asset versioning and migration
- Orphaned entry cleanup in database
- Better error reporting and logging
- Asset import progress tracking
- Parallel asset importing
- Asset thumbnail generation