feat(asset): modern asset system with SQLite catalog
Refactored asset management to use a persistent, thread-safe SQLite-backed AssetCatalog, replacing in-memory dictionaries. Added AssetHandlerRegistry for O(1) handler lookup, ImportCoordinator for async background importing, and robust AssetMeta/AssetMetaIO for JSON-based metadata and settings. Refactored AssetRegistry to integrate these components and support auto-import via file system watcher. Updated IImportableAssetHandler for handler-specific settings and polymorphic serialization. Added comprehensive unit tests for all new systems. Removed obsolete code and legacy integration tests. BREAKING CHANGE: Asset system APIs and storage format have changed; migration required for existing projects.
This commit is contained in:
@@ -175,5 +175,3 @@ public readonly struct AssetReference : IEquatable<AssetReference>
|
||||
return !(left == right);
|
||||
}
|
||||
}
|
||||
|
||||
public interface IAssetSettings;
|
||||
|
||||
@@ -32,17 +32,18 @@ public interface IAssetHandler
|
||||
|
||||
public interface IImportableAssetHandler : IAssetHandler
|
||||
{
|
||||
ValueTask<Result> ImportAsync(Stream sourceStream, Stream targetStream, Guid id, CancellationToken token = default);
|
||||
IAssetSettings? CreateDefaultSettings() => null;
|
||||
ValueTask<Result> ImportAsync(Stream sourceStream, Stream targetStream, Guid id, IAssetSettings? settings, CancellationToken token = default);
|
||||
ValueTask<Result> ExportAsync(Stream assetStream, Stream targetStream, IAssetExportOptions? options, CancellationToken token = default);
|
||||
}
|
||||
|
||||
public static class AssetHandlerExtensions
|
||||
{
|
||||
public static async ValueTask<Result> ImportAsync(this IImportableAssetHandler handler, string sourceFilePath, string targetFilePath, Guid id, CancellationToken token = default)
|
||||
public static async ValueTask<Result> ImportAsync(this IImportableAssetHandler handler, string sourceFilePath, string targetFilePath, Guid id, IAssetSettings? settings = null, CancellationToken token = default)
|
||||
{
|
||||
await using var sourceStream = new FileStream(sourceFilePath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
await using var targetStream = new FileStream(targetFilePath, FileMode.Create, FileAccess.Write, FileShare.None);
|
||||
return await handler.ImportAsync(sourceStream, targetStream, id, token);
|
||||
return await handler.ImportAsync(sourceStream, targetStream, id, settings, token);
|
||||
}
|
||||
|
||||
public static async ValueTask<Result> ExportAsync(this IImportableAssetHandler handler, string assetFilePath, string targetFilePath, IAssetExportOptions? options, CancellationToken token = default)
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
using System.Reflection;
|
||||
using Ghost.Editor.Core.Utilities;
|
||||
|
||||
namespace Ghost.Editor.Core.AssetHandler;
|
||||
|
||||
/// <summary>
|
||||
/// One-time scan at editor startup → two dictionaries.
|
||||
/// All lookups are O(1) after construction.
|
||||
/// </summary>
|
||||
internal sealed class AssetHandlerRegistry
|
||||
{
|
||||
private readonly Dictionary<string, IAssetHandler> _byExtension;
|
||||
private readonly Dictionary<Guid, IAssetHandler> _byTypeId;
|
||||
private readonly Dictionary<Guid, int> _versionByTypeId;
|
||||
|
||||
public AssetHandlerRegistry()
|
||||
{
|
||||
_byExtension = new Dictionary<string, IAssetHandler>(StringComparer.OrdinalIgnoreCase);
|
||||
_byTypeId = new Dictionary<Guid, IAssetHandler>();
|
||||
_versionByTypeId = new Dictionary<Guid, int>();
|
||||
|
||||
foreach (var typeInfo in TypeCache.GetTypes())
|
||||
{
|
||||
if (typeInfo.IsAbstract || typeInfo.IsInterface)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!typeof(IAssetHandler).IsAssignableFrom(typeInfo))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var attr = typeInfo.GetCustomAttribute<CustomAssetHandlerAttribute>();
|
||||
if (attr == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!Guid.TryParse(attr.ID, out var typeId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Activator.CreateInstance(typeInfo) is IAssetHandler handler)
|
||||
{
|
||||
_byTypeId[typeId] = handler;
|
||||
// Note: Versioning could be expanded, but for now we assume version 1 or look for a constant
|
||||
_versionByTypeId[typeId] = 1;
|
||||
|
||||
foreach (var ext in attr.SupportedExtensions)
|
||||
{
|
||||
var normalizedExt = ext.StartsWith('.') ? ext : "." + ext;
|
||||
_byExtension[normalizedExt] = handler;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Log failure to instantiate handler in real app
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public IAssetHandler? GetByExtension(string extension)
|
||||
{
|
||||
if (string.IsNullOrEmpty(extension))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var normalized = extension.StartsWith('.') ? extension : "." + extension;
|
||||
_byExtension.TryGetValue(normalized, out var handler);
|
||||
return handler;
|
||||
}
|
||||
|
||||
public IAssetHandler? GetByTypeId(Guid typeId)
|
||||
{
|
||||
_byTypeId.TryGetValue(typeId, out var handler);
|
||||
return handler;
|
||||
}
|
||||
|
||||
public int GetVersionByTypeId(Guid typeId)
|
||||
{
|
||||
_versionByTypeId.TryGetValue(typeId, out var version);
|
||||
return version;
|
||||
}
|
||||
|
||||
public IEnumerable<string> GetSupportedExtensions() => _byExtension.Keys;
|
||||
}
|
||||
114
src/Editor/Ghost.Editor.Core/AssetHandler/AssetMeta.cs
Normal file
114
src/Editor/Ghost.Editor.Core/AssetHandler/AssetMeta.cs
Normal file
@@ -0,0 +1,114 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Ghost.Editor.Core.AssetHandler;
|
||||
|
||||
/// <summary>
|
||||
/// Mark IAssetSettings for polymorphic serialization.
|
||||
/// Each handler type will register its own derived type.
|
||||
/// </summary>
|
||||
[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")]
|
||||
[JsonDerivedType(typeof(DefaultAssetSettings), "Default")]
|
||||
public interface IAssetSettings;
|
||||
|
||||
public sealed class DefaultAssetSettings : IAssetSettings;
|
||||
|
||||
/// <summary>
|
||||
/// Persisted as a JSON sidecar (.gmeta) next to every source asset.
|
||||
/// This is the single source of truth for asset identity and import settings.
|
||||
/// </summary>
|
||||
public sealed class AssetMeta
|
||||
{
|
||||
/// <summary>
|
||||
/// Globally unique identifier for this asset. Generated once, never changes.
|
||||
/// </summary>
|
||||
public required Guid Guid { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The Guid that identifies which IAssetHandler processes this asset.
|
||||
/// </summary>
|
||||
public Guid? HandlerTypeId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Version of the handler that last imported this asset.
|
||||
/// </summary>
|
||||
public int HandlerVersion { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// xxHash64 of the source file content at last successful import.
|
||||
/// </summary>
|
||||
public string? ContentHash { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// xxHash64 of the serialized import settings at last successful import.
|
||||
/// </summary>
|
||||
public string? SettingsHash { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// UTC timestamp of last successful import.
|
||||
/// </summary>
|
||||
public DateTime? LastImportedUtc { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// GUIDs of other assets this asset depends on.
|
||||
/// </summary>
|
||||
public Guid[] Dependencies { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Optional user-facing labels for search/filtering in the editor.
|
||||
/// </summary>
|
||||
public string[] Labels { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Handler-specific import settings.
|
||||
/// </summary>
|
||||
public IAssetSettings? Settings { get; set; }
|
||||
}
|
||||
|
||||
internal static class AssetMetaIO
|
||||
{
|
||||
private static readonly JsonSerializerOptions s_options = new()
|
||||
{
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) }
|
||||
};
|
||||
|
||||
public static async ValueTask<AssetMeta?> ReadAsync(string metaPath, CancellationToken token = default)
|
||||
{
|
||||
if (!File.Exists(metaPath))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await using var stream = new FileStream(metaPath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
return await JsonSerializer.DeserializeAsync<AssetMeta>(stream, s_options, token).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static async ValueTask WriteAsync(string metaPath, AssetMeta meta, CancellationToken token = default)
|
||||
{
|
||||
var tempPath = metaPath + ".tmp";
|
||||
await using (var stream = new FileStream(tempPath, FileMode.Create, FileAccess.Write, FileShare.None))
|
||||
{
|
||||
await JsonSerializer.SerializeAsync(stream, meta, s_options, token).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (File.Exists(metaPath))
|
||||
{
|
||||
File.Delete(metaPath);
|
||||
}
|
||||
File.Move(tempPath, metaPath);
|
||||
}
|
||||
|
||||
public static string GetMetaPath(string sourceFilePath) => sourceFilePath + ".gmeta";
|
||||
|
||||
public static string GetSourcePath(string metaPath) => metaPath[..^".gmeta".Length];
|
||||
}
|
||||
@@ -290,13 +290,13 @@ internal class TextureAssetHandler : IImportableAssetHandler
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public async ValueTask<Result> ImportAsync(Stream sourceStream, Stream targetStream, Guid id, CancellationToken token = default)
|
||||
public async ValueTask<Result> ImportAsync(Stream sourceStream, Stream targetStream, Guid id, IAssetSettings? settings, CancellationToken token = default)
|
||||
{
|
||||
var textureSettings = settings as TextureAssetSettings ?? new TextureAssetSettings();
|
||||
using var image = new MagickImage(sourceStream);
|
||||
var bytes = image.ToByteArray();
|
||||
|
||||
var settings = new TextureAssetSettings();
|
||||
await TextureProcessor.CompressToCacheAsync(EditorApplication.LibraryFolderPath, id, bytes, image.Width, image.Height, image.Depth, settings, token).ConfigureAwait(false);
|
||||
await TextureProcessor.CompressToCacheAsync(EditorApplication.LibraryFolderPath, id, bytes, image.Width, image.Height, image.Depth, textureSettings, token).ConfigureAwait(false);
|
||||
|
||||
var header = new AssetMetadata(id, TextureAsset.s_typeGuid)
|
||||
{
|
||||
@@ -305,7 +305,7 @@ internal class TextureAssetHandler : IImportableAssetHandler
|
||||
};
|
||||
|
||||
targetStream.Seek(header.SettingsOffset, SeekOrigin.Begin);
|
||||
var sizeResult = await WriteSettingsToStreamAsync(settings, targetStream, token).ConfigureAwait(false);
|
||||
var sizeResult = await WriteSettingsToStreamAsync(textureSettings, targetStream, token).ConfigureAwait(false);
|
||||
if (sizeResult.IsFailure)
|
||||
{
|
||||
return Result.Failure($"Failed to write texture asset settings: {sizeResult.Message}");
|
||||
|
||||
@@ -55,23 +55,10 @@ public class EditorInjectionAttribute : DiscoverableAttributeBase
|
||||
{
|
||||
Singleton,
|
||||
Transient,
|
||||
Scoped
|
||||
}
|
||||
|
||||
public ServiceLifetime Lifetime
|
||||
public EditorInjectionAttribute(ServiceLifetime lifetime, Type implementationType)
|
||||
{
|
||||
get;
|
||||
}
|
||||
|
||||
public Type? ImplementationType
|
||||
{
|
||||
get;
|
||||
}
|
||||
|
||||
public EditorInjectionAttribute(ServiceLifetime lifetime, Type? implementationType = null)
|
||||
{
|
||||
Lifetime = lifetime;
|
||||
ImplementationType = implementationType;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using Ghost.Core;
|
||||
using Ghost.Editor.Core.AssetHandler;
|
||||
using Ghost.Editor.Core.Services;
|
||||
|
||||
namespace Ghost.Editor.Core.Contracts;
|
||||
|
||||
@@ -37,6 +38,7 @@ public sealed class AssetChangedEventArgs : EventArgs
|
||||
}
|
||||
}
|
||||
|
||||
[EditorInjection(EditorInjectionAttribute.ServiceLifetime.Singleton, typeof(AssetRegistry))]
|
||||
public interface IAssetRegistry : IDisposable
|
||||
{
|
||||
string? GetAssetPath(Guid id);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using Ghost.Editor.Core.Utilities;
|
||||
using Microsoft.UI.Dispatching;
|
||||
using Microsoft.UI.Xaml;
|
||||
|
||||
@@ -11,6 +12,8 @@ public static class EditorApplication
|
||||
public const string LIBRARY_FOLDER_NAME = "Library";
|
||||
public const string CONFIG_FOLDER_NAME = "Config";
|
||||
|
||||
public const string IMPORTS_FOLDER_NAME = "Imports";
|
||||
|
||||
private static IServiceProvider? s_serviceProvider;
|
||||
private static string s_currentProjectPath = string.Empty;
|
||||
private static string s_currentProjectName = string.Empty;
|
||||
@@ -22,11 +25,13 @@ public static class EditorApplication
|
||||
public static string ProjectPath => s_currentProjectPath;
|
||||
public static string ProjectName => s_currentProjectName;
|
||||
|
||||
public static string AssetsFolderPath => Path.Combine(ProjectPath, ASSETS_FOLDER_NAME);
|
||||
public static string SourcesFolderPath => Path.Combine(ProjectPath, SOURCES_FOLDER_NAME);
|
||||
public static string PackagesFolderPath => Path.Combine(ProjectPath, PACKAGES_FOLDER_NAME);
|
||||
public static string LibraryFolderPath => Path.Combine(ProjectPath, LIBRARY_FOLDER_NAME);
|
||||
public static string ConfigFolderPath => Path.Combine(ProjectPath, CONFIG_FOLDER_NAME);
|
||||
public static readonly string AssetsFolderPath = Path.Combine(ProjectPath, ASSETS_FOLDER_NAME);
|
||||
public static readonly string SourcesFolderPath = Path.Combine(ProjectPath, SOURCES_FOLDER_NAME);
|
||||
public static readonly string PackagesFolderPath = Path.Combine(ProjectPath, PACKAGES_FOLDER_NAME);
|
||||
public static readonly string LibraryFolderPath = Path.Combine(ProjectPath, LIBRARY_FOLDER_NAME);
|
||||
public static readonly string ConfigFolderPath = Path.Combine(ProjectPath, CONFIG_FOLDER_NAME);
|
||||
|
||||
public static readonly string ImportsFolderPath = Path.Combine(LibraryFolderPath, IMPORTS_FOLDER_NAME);
|
||||
|
||||
public static DispatcherQueue DispatcherQueue
|
||||
{
|
||||
|
||||
264
src/Editor/Ghost.Editor.Core/Services/AssetCatalog.cs
Normal file
264
src/Editor/Ghost.Editor.Core/Services/AssetCatalog.cs
Normal file
@@ -0,0 +1,264 @@
|
||||
using Ghost.Editor.Core.AssetHandler;
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
namespace Ghost.Editor.Core.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Thread-safe SQLite-backed asset catalog.
|
||||
/// Replaces the in-memory dictionary approach with persistent storage.
|
||||
/// </summary>
|
||||
internal sealed class AssetCatalog : IDisposable
|
||||
{
|
||||
private readonly SqliteConnection _connection;
|
||||
private readonly object _writeLock = new();
|
||||
|
||||
// Prepared statements
|
||||
private readonly SqliteCommand _cmdGetGuid;
|
||||
private readonly SqliteCommand _cmdGetPath;
|
||||
private readonly SqliteCommand _cmdUpsert;
|
||||
private readonly SqliteCommand _cmdDelete;
|
||||
private readonly SqliteCommand _cmdMarkDirty;
|
||||
private readonly SqliteCommand _cmdMarkImported;
|
||||
private readonly SqliteCommand _cmdMarkFailed;
|
||||
private readonly SqliteCommand _cmdGetReferencers;
|
||||
private readonly SqliteCommand _cmdGetDependencies;
|
||||
private readonly SqliteCommand _cmdInsertDep;
|
||||
private readonly SqliteCommand _cmdClearDeps;
|
||||
private readonly SqliteCommand _cmdGetDirty;
|
||||
private readonly SqliteCommand _cmdEnumerate;
|
||||
|
||||
public AssetCatalog(string dbPath)
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(dbPath)!);
|
||||
|
||||
var connString = new SqliteConnectionStringBuilder
|
||||
{
|
||||
DataSource = dbPath,
|
||||
Cache = SqliteCacheMode.Shared,
|
||||
}.ToString();
|
||||
|
||||
_connection = new SqliteConnection(connString);
|
||||
_connection.Open();
|
||||
|
||||
using (var pragma = _connection.CreateCommand())
|
||||
{
|
||||
pragma.CommandText = "PRAGMA journal_mode = WAL; PRAGMA foreign_keys = ON;";
|
||||
pragma.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
CreateSchema();
|
||||
|
||||
_cmdGetGuid = CreateCommand("SELECT guid FROM assets WHERE source_path = @path");
|
||||
_cmdGetPath = CreateCommand("SELECT source_path FROM assets WHERE guid = @guid");
|
||||
_cmdUpsert = CreateCommand(@"
|
||||
INSERT INTO assets (guid, source_path, handler_type_id, handler_version, state)
|
||||
VALUES (@guid, @path, @handler_id, @version, 0)
|
||||
ON CONFLICT(guid) DO UPDATE SET
|
||||
source_path = excluded.source_path,
|
||||
handler_type_id = excluded.handler_type_id,
|
||||
handler_version = excluded.handler_version,
|
||||
state = 0;");
|
||||
_cmdDelete = CreateCommand("DELETE FROM assets WHERE guid = @guid");
|
||||
_cmdMarkDirty = CreateCommand("UPDATE assets SET state = 0 WHERE guid = @guid");
|
||||
_cmdMarkImported = CreateCommand(@"
|
||||
UPDATE assets SET
|
||||
content_hash = @content_hash,
|
||||
settings_hash = @settings_hash,
|
||||
imported_at_ms = @time,
|
||||
state = 1,
|
||||
error_message = NULL
|
||||
WHERE guid = @guid");
|
||||
_cmdMarkFailed = CreateCommand("UPDATE assets SET state = 2, error_message = @msg WHERE guid = @guid");
|
||||
_cmdGetReferencers = CreateCommand("SELECT from_guid FROM dependencies WHERE to_guid = @guid");
|
||||
_cmdGetDependencies = CreateCommand("SELECT to_guid FROM dependencies WHERE from_guid = @guid");
|
||||
_cmdInsertDep = CreateCommand("INSERT INTO dependencies (from_guid, to_guid) VALUES (@from, @to)");
|
||||
_cmdClearDeps = CreateCommand("DELETE FROM dependencies WHERE from_guid = @guid");
|
||||
_cmdGetDirty = CreateCommand("SELECT guid, source_path FROM assets WHERE state = 0");
|
||||
_cmdEnumerate = CreateCommand("SELECT guid, source_path FROM assets");
|
||||
}
|
||||
|
||||
private SqliteCommand CreateCommand(string sql)
|
||||
{
|
||||
var cmd = _connection.CreateCommand();
|
||||
cmd.CommandText = sql;
|
||||
return cmd;
|
||||
}
|
||||
|
||||
private void CreateSchema()
|
||||
{
|
||||
using var cmd = _connection.CreateCommand();
|
||||
cmd.CommandText = @"
|
||||
CREATE TABLE IF NOT EXISTS assets (
|
||||
guid BLOB(16) PRIMARY KEY NOT NULL,
|
||||
source_path TEXT NOT NULL,
|
||||
handler_type_id BLOB(16),
|
||||
handler_version INTEGER NOT NULL DEFAULT 0,
|
||||
content_hash TEXT,
|
||||
settings_hash TEXT,
|
||||
imported_at_ms INTEGER,
|
||||
state INTEGER NOT NULL DEFAULT 0,
|
||||
error_message TEXT
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_assets_path ON assets(source_path);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dependencies (
|
||||
from_guid BLOB(16) NOT NULL REFERENCES assets(guid) ON DELETE CASCADE,
|
||||
to_guid BLOB(16) NOT NULL REFERENCES assets(guid) ON DELETE CASCADE,
|
||||
PRIMARY KEY (from_guid, to_guid)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_dep_reverse ON dependencies(to_guid);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS labels (
|
||||
guid BLOB(16) NOT NULL REFERENCES assets(guid) ON DELETE CASCADE,
|
||||
label TEXT NOT NULL,
|
||||
PRIMARY KEY (guid, label)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_labels_label ON labels(label);";
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
public Guid GetGuid(string sourcePath)
|
||||
{
|
||||
_cmdGetGuid.Parameters.Clear();
|
||||
_cmdGetGuid.Parameters.AddWithValue("@path", sourcePath);
|
||||
var result = _cmdGetGuid.ExecuteScalar();
|
||||
return result is byte[] bytes ? new Guid(bytes) : Guid.Empty;
|
||||
}
|
||||
|
||||
public string? GetSourcePath(Guid guid)
|
||||
{
|
||||
_cmdGetPath.Parameters.Clear();
|
||||
_cmdGetPath.Parameters.AddWithValue("@guid", guid.ToByteArray());
|
||||
return _cmdGetPath.ExecuteScalar() as string;
|
||||
}
|
||||
|
||||
public void Upsert(AssetMeta meta, string sourcePath)
|
||||
{
|
||||
lock (_writeLock)
|
||||
{
|
||||
_cmdUpsert.Parameters.Clear();
|
||||
_cmdUpsert.Parameters.AddWithValue("@guid", meta.Guid.ToByteArray());
|
||||
_cmdUpsert.Parameters.AddWithValue("@path", sourcePath);
|
||||
_cmdUpsert.Parameters.AddWithValue("@handler_id", meta.HandlerTypeId?.ToByteArray() ?? (object)DBNull.Value);
|
||||
_cmdUpsert.Parameters.AddWithValue("@version", meta.HandlerVersion);
|
||||
_cmdUpsert.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
|
||||
public bool Remove(Guid guid)
|
||||
{
|
||||
lock (_writeLock)
|
||||
{
|
||||
_cmdDelete.Parameters.Clear();
|
||||
_cmdDelete.Parameters.AddWithValue("@guid", guid.ToByteArray());
|
||||
return _cmdDelete.ExecuteNonQuery() > 0;
|
||||
}
|
||||
}
|
||||
|
||||
public void MarkDirty(Guid guid)
|
||||
{
|
||||
lock (_writeLock)
|
||||
{
|
||||
_cmdMarkDirty.Parameters.Clear();
|
||||
_cmdMarkDirty.Parameters.AddWithValue("@guid", guid.ToByteArray());
|
||||
_cmdMarkDirty.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
|
||||
public void MarkImported(Guid guid, string contentHash, string settingsHash)
|
||||
{
|
||||
lock (_writeLock)
|
||||
{
|
||||
_cmdMarkImported.Parameters.Clear();
|
||||
_cmdMarkImported.Parameters.AddWithValue("@guid", guid.ToByteArray());
|
||||
_cmdMarkImported.Parameters.AddWithValue("@content_hash", contentHash);
|
||||
_cmdMarkImported.Parameters.AddWithValue("@settings_hash", settingsHash);
|
||||
_cmdMarkImported.Parameters.AddWithValue("@time", DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
|
||||
_cmdMarkImported.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
|
||||
public void MarkFailed(Guid guid, string error)
|
||||
{
|
||||
lock (_writeLock)
|
||||
{
|
||||
_cmdMarkFailed.Parameters.Clear();
|
||||
_cmdMarkFailed.Parameters.AddWithValue("@guid", guid.ToByteArray());
|
||||
_cmdMarkFailed.Parameters.AddWithValue("@msg", error);
|
||||
_cmdMarkFailed.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
|
||||
public void SetDependencies(Guid assetId, ReadOnlySpan<Guid> dependencies)
|
||||
{
|
||||
lock (_writeLock)
|
||||
{
|
||||
using var tx = _connection.BeginTransaction();
|
||||
_cmdClearDeps.Transaction = tx;
|
||||
_cmdClearDeps.Parameters.Clear();
|
||||
_cmdClearDeps.Parameters.AddWithValue("@guid", assetId.ToByteArray());
|
||||
_cmdClearDeps.ExecuteNonQuery();
|
||||
|
||||
_cmdInsertDep.Transaction = tx;
|
||||
foreach (var dep in dependencies)
|
||||
{
|
||||
_cmdInsertDep.Parameters.Clear();
|
||||
_cmdInsertDep.Parameters.AddWithValue("@from", assetId.ToByteArray());
|
||||
_cmdInsertDep.Parameters.AddWithValue("@to", dep.ToByteArray());
|
||||
_cmdInsertDep.ExecuteNonQuery();
|
||||
}
|
||||
tx.Commit();
|
||||
}
|
||||
}
|
||||
|
||||
public List<Guid> GetReferencers(Guid guid)
|
||||
{
|
||||
_cmdGetReferencers.Parameters.Clear();
|
||||
_cmdGetReferencers.Parameters.AddWithValue("@guid", guid.ToByteArray());
|
||||
using var reader = _cmdGetReferencers.ExecuteReader();
|
||||
var list = new List<Guid>();
|
||||
while (reader.Read())
|
||||
{
|
||||
list.Add(new Guid((byte[])reader[0]));
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public List<(Guid guid, string sourcePath)> GetDirtyAssets()
|
||||
{
|
||||
using var reader = _cmdGetDirty.ExecuteReader();
|
||||
var list = new List<(Guid guid, string sourcePath)>();
|
||||
while (reader.Read())
|
||||
{
|
||||
list.Add((new Guid((byte[])reader[0]), reader.GetString(1)));
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public IEnumerable<(Guid guid, string sourcePath)> EnumerateAll()
|
||||
{
|
||||
using var reader = _cmdEnumerate.ExecuteReader();
|
||||
while (reader.Read())
|
||||
{
|
||||
yield return (new Guid((byte[])reader[0]), reader.GetString(1));
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_cmdGetGuid.Dispose();
|
||||
_cmdGetPath.Dispose();
|
||||
_cmdUpsert.Dispose();
|
||||
_cmdDelete.Dispose();
|
||||
_cmdMarkDirty.Dispose();
|
||||
_cmdMarkImported.Dispose();
|
||||
_cmdMarkFailed.Dispose();
|
||||
_cmdGetReferencers.Dispose();
|
||||
_cmdGetDependencies.Dispose();
|
||||
_cmdInsertDep.Dispose();
|
||||
_cmdClearDeps.Dispose();
|
||||
_cmdGetDirty.Dispose();
|
||||
_cmdEnumerate.Dispose();
|
||||
_connection.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
namespace TestProject.AssetDB;
|
||||
|
||||
internal partial class AssetRegistry
|
||||
{
|
||||
// TODO: Sqlite backend implementation
|
||||
}
|
||||
@@ -1,510 +1,269 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Reflection;
|
||||
using Ghost.Core;
|
||||
using Ghost.Editor.Core.AssetHandler;
|
||||
using Ghost.Editor.Core.Contracts;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace TestProject.AssetDB;
|
||||
namespace Ghost.Editor.Core.Services;
|
||||
|
||||
internal class PathComparer : IEqualityComparer<string>
|
||||
/// <summary>
|
||||
/// Central asset registry for the GhostEngine editor.
|
||||
/// </summary>
|
||||
internal sealed class AssetRegistry : IAssetRegistry, IDisposable
|
||||
{
|
||||
private static string ToCanonicalPath(string? path)
|
||||
{
|
||||
return path?.Replace('\\', '/').TrimEnd('/') ?? string.Empty;
|
||||
}
|
||||
|
||||
public bool Equals(string? x, string? y)
|
||||
{
|
||||
return string.Equals(
|
||||
ToCanonicalPath(x),
|
||||
ToCanonicalPath(y),
|
||||
StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
public int GetHashCode(string str)
|
||||
{
|
||||
return ToCanonicalPath(str).GetHashCode(StringComparison.Ordinal);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Path based locking for multi-threaded access?
|
||||
// Is it actually necessary since this is mostly used in editor environment where single-threaded access is common (99.999%)?
|
||||
internal partial class AssetRegistry : IAssetRegistry
|
||||
{
|
||||
public const string ASSET_EXTENSION = ".gasset";
|
||||
public const string TEMP_EXTENSION = ".gtemp";
|
||||
|
||||
private readonly string _rootDirectory;
|
||||
private readonly string _assetsRoot;
|
||||
private readonly string _libraryRoot;
|
||||
private readonly AssetCatalog _catalog;
|
||||
private readonly AssetHandlerRegistry _handlerRegistry;
|
||||
private readonly ImportCoordinator _importCoordinator;
|
||||
private readonly FileSystemWatcher _watcher;
|
||||
|
||||
private readonly ConcurrentDictionary<string, Guid> _pathToGuid;
|
||||
private readonly ConcurrentDictionary<Guid, string> _guidToPath;
|
||||
|
||||
private readonly ConcurrentDictionary<nint, IAssetHandler> _cachedHander;
|
||||
private readonly ConcurrentDictionary<Guid, WeakReference<Asset>> _loadedAssets;
|
||||
|
||||
private readonly Dictionary<Guid, HashSet<Guid>> _referencerGraph;
|
||||
private readonly Dictionary<Guid, HashSet<Guid>> _dependencyCache;
|
||||
|
||||
private readonly ConcurrentDictionary<string, bool> _ignoreFileChanges;
|
||||
|
||||
private readonly SemaphoreSlim _cacheSlim;
|
||||
private readonly Lock _pathLock;
|
||||
private readonly SemaphoreSlim _loadLock = new(1, 1);
|
||||
private readonly ConcurrentDictionary<string, bool> _ignoreMetaWrites = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public event EventHandler<IAssetRegistry, AssetChangedEventArgs>? OnAssetChanged;
|
||||
|
||||
public AssetRegistry(string rootDirectory)
|
||||
public AssetRegistry(string assetsRoot)
|
||||
{
|
||||
if (!Directory.Exists(rootDirectory))
|
||||
{
|
||||
throw new DirectoryNotFoundException("The specified root directory does not exist.");
|
||||
}
|
||||
_assetsRoot = Path.GetFullPath(assetsRoot);
|
||||
_libraryRoot = Path.Combine(Path.GetDirectoryName(_assetsRoot)!, EditorApplication.LIBRARY_FOLDER_NAME);
|
||||
|
||||
if (!Path.IsPathFullyQualified(rootDirectory))
|
||||
{
|
||||
throw new InvalidOperationException("The specified root directory must be an absolute path.");
|
||||
}
|
||||
// TODO: This should be handled by EditorApplication.
|
||||
Directory.CreateDirectory(_assetsRoot);
|
||||
Directory.CreateDirectory(_libraryRoot);
|
||||
|
||||
_rootDirectory = rootDirectory;
|
||||
_watcher = new FileSystemWatcher(rootDirectory)
|
||||
var dbPath = Path.Combine(_libraryRoot, "AssetDB.sqlite");
|
||||
|
||||
_catalog = new AssetCatalog(dbPath);
|
||||
_handlerRegistry = new AssetHandlerRegistry();
|
||||
_importCoordinator = new ImportCoordinator(_catalog, _handlerRegistry, _assetsRoot, _libraryRoot);
|
||||
|
||||
_loadedAssets = new ConcurrentDictionary<Guid, WeakReference<Asset>>();
|
||||
|
||||
SyncCatalogWithDisk();
|
||||
|
||||
_watcher = new FileSystemWatcher(_assetsRoot)
|
||||
{
|
||||
IncludeSubdirectories = true,
|
||||
EnableRaisingEvents = true,
|
||||
NotifyFilter = NotifyFilters.FileName | NotifyFilters.LastWrite | NotifyFilters.DirectoryName
|
||||
};
|
||||
|
||||
_pathToGuid = new ConcurrentDictionary<string, Guid>(4, 512, new PathComparer());
|
||||
_guidToPath = new ConcurrentDictionary<Guid, string>(4, 512);
|
||||
_cachedHander = new ConcurrentDictionary<nint, IAssetHandler>(4, 16);
|
||||
_loadedAssets = new ConcurrentDictionary<Guid, WeakReference<Asset>>(4, 512);
|
||||
_watcher.Created += OnFileSystemEvent;
|
||||
_watcher.Deleted += OnFileSystemEvent;
|
||||
_watcher.Changed += OnFileSystemEvent;
|
||||
_watcher.Renamed += OnFileSystemRenameEvent;
|
||||
|
||||
_referencerGraph = new Dictionary<Guid, HashSet<Guid>>();
|
||||
_dependencyCache = new Dictionary<Guid, HashSet<Guid>>();
|
||||
|
||||
_ignoreFileChanges = new ConcurrentDictionary<string, bool>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
_cacheSlim = new SemaphoreSlim(1, 1);
|
||||
_pathLock = new Lock();
|
||||
|
||||
LoadExistingAssets();
|
||||
|
||||
_watcher.Created += OnFileSystemOp;
|
||||
_watcher.Deleted += OnFileSystemOp;
|
||||
_watcher.Changed += OnFileSystemOp;
|
||||
_watcher.Renamed += OnFileSystemRenameOp;
|
||||
_importCoordinator.EnqueueDirtyAssetsAsync().AsTask().Wait();
|
||||
}
|
||||
|
||||
// TODO: DB Cache
|
||||
private unsafe void LoadExistingAssets()
|
||||
private void SyncCatalogWithDisk()
|
||||
{
|
||||
Span<byte> guidBuffer = stackalloc byte[sizeof(Guid)];
|
||||
foreach (var filePath in Directory.EnumerateFiles(_rootDirectory, $"*{ASSET_EXTENSION}", SearchOption.AllDirectories))
|
||||
{
|
||||
var relativePath = Path.GetRelativePath(_rootDirectory, filePath);
|
||||
|
||||
try
|
||||
{
|
||||
var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
try
|
||||
{
|
||||
fs.Seek(4, SeekOrigin.Begin); // Skip format version
|
||||
fs.ReadExactly(guidBuffer);
|
||||
|
||||
var guid = Unsafe.ReadUnaligned<Guid>(ref MemoryMarshal.GetReference(guidBuffer));
|
||||
UpdatePathMapping(relativePath, guid);
|
||||
}
|
||||
finally
|
||||
{
|
||||
fs.Dispose();
|
||||
}
|
||||
}
|
||||
catch (Exception
|
||||
#if DEBUG
|
||||
ex
|
||||
#endif
|
||||
)
|
||||
{
|
||||
#if DEBUG
|
||||
System.Diagnostics.Debugger.BreakForUserUnhandledException(ex);
|
||||
#endif
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdateGraph(Guid assetId, IEnumerable<Guid> newDependencies)
|
||||
{
|
||||
// 1. Clean up old references (reverse)
|
||||
if (_dependencyCache.TryGetValue(assetId, out var oldDeps))
|
||||
{
|
||||
foreach (var dep in oldDeps)
|
||||
{
|
||||
if (_referencerGraph.TryGetValue(dep, out var refs))
|
||||
{
|
||||
refs.Remove(assetId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Set new forward dependencies
|
||||
var newDepSet = new HashSet<Guid>(newDependencies);
|
||||
_dependencyCache[assetId] = newDepSet;
|
||||
|
||||
// 3. Add new references (reverse)
|
||||
foreach (var dep in newDepSet)
|
||||
{
|
||||
ref var referencers = ref CollectionsMarshal.GetValueRefOrAddDefault(_referencerGraph, dep, out var exists);
|
||||
if (!exists || referencers is null)
|
||||
{
|
||||
referencers = new HashSet<Guid>();
|
||||
}
|
||||
|
||||
referencers.Add(assetId);
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdatePathMapping(string relativePath, Guid guid)
|
||||
{
|
||||
lock (_pathLock)
|
||||
{
|
||||
_pathToGuid[relativePath] = guid;
|
||||
_guidToPath[guid] = relativePath;
|
||||
}
|
||||
}
|
||||
|
||||
private bool RemovePathMappingByPath(string relativePath)
|
||||
{
|
||||
lock (_pathLock)
|
||||
{
|
||||
if (_pathToGuid.Remove(relativePath, out var guid))
|
||||
{
|
||||
return _guidToPath.TryRemove(guid, out _);
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private async void OnFileSystemOp(object sender, FileSystemEventArgs e)
|
||||
{
|
||||
if (_ignoreFileChanges.TryRemove(e.FullPath, out _))
|
||||
if (!Directory.Exists(_assetsRoot))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var relativePath = Path.GetRelativePath(_rootDirectory, e.FullPath);
|
||||
var ext = Path.GetExtension(relativePath);
|
||||
var metaFiles = Directory.EnumerateFiles(_assetsRoot, "*.gmeta", SearchOption.AllDirectories);
|
||||
var foundGuids = new HashSet<Guid>();
|
||||
|
||||
var changeType = AssetChangeType.None;
|
||||
var fireEvent = false;
|
||||
var isAsset = ext.Equals(ASSET_EXTENSION, StringComparison.Ordinal);
|
||||
var isTemp = ext.Equals(TEMP_EXTENSION, StringComparison.Ordinal);
|
||||
|
||||
switch (e.ChangeType)
|
||||
foreach (var metaPath in metaFiles)
|
||||
{
|
||||
case WatcherChangeTypes.Created:
|
||||
changeType = AssetChangeType.Created;
|
||||
if (!isAsset && !isTemp)
|
||||
{
|
||||
var handler = GetAssetHandlerForExtension(ext);
|
||||
if (handler is IImportableAssetHandler importableHandler)
|
||||
{
|
||||
var assetPath = string.Create(e.FullPath.Length - ext.Length + ASSET_EXTENSION.Length, e.FullPath, (destSpan, source) =>
|
||||
{
|
||||
source.AsSpan(0, source.Length - ext.Length).CopyTo(destSpan);
|
||||
ASSET_EXTENSION.AsSpan().CopyTo(destSpan.Slice(source.Length - ext.Length));
|
||||
});
|
||||
|
||||
var newGuid = Guid.NewGuid();
|
||||
await using var sourceStream = new FileStream(e.FullPath, FileMode.Open, FileAccess.Read);
|
||||
await using var targetStream = new FileStream(assetPath, FileMode.Create, FileAccess.Write);
|
||||
await importableHandler.ImportAsync(sourceStream, targetStream, newGuid);
|
||||
|
||||
File.Delete(assetPath);
|
||||
UpdatePathMapping(relativePath, newGuid);
|
||||
|
||||
fireEvent = true;
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case WatcherChangeTypes.Deleted:
|
||||
changeType = AssetChangeType.Deleted;
|
||||
if (isAsset)
|
||||
{
|
||||
fireEvent = RemovePathMappingByPath(relativePath);
|
||||
}
|
||||
break;
|
||||
|
||||
case WatcherChangeTypes.Changed:
|
||||
changeType = AssetChangeType.Modified;
|
||||
fireEvent = isAsset;
|
||||
break;
|
||||
case WatcherChangeTypes.All:
|
||||
// Can this even happen?
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
var meta = AssetMetaIO.ReadAsync(metaPath).AsTask().Result;
|
||||
if (meta != null)
|
||||
{
|
||||
var sourceRelative = AssetMetaIO.GetSourcePath(Path.GetRelativePath(_assetsRoot, metaPath));
|
||||
_catalog.Upsert(meta, sourceRelative.Replace('\\', '/'));
|
||||
foundGuids.Add(meta.Guid);
|
||||
}
|
||||
}
|
||||
|
||||
if (fireEvent)
|
||||
foreach (var (guid, path) in _catalog.EnumerateAll())
|
||||
{
|
||||
OnAssetChanged?.Invoke(this, new AssetChangedEventArgs(relativePath, null, changeType));
|
||||
if (!foundGuids.Contains(guid))
|
||||
{
|
||||
_catalog.Remove(guid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void OnFileSystemRenameOp(object sender, RenamedEventArgs e)
|
||||
private async void OnFileSystemEvent(object sender, FileSystemEventArgs e)
|
||||
{
|
||||
var ext = Path.GetExtension(e.FullPath);
|
||||
if (!ext.Equals(ASSET_EXTENSION, StringComparison.Ordinal))
|
||||
var relativePath = Path.GetRelativePath(_assetsRoot, e.FullPath).Replace('\\', '/');
|
||||
|
||||
if (_ignoreMetaWrites.TryRemove(e.FullPath, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var oldRelativePath = Path.GetRelativePath(_rootDirectory, e.OldFullPath);
|
||||
var newRelativePath = Path.GetRelativePath(_rootDirectory, e.FullPath);
|
||||
|
||||
if (_pathToGuid.Remove(oldRelativePath, out var guid))
|
||||
if (ext is ".tmp" or ".gtemp")
|
||||
{
|
||||
UpdatePathMapping(newRelativePath, guid);
|
||||
OnAssetChanged?.Invoke(this, new AssetChangedEventArgs(newRelativePath, oldRelativePath, AssetChangeType.Renamed));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
public string? GetAssetPath(Guid id)
|
||||
{
|
||||
lock (_pathLock)
|
||||
if (ext == ".gmeta")
|
||||
{
|
||||
if (_guidToPath.TryGetValue(id, out var path))
|
||||
if (e.ChangeType == WatcherChangeTypes.Changed || e.ChangeType == WatcherChangeTypes.Created)
|
||||
{
|
||||
return path;
|
||||
var meta = AssetMetaIO.ReadAsync(e.FullPath).AsTask().Result;
|
||||
if (meta != null)
|
||||
{
|
||||
_catalog.Upsert(meta, AssetMetaIO.GetSourcePath(relativePath));
|
||||
await _importCoordinator.EnqueueAsync(new ImportJob(meta.Guid, AssetMetaIO.GetSourcePath(relativePath), e.FullPath, ImportReason.SettingsChanged));
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (e.ChangeType == WatcherChangeTypes.Created)
|
||||
{
|
||||
await HandleNewSourceFileAsync(e.FullPath, relativePath);
|
||||
}
|
||||
else if (e.ChangeType == WatcherChangeTypes.Changed)
|
||||
{
|
||||
var guid = _catalog.GetGuid(relativePath);
|
||||
if (guid != Guid.Empty)
|
||||
{
|
||||
await _importCoordinator.EnqueueAsync(new ImportJob(guid, relativePath, AssetMetaIO.GetMetaPath(e.FullPath), ImportReason.SourceChanged));
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public Guid GetAssetGuid(string path)
|
||||
private void OnFileSystemRenameEvent(object sender, RenamedEventArgs e)
|
||||
{
|
||||
lock (_pathLock)
|
||||
var oldRelative = Path.GetRelativePath(_assetsRoot, e.OldFullPath).Replace('\\', '/');
|
||||
var newRelative = Path.GetRelativePath(_assetsRoot, e.FullPath).Replace('\\', '/');
|
||||
|
||||
var guid = _catalog.GetGuid(oldRelative);
|
||||
if (guid != Guid.Empty)
|
||||
{
|
||||
if (_pathToGuid.TryGetValue(path, out var guid))
|
||||
_catalog.Remove(guid);
|
||||
var metaFile = AssetMetaIO.GetMetaPath(e.FullPath);
|
||||
if (File.Exists(metaFile))
|
||||
{
|
||||
return guid;
|
||||
var meta = AssetMetaIO.ReadAsync(metaFile).AsTask().Result;
|
||||
if (meta != null)
|
||||
{
|
||||
_catalog.Upsert(meta, newRelative);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Guid.Empty;
|
||||
}
|
||||
|
||||
private IAssetHandler GetAssetHandler(Type type)
|
||||
private async Task HandleNewSourceFileAsync(string fullPath, string relativePath)
|
||||
{
|
||||
var typeHandle = type.TypeHandle.Value;
|
||||
if (_cachedHander.TryGetValue(typeHandle, out var handler))
|
||||
var ext = Path.GetExtension(relativePath);
|
||||
|
||||
var handler = _handlerRegistry.GetByExtension(ext);
|
||||
var importable = handler as IImportableAssetHandler;
|
||||
|
||||
var metaPath = AssetMetaIO.GetMetaPath(fullPath);
|
||||
if (File.Exists(metaPath))
|
||||
{
|
||||
return handler;
|
||||
return;
|
||||
}
|
||||
|
||||
var obj = Activator.CreateInstance(type);
|
||||
if (obj is not IAssetHandler newHandler)
|
||||
var handlerTypeId = handler?.GetType().GetCustomAttribute<CustomAssetHandlerAttribute>()?.ID;
|
||||
var meta = new AssetMeta
|
||||
{
|
||||
throw new InvalidOperationException($"Type {type.FullName} is not an IAssetHandler.");
|
||||
}
|
||||
Guid = Guid.NewGuid(),
|
||||
HandlerTypeId = handlerTypeId == null ? null : Guid.Parse(handlerTypeId),
|
||||
HandlerVersion = 1,
|
||||
Settings = importable?.CreateDefaultSettings()
|
||||
};
|
||||
|
||||
var attr = type.GetCustomAttribute<CustomAssetHandlerAttribute>(false);
|
||||
if (attr is null || attr.AllowCaching)
|
||||
{
|
||||
_cachedHander[typeHandle] = newHandler;
|
||||
}
|
||||
_ignoreMetaWrites[metaPath] = true;
|
||||
await AssetMetaIO.WriteAsync(metaPath, meta);
|
||||
|
||||
_catalog.Upsert(meta, relativePath);
|
||||
|
||||
return newHandler;
|
||||
await _importCoordinator.EnqueueAsync(new ImportJob(meta.Guid, relativePath, metaPath, ImportReason.NewAsset));
|
||||
}
|
||||
|
||||
private IAssetHandler? GetAssetHandlerForExtension(string extension)
|
||||
{
|
||||
foreach (var handlerType in AppDomain.CurrentDomain.GetAssemblies()
|
||||
.SelectMany(assembly => assembly.GetTypes())
|
||||
.Where(type => typeof(IAssetHandler).IsAssignableFrom(type) && !type.IsInterface && !type.IsAbstract))
|
||||
{
|
||||
var attr = handlerType.GetCustomAttribute<CustomAssetHandlerAttribute>(false);
|
||||
if (attr is not null && attr.SupportedExtensions.Contains(extension, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
return GetAssetHandler(handlerType);
|
||||
}
|
||||
}
|
||||
public string? GetAssetPath(Guid id) => _catalog.GetSourcePath(id);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private IAssetHandler? GetAssetHandlerForTypeId(Guid typeId)
|
||||
{
|
||||
foreach (var handlerType in AppDomain.CurrentDomain.GetAssemblies()
|
||||
.SelectMany(assembly => assembly.GetTypes())
|
||||
.Where(type => typeof(IAssetHandler).IsAssignableFrom(type) && !type.IsInterface && !type.IsAbstract))
|
||||
{
|
||||
var attr = handlerType.GetCustomAttribute<CustomAssetHandlerAttribute>(false);
|
||||
if (attr is not null && new Guid(attr.ID) == typeId)
|
||||
{
|
||||
return GetAssetHandler(handlerType);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
public Guid GetAssetGuid(string path) => _catalog.GetGuid(path.Replace('\\', '/'));
|
||||
|
||||
public async ValueTask<Result<Guid>> ImportAssetAsync(string sourceFilePath, string targetAssetPath, CancellationToken token = default)
|
||||
{
|
||||
if (!File.Exists(sourceFilePath))
|
||||
{
|
||||
return Result.Failure("Source file not found.");
|
||||
}
|
||||
// Simple copy + wait for FSW or manually trigger?
|
||||
// Current requirement: "returns the new GUID immediately (import happens in background)"
|
||||
|
||||
var ext = Path.GetExtension(sourceFilePath);
|
||||
var handler = GetAssetHandlerForExtension(ext);
|
||||
if (handler is not IImportableAssetHandler importableHandler)
|
||||
{
|
||||
return Result.Failure("No importable asset handler found for the given file extension.");
|
||||
}
|
||||
var relativePath = targetAssetPath.Replace('\\', '/');
|
||||
var fullPath = Path.Combine(_assetsRoot, relativePath);
|
||||
|
||||
var guid = Guid.NewGuid();
|
||||
var fullTargetPath = Path.GetFullPath(targetAssetPath, _rootDirectory);
|
||||
if (!await importableHandler.ImportAsync(sourceFilePath, fullTargetPath, guid, token: token))
|
||||
{
|
||||
return Result.Failure("Asset import failed.");
|
||||
}
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!);
|
||||
File.Copy(sourceFilePath, fullPath, true);
|
||||
|
||||
UpdatePathMapping(targetAssetPath, guid);
|
||||
return guid;
|
||||
// FSW will trigger but we can speed it up
|
||||
await HandleNewSourceFileAsync(fullPath, relativePath);
|
||||
|
||||
var guid = _catalog.GetGuid(relativePath);
|
||||
return Result.Success(guid);
|
||||
}
|
||||
|
||||
public async ValueTask<Result> ReimportAssetAsync(Guid assetId, string sourceFilePath, CancellationToken token = default)
|
||||
{
|
||||
var assetPath = GetAssetPath(assetId);
|
||||
if (string.IsNullOrEmpty(assetPath))
|
||||
var path = _catalog.GetSourcePath(assetId);
|
||||
if (path == null)
|
||||
{
|
||||
return Result.Failure("Asset not found in DB");
|
||||
return Result.Failure("Asset not found");
|
||||
}
|
||||
|
||||
var fullAssetPath = Path.GetFullPath(assetPath, _rootDirectory);
|
||||
|
||||
// 2. Identify the Handler
|
||||
// (You might want to store SourcePath in metadata later so you don't need to pass it here)
|
||||
var ext = Path.GetExtension(sourceFilePath);
|
||||
var handler = GetAssetHandlerForExtension(ext);
|
||||
if (handler is not IImportableAssetHandler importableHandler)
|
||||
{
|
||||
return Result.Failure("No importable asset handler found for the given file extension.");
|
||||
}
|
||||
|
||||
_ignoreFileChanges[fullAssetPath] = true;
|
||||
|
||||
await using var sourceStream = new FileStream(sourceFilePath, FileMode.Open, FileAccess.Read);
|
||||
await using var targetStream = new FileStream(fullAssetPath, FileMode.Create, FileAccess.Write);
|
||||
|
||||
await importableHandler.ImportAsync(sourceStream, targetStream, assetId, token);
|
||||
if (_loadedAssets.TryGetValue(assetId, out var weakRef) && weakRef.TryGetTarget(out var liveAsset))
|
||||
{
|
||||
await liveAsset.RefreshAsync(this, token);
|
||||
}
|
||||
var fullPath = Path.Combine(_assetsRoot, path);
|
||||
var metaPath = AssetMetaIO.GetMetaPath(fullPath);
|
||||
|
||||
await _importCoordinator.EnqueueAsync(new ImportJob(assetId, path, metaPath, ImportReason.ManualReimport), token);
|
||||
return Result.Success();
|
||||
}
|
||||
|
||||
public async ValueTask<Result<Asset>> LoadAssetAsync(Guid id, CancellationToken token = default)
|
||||
{
|
||||
// TODO: weakRef based locking instead of global lock for better concurrency.
|
||||
// We should use GetOrAdd here.
|
||||
if (_loadedAssets.TryGetValue(id, out var weakRef)
|
||||
&& weakRef.TryGetTarget(out var existingAsset))
|
||||
if (_loadedAssets.TryGetValue(id, out var weakRef) && weakRef.TryGetTarget(out var asset))
|
||||
{
|
||||
return existingAsset;
|
||||
}
|
||||
|
||||
await _cacheSlim.WaitAsync(token);
|
||||
|
||||
// Double check after acquiring the lock to make sure the assetResult wasn't loaded while waiting.
|
||||
if (_loadedAssets.TryGetValue(id, out weakRef)
|
||||
&& weakRef.TryGetTarget(out existingAsset))
|
||||
{
|
||||
return existingAsset;
|
||||
return Result.Success(asset);
|
||||
}
|
||||
|
||||
await _loadLock.WaitAsync(token);
|
||||
try
|
||||
{
|
||||
var path = GetAssetPath(id);
|
||||
if (string.IsNullOrEmpty(path))
|
||||
if (_loadedAssets.TryGetValue(id, out weakRef) && weakRef.TryGetTarget(out asset))
|
||||
{
|
||||
return null;
|
||||
return Result.Success(asset);
|
||||
}
|
||||
|
||||
var assetPath = Path.GetFullPath(path, _rootDirectory);
|
||||
await using var fs = new FileStream(assetPath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
|
||||
int sizeofGuid;
|
||||
unsafe
|
||||
var importedPath = Path.Combine(_libraryRoot, "Imports", $"{id:N}.imported");
|
||||
if (!File.Exists(importedPath))
|
||||
{
|
||||
sizeofGuid = sizeof(Guid);
|
||||
return Result.Failure<Asset>("Asset not imported");
|
||||
}
|
||||
|
||||
Span<byte> typeIdBuffer = stackalloc byte[sizeofGuid];
|
||||
fs.Seek(sizeof(int) + sizeofGuid, SeekOrigin.Begin);
|
||||
fs.ReadExactly(typeIdBuffer);
|
||||
// For now, we use a basic LoadAsync implementation.
|
||||
// In a better design, we'd read the handler ID from the header.
|
||||
// Here we we assume the catalog is correct (it's synced with gmeta).
|
||||
|
||||
var guid = Unsafe.ReadUnaligned<Guid>(ref MemoryMarshal.GetReference(typeIdBuffer));
|
||||
var handler = GetAssetHandlerForTypeId(guid);
|
||||
if (handler == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
// Looking up TypeId from catalog isn't implemented in AssetCatalog yet.
|
||||
// We should add it or use the header.
|
||||
// The existing Asset class might still be tied to the old binary format.
|
||||
|
||||
var assetResult = await handler.LoadAsync(fs, this, token);
|
||||
if (assetResult.IsFailure)
|
||||
{
|
||||
return assetResult;
|
||||
}
|
||||
|
||||
var asset = assetResult.Value;
|
||||
_loadedAssets.AddOrUpdate(id, new WeakReference<Asset>(asset), (key, oldRef) =>
|
||||
{
|
||||
// If the early return fails (find existing assetResult), it means either the assetResult haven't been loaded before, or the previous reference has been collected.
|
||||
// If the assetResult haven't been loaded before, we are in the addValue path, not here.
|
||||
// If the previous reference has been collected, we can just replace it with the new one.
|
||||
// Since we are using _cacheSlim to protect this section, we don't need check if the oldRef is still valid because only one thread can be here at a time.
|
||||
oldRef.SetTarget(asset);
|
||||
return oldRef;
|
||||
});
|
||||
|
||||
return assetResult;
|
||||
return Result.Failure<Asset>("Full asset loading would require updating all assets to the new format first.");
|
||||
}
|
||||
finally
|
||||
{
|
||||
_cacheSlim.Release();
|
||||
_loadLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask<Result> SaveAssetAsync(Asset asset, CancellationToken token = default)
|
||||
{
|
||||
var path = GetAssetPath(asset.ID);
|
||||
if (path == null)
|
||||
{
|
||||
return Result.Failure("Asset not found.");
|
||||
}
|
||||
|
||||
var handler = GetAssetHandlerForTypeId(asset.TypeID);
|
||||
if (handler == null)
|
||||
{
|
||||
return Result.Failure("No asset handler found for the given asset type.");
|
||||
}
|
||||
|
||||
var fullPath = Path.GetFullPath(path, _rootDirectory);
|
||||
await using var fs = new FileStream(fullPath, FileMode.Create, FileAccess.Write);
|
||||
return await handler.SaveAsync(asset, fs, this, token);
|
||||
}
|
||||
public ValueTask<Result> SaveAssetAsync(Asset asset, CancellationToken token = default) => throw new NotImplementedException();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_cacheSlim.Dispose();
|
||||
_watcher.Dispose();
|
||||
_importCoordinator.Dispose();
|
||||
_catalog.Dispose();
|
||||
_loadLock.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
179
src/Editor/Ghost.Editor.Core/Services/ImportCoordinator.cs
Normal file
179
src/Editor/Ghost.Editor.Core/Services/ImportCoordinator.cs
Normal file
@@ -0,0 +1,179 @@
|
||||
using System.Threading.Channels;
|
||||
using Ghost.Core;
|
||||
using Ghost.Editor.Core.AssetHandler;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace Ghost.Editor.Core.Services;
|
||||
|
||||
internal enum ImportReason
|
||||
{
|
||||
NewAsset,
|
||||
SourceChanged,
|
||||
SettingsChanged,
|
||||
HandlerUpgraded,
|
||||
ManualReimport,
|
||||
Startup,
|
||||
}
|
||||
|
||||
internal readonly record struct ImportJob(
|
||||
Guid AssetGuid,
|
||||
string SourcePath,
|
||||
string MetaPath,
|
||||
ImportReason Reason
|
||||
);
|
||||
|
||||
internal sealed class ImportCoordinator : IDisposable
|
||||
{
|
||||
private readonly Channel<ImportJob> _importChannel;
|
||||
private readonly AssetCatalog _catalog;
|
||||
private readonly AssetHandlerRegistry _handlers;
|
||||
private readonly string _assetsRoot;
|
||||
private readonly string _libraryRoot;
|
||||
private readonly CancellationTokenSource _cts;
|
||||
private readonly Task[] _workers;
|
||||
|
||||
// In a real implementation, this event would be used to notify the UI/Rest of engine
|
||||
// For now we just focus on the core logic
|
||||
// public event EventHandler<AssetChangedEventArgs>? OnAssetChanged;
|
||||
|
||||
public ImportCoordinator(AssetCatalog catalog, AssetHandlerRegistry handlers, string assetsRoot, string libraryRoot, int workerCount = 2)
|
||||
{
|
||||
_catalog = catalog;
|
||||
_handlers = handlers;
|
||||
_assetsRoot = assetsRoot;
|
||||
_libraryRoot = libraryRoot;
|
||||
_cts = new CancellationTokenSource();
|
||||
|
||||
_importChannel = Channel.CreateBounded<ImportJob>(new BoundedChannelOptions(256)
|
||||
{
|
||||
FullMode = BoundedChannelFullMode.Wait,
|
||||
SingleWriter = false,
|
||||
});
|
||||
|
||||
_workers = new Task[workerCount];
|
||||
for (var i = 0; i < workerCount; i++)
|
||||
{
|
||||
_workers[i] = Task.Run(() => WorkerLoop(_cts.Token));
|
||||
}
|
||||
}
|
||||
|
||||
public ValueTask EnqueueAsync(ImportJob job, CancellationToken token = default)
|
||||
{
|
||||
return _importChannel.Writer.WriteAsync(job, token);
|
||||
}
|
||||
|
||||
public async ValueTask EnqueueDirtyAssetsAsync(CancellationToken token = default)
|
||||
{
|
||||
foreach (var (guid, sourcePath) in _catalog.GetDirtyAssets())
|
||||
{
|
||||
var metaPath = AssetMetaIO.GetMetaPath(Path.Combine(_assetsRoot, sourcePath));
|
||||
await EnqueueAsync(new ImportJob(guid, sourcePath, metaPath, ImportReason.Startup), token);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task WorkerLoop(CancellationToken token)
|
||||
{
|
||||
await foreach (var job in _importChannel.Reader.ReadAllAsync(token))
|
||||
{
|
||||
try
|
||||
{
|
||||
await ProcessImportAsync(job, token);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_catalog.MarkFailed(job.AssetGuid, ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask ProcessImportAsync(ImportJob job, CancellationToken token)
|
||||
{
|
||||
var fullSourcePath = Path.Combine(_assetsRoot, job.SourcePath);
|
||||
var meta = await AssetMetaIO.ReadAsync(job.MetaPath, token);
|
||||
if (meta is null)
|
||||
{
|
||||
_catalog.MarkFailed(job.AssetGuid, "Missing .gmeta file");
|
||||
return;
|
||||
}
|
||||
|
||||
var handler = (meta.HandlerTypeId.HasValue)
|
||||
? _handlers.GetByTypeId(meta.HandlerTypeId.Value)
|
||||
: _handlers.GetByExtension(Path.GetExtension(job.SourcePath));
|
||||
|
||||
var contentHash = await ComputeFileHashAsync(fullSourcePath, token);
|
||||
var settingsHash = ComputeSettingsHash(meta.Settings);
|
||||
|
||||
// Check if we can skip (if not a manual reimport)
|
||||
if (job.Reason != ImportReason.ManualReimport &&
|
||||
meta.ContentHash == contentHash &&
|
||||
meta.SettingsHash == settingsHash &&
|
||||
meta.HandlerVersion == _handlers.GetVersionByTypeId(meta.HandlerTypeId ?? Guid.Empty))
|
||||
{
|
||||
_catalog.MarkImported(job.AssetGuid, contentHash, settingsHash);
|
||||
return;
|
||||
}
|
||||
|
||||
var importResult = Result.Success();
|
||||
if (handler is IImportableAssetHandler importable)
|
||||
{
|
||||
// TODO: This should be handled by EditorApplication.
|
||||
var importsDir = Path.Combine(_libraryRoot, "Imports");
|
||||
Directory.CreateDirectory(importsDir);
|
||||
var targetPath = Path.Combine(importsDir, $"{job.AssetGuid:N}.imported");
|
||||
|
||||
await using var sourceStream = new FileStream(fullSourcePath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
await using var targetStream = new FileStream(targetPath, FileMode.Create, FileAccess.Write, FileShare.None);
|
||||
|
||||
importResult = await importable.ImportAsync(sourceStream, targetStream, job.AssetGuid, meta.Settings, token);
|
||||
}
|
||||
|
||||
if (importResult.IsSuccess)
|
||||
{
|
||||
meta.ContentHash = contentHash;
|
||||
meta.SettingsHash = settingsHash;
|
||||
meta.HandlerVersion = _handlers.GetVersionByTypeId(meta.HandlerTypeId ?? Guid.Empty);
|
||||
meta.LastImportedUtc = DateTime.UtcNow;
|
||||
|
||||
await AssetMetaIO.WriteAsync(job.MetaPath, meta, token);
|
||||
_catalog.MarkImported(job.AssetGuid, contentHash, settingsHash);
|
||||
}
|
||||
else
|
||||
{
|
||||
_catalog.MarkFailed(job.AssetGuid, importResult.Message ?? "Unknown import error");
|
||||
}
|
||||
}
|
||||
|
||||
private static async ValueTask<string> ComputeFileHashAsync(string filePath, CancellationToken token)
|
||||
{
|
||||
if (!File.Exists(filePath))
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
using var sha = SHA256.Create();
|
||||
await using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
var hash = await sha.ComputeHashAsync(stream, token);
|
||||
return Convert.ToHexString(hash);
|
||||
}
|
||||
|
||||
private static string ComputeSettingsHash(IAssetSettings? settings)
|
||||
{
|
||||
if (settings is null)
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
var json = JsonSerializer.SerializeToUtf8Bytes(settings);
|
||||
using var sha = SHA256.Create();
|
||||
var hash = sha.ComputeHash(json);
|
||||
return Convert.ToHexString(hash);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_importChannel.Writer.TryComplete();
|
||||
_cts.Cancel();
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -62,7 +62,6 @@ internal static class ActivationHandler
|
||||
};
|
||||
|
||||
AllocationManager.Initialize(opts);
|
||||
TypeCache.Initialize();
|
||||
|
||||
//App.GetService<EngineCore>();
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ using Ghost.Core;
|
||||
using Ghost.Editor.Core;
|
||||
using Ghost.Editor.Core.Contracts;
|
||||
using Ghost.Editor.Core.Services;
|
||||
using Ghost.Editor.Core.Utilities;
|
||||
using Ghost.Editor.View.Pages.EngineEditor;
|
||||
using Ghost.Editor.View.Windows;
|
||||
using Ghost.Editor.ViewModels.Controls;
|
||||
@@ -52,6 +53,8 @@ public partial class App : Application
|
||||
{
|
||||
InitializeComponent();
|
||||
|
||||
TypeCache.Initialize();
|
||||
|
||||
Host = Microsoft.Extensions.Hosting.Host.
|
||||
CreateDefaultBuilder().
|
||||
UseContentRoot(AppContext.BaseDirectory).
|
||||
@@ -69,6 +72,31 @@ public partial class App : Application
|
||||
|
||||
services.AddTransient<ProjectBrowserViewModel>();
|
||||
|
||||
foreach (var type in TypeCache.GetTypes())
|
||||
{
|
||||
var data = type.GetCustomAttributesData().FirstOrDefault(a => a.AttributeType == typeof(EditorInjectionAttribute));
|
||||
if (data is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var lifeTime = (EditorInjectionAttribute.ServiceLifetime)data.ConstructorArguments[0].Value!;
|
||||
var implementationType = (Type)data.ConstructorArguments[1].Value!;
|
||||
var serviceType = type.IsInterface ? type.AsType() : implementationType;
|
||||
|
||||
switch (lifeTime)
|
||||
{
|
||||
case EditorInjectionAttribute.ServiceLifetime.Singleton:
|
||||
services.AddSingleton(serviceType, implementationType);
|
||||
break;
|
||||
case EditorInjectionAttribute.ServiceLifetime.Transient:
|
||||
services.AddTransient(serviceType, implementationType);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
#region Should be deleted
|
||||
services.AddTransient<ScenePage>();
|
||||
|
||||
@@ -119,7 +147,6 @@ public partial class App : Application
|
||||
try
|
||||
{
|
||||
EditorApplication.Initialize(Host.Services, arguments.ProjectPath, arguments.ProjectName);
|
||||
|
||||
// NOTE: We must call DispatcherQueue.GetForCurrentThread() on the UI thread before any await.
|
||||
EditorApplication.SetDispatcherQueue(DispatcherQueue.GetForCurrentThread());
|
||||
|
||||
|
||||
Reference in New Issue
Block a user