From 9cee32aa832b0f57fe24bcd3eed79c9047f55efe Mon Sep 17 00:00:00 2001 From: Misaki Date: Tue, 17 Mar 2026 20:58:31 +0900 Subject: [PATCH] feat(allocator): add per-thread caches to FreeList Refactored FreeList allocator to use per-thread caches for improved scalability and performance, with configurable max concurrency and overflow cache. AllocationManager debug layer is now compile-time via ENABLE_DEBUG_LAYER. MemoryUtility methods no longer catch exceptions. Argument validation standardized with ThrowIfNegative. JobScheduler passes maxConcurrencyLevel to allocator. CollectionUtility's GetElementUnsafe returns mutable ref. AssemblyVersion incremented. Added comprehensive FreeList unit tests. Improved robustness and error handling in allocation classes. BREAKING CHANGE: Debug layer APIs removed; FreeList allocator interface changed for thread cache support. --- Misaki.HighPerformance.Jobs/JobScheduler.cs | 5 +- .../Misaki.HighPerformance.Jobs.csproj | 2 +- .../Buffer/AllocationManager.cs | 152 ++-- .../Buffer/AllocationOption.cs | 6 +- .../Buffer/Arena.cs | 4 +- .../Buffer/DynamicArena.cs | 2 +- .../Buffer/FreeList.cs | 812 +++++++++++------- .../Buffer/Stack.cs | 13 +- .../Collections/HashMapHelper.cs | 12 +- .../Collections/UnsafeArray.cs | 6 +- .../Misaki.HighPerformance.LowLevel.csproj | 2 +- .../Utilities/MemoryUtility.cs | 79 +- .../UnitTest/Buffer/TestAllocationManager.cs | 6 - .../UnitTest/Buffer/TestFreeList.cs | 151 ++++ .../Utilities/CollectionUtility.cs | 2 +- 15 files changed, 772 insertions(+), 482 deletions(-) create mode 100644 Misaki.HighPerformance.Test/UnitTest/Buffer/TestFreeList.cs diff --git a/Misaki.HighPerformance.Jobs/JobScheduler.cs b/Misaki.HighPerformance.Jobs/JobScheduler.cs index 5728e96..abe7f20 100644 --- a/Misaki.HighPerformance.Jobs/JobScheduler.cs +++ b/Misaki.HighPerformance.Jobs/JobScheduler.cs @@ -270,14 +270,15 @@ public sealed unsafe partial class JobScheduler : IJobScheduler, IDisposable /// The number of worker threads to create. If less than 1, at least one thread will be created. public JobScheduler(int threadCount) { - _jobDataAllocator = new(8); + var workerCount = Math.Max(1, threadCount); + + _jobDataAllocator = new(8, maxConcurrencyLevel: workerCount + 1); _jobInfoPool = new(); _jobQueue = new(); _workSignal = new(0); _cts = new(); - var workerCount = Math.Max(1, threadCount); _workerThreads = new WorkerThread[workerCount]; for (var i = 0; i < workerCount; i++) diff --git a/Misaki.HighPerformance.Jobs/Misaki.HighPerformance.Jobs.csproj b/Misaki.HighPerformance.Jobs/Misaki.HighPerformance.Jobs.csproj index c6548ad..69a77b5 100644 --- a/Misaki.HighPerformance.Jobs/Misaki.HighPerformance.Jobs.csproj +++ b/Misaki.HighPerformance.Jobs/Misaki.HighPerformance.Jobs.csproj @@ -6,7 +6,7 @@ enable True true - 1.5.1 + 1.5.2 $(AssemblyVersion) Misaki https://git.personalnas.com/Misaki/Misaki.HighPerformance.git diff --git a/Misaki.HighPerformance.LowLevel/Buffer/AllocationManager.cs b/Misaki.HighPerformance.LowLevel/Buffer/AllocationManager.cs index 3b49915..da426ae 100644 --- a/Misaki.HighPerformance.LowLevel/Buffer/AllocationManager.cs +++ b/Misaki.HighPerformance.LowLevel/Buffer/AllocationManager.cs @@ -33,16 +33,17 @@ public readonly struct AllocationInfo /// public static unsafe class AllocationManager { - // === Intrusive allocation tracking (enabled when debug layer is on) === +#if ENABLE_DEBUG_LAYER [StructLayout(LayoutKind.Sequential)] private struct AllocationHeader { public AllocationHeader* prev; public AllocationHeader* next; - public void* basePtr; // pointer returned by underlying allocator - public nuint userSize; // requested size from the user - public GCHandle stackHandle; // GCHandle to managed StackTrace + public void* basePtr; // pointer returned by underlying allocator + public nuint userSize; // requested size from the user + public GCHandle stackHandle; // GCHandle to managed StackTrace } +#endif private struct ArenaAllocator : IAllocator, IDisposable { @@ -270,11 +271,12 @@ public static unsafe class AllocationManager private static readonly HeapAllocator* s_pHeapAllocator; private static readonly StackAllocator* s_pStackAllocator; - private static bool s_debugLayer; private static bool s_disposed; - private static AllocationHeader* s_pLiveHead; +#if ENABLE_DEBUG_LAYER private static SpinLock s_liveLock; + private static AllocationHeader* s_pLiveHead; +#endif private static readonly ConcurrentSlotMap s_allocations; @@ -285,31 +287,28 @@ public static unsafe class AllocationManager /// public static int LiveAllocationCount => s_allocations.Count; - /// - /// Gets a value indicating whether the debug layer is currently enabled. - /// - public static bool IsDebugLayerEnabled => s_debugLayer; - - static AllocationManager() { var allocatorTotalSize = (nuint)(sizeof(ArenaAllocator) + sizeof(HeapAllocator) + sizeof(StackAllocator)); - var basePtr = NativeMemory.Alloc(allocatorTotalSize); + var basePtr = Malloc(allocatorTotalSize); + s_pArenaAllocator = (ArenaAllocator*)basePtr; s_pHeapAllocator = (HeapAllocator*)((byte*)basePtr + (nuint)sizeof(ArenaAllocator)); s_pStackAllocator = (StackAllocator*)((byte*)basePtr + (nuint)(sizeof(ArenaAllocator) + sizeof(HeapAllocator))); +#if ENABLE_DEBUG_LAYER s_liveLock = new SpinLock(false); + s_pLiveHead = null; +#endif s_allocations = new ConcurrentSlotMap(256); s_pArenaAllocator->Init(_DEFAULT_MEMORY_POOL_SIZE); s_pHeapAllocator->Init(); s_pStackAllocator->Init(); - - s_pLiveHead = null; } +#if ENABLE_DEBUG_LAYER [MethodImpl(MethodImplOptions.AggressiveInlining)] private static byte* AlignUp(byte* p, nuint alignment) { @@ -462,22 +461,7 @@ public static unsafe class AllocationManager return newUser; } - - /// - /// Enables the debug layer, allowing additional diagnostic information to be collected. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void EnableDebugLayer() - { - // To avoid ambiguity between pointers allocated before/after enabling, this must be called - // before any heap allocations are live. - if (s_allocations.Count != 0) - { - throw new InvalidOperationException("EnableDebugLayer must be called before any allocations are active."); - } - - s_debugLayer = true; - } +#endif /// /// Gets a reference to the allocation pHandle for the specified allocator type. @@ -499,10 +483,6 @@ public static unsafe class AllocationManager /// /// Allocates a block of memory from the heap with the specified size and alignment, using the given allocation options. /// - /// - /// This will allocate memory from the heap. If the debug layer is enabled, additional tracking information will be recorded. - /// The memory handle is always tracked unless the flag is specified. - /// /// The number of bytes to allocate. Must be greater than zero. /// The alignment, in bytes, for the allocated memory block. Must be a power of two. /// An optional set of flags that control allocation behavior, such as whether the memory should be cleared or @@ -511,17 +491,11 @@ public static unsafe class AllocationManager /// Thrown if the allocation fails. public static void* HeapAlloc(nuint size, nuint alignment, AllocationOption allocationOption, MemoryHandle* pHandle) { - var isUntrack = allocationOption.HasFlag(AllocationOption.Untrack); - - void* ptr; - if (s_debugLayer && !isUntrack) - { - ptr = DebugAllocate(size, alignment); - } - else - { - ptr = AlignedAlloc(size, alignment); - } +#if ENABLE_DEBUG_LAYER + var ptr = DebugAllocate(size, alignment); +#else + var ptr = AlignedAlloc(size, alignment); +#endif if (ptr == null) { @@ -534,15 +508,7 @@ public static unsafe class AllocationManager MemClear(ptr, size); } - if (isUntrack) - { - *pHandle = MagicHandle; - } - else - { - *pHandle = AddAllocation((IntPtr)ptr); - } - + *pHandle = AddAllocation((IntPtr)ptr); return ptr; } @@ -554,11 +520,13 @@ public static unsafe class AllocationManager /// The handle representing the memory allocation to free. The handle must be valid and previously allocated. public static void HeapFree(void* ptr, MemoryHandle handle) { - if (s_debugLayer && handle != MagicHandle) +#if ENABLE_DEBUG_LAYER + if (handle != MagicHandle) { DebugFree(ptr); } else +#endif { AlignedFree(ptr); } @@ -662,53 +630,53 @@ public static unsafe class AllocationManager return; } +#if ENABLE_DEBUG_LAYER // In debug mode, walk the intrusive list to surface any leaks. - if (s_debugLayer) + var snapshot = new List(); + var taken = false; + try { - var snapshot = new List(); - var taken = false; - try + s_liveLock.Enter(ref taken); + if (s_pLiveHead != null) { - s_liveLock.Enter(ref taken); - if (s_pLiveHead != null) + snapshot.Capacity = 128; + for (var p = s_pLiveHead; p != null; p = p->next) { - snapshot.Capacity = 128; - for (var p = s_pLiveHead; p != null; p = p->next) + var trace = (StackTrace)HeaderGetHandle(p).Target!; + snapshot.Add(new AllocationInfo { - var trace = (StackTrace)HeaderGetHandle(p).Target!; - snapshot.Add(new AllocationInfo - { - Size = p->userSize, - StackTrace = trace - }); - } + Size = p->userSize, + StackTrace = trace + }); } } - finally - { - if (taken) - { - s_liveLock.Exit(); - } - } - - nuint unfreeBytes = 0u; - foreach (var info in snapshot) - { - unfreeBytes += info.Size; - } - - if (unfreeBytes > 0u) - { - throw new MemoryLeakException(CollectionsMarshal.AsSpan(snapshot)); - } - - Debug.Assert(LiveAllocationCount == 0); } - else if (LiveAllocationCount != 0) + finally + { + if (taken) + { + s_liveLock.Exit(); + } + } + + nuint unfreeBytes = 0u; + foreach (var info in snapshot) + { + unfreeBytes += info.Size; + } + + if (unfreeBytes > 0u) + { + throw new MemoryLeakException(CollectionsMarshal.AsSpan(snapshot)); + } + + Debug.Assert(LiveAllocationCount == 0); +#else + if (LiveAllocationCount != 0) { throw new MemoryLeakException($"Found {LiveAllocationCount} memory lakes! Please enable debug layer for more informations."); } +#endif // NOTE: Arena allocator holds the base ptr for all allocators, heap and stack allocators do not own any memory themselves. if (s_pArenaAllocator != null) diff --git a/Misaki.HighPerformance.LowLevel/Buffer/AllocationOption.cs b/Misaki.HighPerformance.LowLevel/Buffer/AllocationOption.cs index f02bd22..3acf6b4 100644 --- a/Misaki.HighPerformance.LowLevel/Buffer/AllocationOption.cs +++ b/Misaki.HighPerformance.LowLevel/Buffer/AllocationOption.cs @@ -10,11 +10,7 @@ public enum AllocationOption : byte /// /// Clear the memory to zero upon allocation. /// - Clear = 1 << 0, - /// - /// Specify that this memory allocation should not been tracked competly, which will not perform any safty check like use after free and leack detection. - /// - Untrack = 1 << 1, + Clear = 1 << 0 } public enum Allocator : byte diff --git a/Misaki.HighPerformance.LowLevel/Buffer/Arena.cs b/Misaki.HighPerformance.LowLevel/Buffer/Arena.cs index cfe80c4..633f056 100644 --- a/Misaki.HighPerformance.LowLevel/Buffer/Arena.cs +++ b/Misaki.HighPerformance.LowLevel/Buffer/Arena.cs @@ -1,4 +1,4 @@ -using System.Runtime.InteropServices; +using System.Runtime.InteropServices; namespace Misaki.HighPerformance.LowLevel.Buffer; @@ -17,6 +17,8 @@ public unsafe struct Arena : IDisposable public Arena(nuint size) { + ArgumentOutOfRangeException.ThrowIfNegative(size); + if (_buffer != null) { return; diff --git a/Misaki.HighPerformance.LowLevel/Buffer/DynamicArena.cs b/Misaki.HighPerformance.LowLevel/Buffer/DynamicArena.cs index 43706bc..8fc8c45 100644 --- a/Misaki.HighPerformance.LowLevel/Buffer/DynamicArena.cs +++ b/Misaki.HighPerformance.LowLevel/Buffer/DynamicArena.cs @@ -1,4 +1,4 @@ -using System.Runtime.InteropServices; +using System.Runtime.InteropServices; namespace Misaki.HighPerformance.LowLevel.Buffer; diff --git a/Misaki.HighPerformance.LowLevel/Buffer/FreeList.cs b/Misaki.HighPerformance.LowLevel/Buffer/FreeList.cs index aca13f6..2ae8c32 100644 --- a/Misaki.HighPerformance.LowLevel/Buffer/FreeList.cs +++ b/Misaki.HighPerformance.LowLevel/Buffer/FreeList.cs @@ -4,17 +4,19 @@ using System.Runtime.InteropServices; namespace Misaki.HighPerformance.LowLevel.Buffer; /// -/// A lock-free, thread-safe variable-size allocator that manages memory blocks of different sizes. -/// Optimized for high-performance scenarios with frequent allocations and deallocations. +/// A thread-safe variable-size allocator that uses per-thread caches for the hot path and +/// a remote-free queue for cross-thread deallocation. /// -[StructLayout(LayoutKind.Explicit, Size = 256)] // Cache line aligned to prevent false sharing +[StructLayout(LayoutKind.Sequential)] public unsafe struct FreeList : IDisposable { [StructLayout(LayoutKind.Sequential)] private struct FreeNode { public FreeNode* next; - public nuint size; + public MemoryChunk* ownerChunk; + public nuint blockSize; + public int bucketIndex; } [StructLayout(LayoutKind.Sequential)] @@ -23,81 +25,75 @@ public unsafe struct FreeList : IDisposable public MemoryChunk* next; public byte* memory; public nuint size; - public nuint used; // Amount of memory used in this chunk + public nuint used; } [StructLayout(LayoutKind.Explicit, Size = 32)] private struct SizeBucket { [FieldOffset(0)] - public long freeCount; // Number of free blocks + public long freeCount; [FieldOffset(8)] - public nint freeHead; // Free list head for this size + public nint freeHead; [FieldOffset(16)] - public nuint blockSize; // Fixed size for this bucket + public nuint blockSize; [FieldOffset(24)] public int creationLock; } - [StructLayout(LayoutKind.Explicit, Size = 24)] + [StructLayout(LayoutKind.Sequential)] + private struct ThreadCache + { + public fixed byte buckets[_MAX_BUCKETS * 32]; + public nint remoteFreeHead; + public int threadId; + public int active; + } + + [StructLayout(LayoutKind.Explicit, Size = 32)] private struct BlockHeader { - // Ensure the size is fixed across x86 and x64 [FieldOffset(0)] public MemoryChunk* ownerChunk; [FieldOffset(8)] public nuint blockSize; [FieldOffset(16)] public ulong magicNumber; + [FieldOffset(24)] + public int ownerCacheIndex; } - private const int _MAX_BUCKETS = 16; // Number of size buckets - private const nuint _MIN_BLOCK_SIZE = 16; // Minimum block size - private const nuint _DEFAULT_CHUNK_SIZE = 64 * 1024; // 64KB chunks - private const ulong _MAGIC_NUMBER = 0xDEADBEEFDEADBEEF; // For validating blocks + private const int _MAX_BUCKETS = 16; + private const int _DEFAULT_MAX_CONCURRENCY_LEVEL = 16; + private const int _OVERFLOW_CACHE_INDEX = 0; + private const nuint _MIN_BLOCK_SIZE = 16; + private const nuint _DEFAULT_CHUNK_SIZE = 64 * 1024; + private const ulong _MAGIC_NUMBER = 0xDEADBEEFDEADBEEF; - [FieldOffset(0)] - private fixed byte _buckets[_MAX_BUCKETS * 32]; // SizeBucket array (32 bytes per bucket) + [ThreadStatic] + private static int t_cacheIndex; - [FieldOffset(512)] - private DynamicArena _chunkArena; // 128 + [ThreadStatic] + private static void* t_ownerId; - [FieldOffset(640)] - private MemoryChunk* _chunks; // 8 - - [FieldOffset(648)] - private readonly nuint _chunkSize; // 8 - - [FieldOffset(656)] - private readonly nuint _alignment; // 8 - - [FieldOffset(664)] - private long _totalAllocatedBytes; // 8 - - [FieldOffset(672)] - private long _totalFreeBytes; // 8 - - [FieldOffset(676)] - private volatile int _disposed; // 4 - - [FieldOffset(680)] - private volatile int _chunkCreationLock; // 4 + private void* _instanceId; + private ThreadCache** _caches; + private DynamicArena _chunkArena; + private MemoryChunk* _chunks; + private readonly nuint _chunkSize; + private readonly nuint _alignment; + private readonly int _maxConcurrencyLevel; + private int _cacheCount; + private volatile int _disposed; + private volatile int _chunkCreationLock; + private volatile int _cacheRegistrationLock; + private volatile int _overflowLock; /// /// Gets the alignment requirement for allocations. /// public readonly nuint Alignment => _alignment; - /// - /// Gets the total number of allocated bytes. - /// - public readonly long TotalAllocatedBytes => Interlocked.Read(ref Unsafe.AsRef(in _totalAllocatedBytes)); - - /// - /// Gets the total number of free bytes available. - /// - public readonly long TotalFreeBytes => Interlocked.Read(ref Unsafe.AsRef(in _totalFreeBytes)); - /// /// Gets whether the allocator has been disposed. /// @@ -108,12 +104,18 @@ public unsafe struct FreeList : IDisposable /// public readonly nuint ChunkSize => _chunkSize; + /// + /// Gets the maximum number of dedicated thread caches. + /// + public readonly int MaxConcurrencyLevel => _maxConcurrencyLevel; + /// /// Initializes a new variable-size FreeList allocator with the specified parameters. /// /// Alignment requirement for blocks (must be power of 2). /// Size of memory chunks to allocate (default: 64KB). - public FreeList(nuint alignment, nuint chunkSize = _DEFAULT_CHUNK_SIZE) + /// Maximum number of dedicated thread caches. + public FreeList(nuint alignment, nuint chunkSize = _DEFAULT_CHUNK_SIZE, int maxConcurrencyLevel = _DEFAULT_MAX_CONCURRENCY_LEVEL) { if (alignment == 0 || (alignment & (alignment - 1)) != 0) { @@ -125,21 +127,72 @@ public unsafe struct FreeList : IDisposable throw new ArgumentException("Chunk size must be at least 1KB", nameof(chunkSize)); } + if (maxConcurrencyLevel < 1) + { + throw new ArgumentOutOfRangeException(nameof(maxConcurrencyLevel), "Max concurrency level must be greater than zero."); + } + _alignment = alignment; _chunkSize = chunkSize; - _chunks = null; - _totalAllocatedBytes = 0; - _totalFreeBytes = 0; - _disposed = 0; - _chunkCreationLock = 0; + _maxConcurrencyLevel = maxConcurrencyLevel; - _chunkArena = new DynamicArena(1024); - InitializeBuckets(); + try + { + _instanceId = Malloc((nuint)sizeof(nint)); + + _chunks = null; + _cacheCount = 0; + _disposed = 0; + _chunkCreationLock = 0; + _cacheRegistrationLock = 0; + _overflowLock = 0; + _chunkArena = new DynamicArena(1024); + _caches = (ThreadCache**)Malloc((nuint)sizeof(ThreadCache*) * (nuint)(maxConcurrencyLevel + 1)); + + for (var i = 0; i <= maxConcurrencyLevel; i++) + { + _caches[i] = null; + } + + var overflowCache = CreateCacheForThread(0); + if (overflowCache == null) + { + throw new OutOfMemoryException("Failed to initialize free list overflow cache."); + } + + _caches[_OVERFLOW_CACHE_INDEX] = overflowCache; + + } + catch + { + if (_instanceId != null) + { + Free(_instanceId); + _instanceId = null; + } + + if (_caches != null) + { + Free(_caches); + _caches = null; + } + + _chunkArena.Dispose(); + + throw; + } } - private readonly void InitializeBuckets() + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static SizeBucket* GetBuckets(ThreadCache* cache) { - var buckets = GetBuckets(); + return (SizeBucket*)cache->buckets; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InitializeBuckets(ThreadCache* cache) + { + var buckets = GetBuckets(cache); var size = _MIN_BLOCK_SIZE; for (var i = 0; i < _MAX_BUCKETS; i++) @@ -147,42 +200,296 @@ public unsafe struct FreeList : IDisposable buckets[i].blockSize = size; buckets[i].freeHead = 0; buckets[i].freeCount = 0; - size *= 2; // Exponential size increase + buckets[i].creationLock = 0; + size *= 2; } - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private readonly SizeBucket* GetBuckets() - { - fixed (byte* ptr = _buckets) - { - return (SizeBucket*)ptr; - } + cache->remoteFreeHead = 0; } [MethodImpl(MethodImplOptions.AggressiveInlining)] private readonly int FindBucket(nuint size) { - var buckets = GetBuckets(); - + var blockSize = _MIN_BLOCK_SIZE; for (var i = 0; i < _MAX_BUCKETS; i++) { - if (size <= buckets[i].blockSize) + if (size <= blockSize) { return i; } + + blockSize <<= 1; } - return -1; // Size too large for buckets + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private ThreadCache* CreateCacheForThread(int threadId) + { + var cache = (ThreadCache*)_chunkArena.Allocate(SizeOf(), AlignOf(), AllocationOption.Clear); + if (cache == null) + { + return null; + } + + InitializeBuckets(cache); + cache->threadId = threadId; + cache->active = 1; + return cache; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void DrainRemoteFrees(ThreadCache* cache) + { + var head = (FreeNode*)Interlocked.Exchange(ref cache->remoteFreeHead, 0); + while (head != null) + { + var next = head->next; + PushToBucket(cache, head->bucketIndex, head, head->ownerChunk, head->blockSize); + head = next; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private readonly ThreadCache* GetOverflowCache() + { + return _caches[_OVERFLOW_CACHE_INDEX]; + } + + private ThreadCache* RegisterThreadCache() + { + while (Interlocked.CompareExchange(ref _cacheRegistrationLock, 1, 0) != 0) + { + Thread.SpinWait(1); + } + + try + { + if (t_ownerId == _instanceId && t_cacheIndex > 0 && t_cacheIndex <= _cacheCount) + { + return _caches[t_cacheIndex]; + } + + if (_cacheCount >= _maxConcurrencyLevel) + { + t_ownerId = _instanceId; + t_cacheIndex = _OVERFLOW_CACHE_INDEX; + return GetOverflowCache(); + } + + var threadId = Environment.CurrentManagedThreadId; + var cache = CreateCacheForThread(threadId); + if (cache == null) + { + t_ownerId = _instanceId; + t_cacheIndex = _OVERFLOW_CACHE_INDEX; + return GetOverflowCache(); + } + + _cacheCount++; + _caches[_cacheCount] = cache; + + t_ownerId = _instanceId; + t_cacheIndex = _cacheCount; + return cache; + } + finally + { + Interlocked.Exchange(ref _cacheRegistrationLock, 0); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private ThreadCache* GetCurrentCache() + { + if (t_ownerId == _instanceId) + { + var index = t_cacheIndex; + if ((uint)index <= (uint)_cacheCount) + { + var cache = _caches[index]; + if (cache != null) + { + return cache; + } + } + } + + return RegisterThreadCache(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void* TryPopFromBucket(ThreadCache* cache, int cacheIndex, int bucketIndex) + { + var buckets = GetBuckets(cache); + var bucket = &buckets[bucketIndex]; + var head = (FreeNode*)bucket->freeHead; + if (head == null) + { + return null; + } + + bucket->freeHead = (nint)head->next; + bucket->freeCount--; + + AssignBlockHeader((BlockHeader*)head, head->ownerChunk, head->blockSize, cacheIndex); + return head; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void PushToBucket(ThreadCache* cache, int bucketIndex, void* ptr, MemoryChunk* ownerChunk, nuint blockSize) + { + var buckets = GetBuckets(cache); + var bucket = &buckets[bucketIndex]; + var node = (FreeNode*)ptr; + node->ownerChunk = ownerChunk; + node->blockSize = blockSize; + node->bucketIndex = bucketIndex; + node->next = (FreeNode*)bucket->freeHead; + bucket->freeHead = (nint)node; + bucket->freeCount++; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void AssignBlockHeader(BlockHeader* header, MemoryChunk* ownerChunk, nuint blockSize, int ownerCacheIndex) + { + header->ownerChunk = ownerChunk; + header->blockSize = blockSize; + header->magicNumber = _MAGIC_NUMBER; + header->ownerCacheIndex = ownerCacheIndex; + } + + private bool TryCreateBlocksForBucket(ThreadCache* cache, int cacheIndex, int bucketIndex) + { + var buckets = GetBuckets(cache); + var bucket = &buckets[bucketIndex]; + + while (Interlocked.CompareExchange(ref bucket->creationLock, 1, 0) != 0) + { + Thread.SpinWait(1); + } + + try + { + DrainRemoteFrees(cache); + if (bucket->freeHead != 0) + { + return true; + } + + var blockSize = bucket->blockSize; + var blocksToCreate = Math.Min(_chunkSize / blockSize, 256); + if (blocksToCreate == 0) + { + return false; + } + + var totalSize = blocksToCreate * blockSize; + var memory = (byte*)AlignedAlloc(totalSize, _alignment); + if (memory == null) + { + return false; + } + + var chunk = (MemoryChunk*)_chunkArena.Allocate(SizeOf(), AlignOf(), AllocationOption.None); + if (chunk == null) + { + AlignedFree(memory); + return false; + } + + while (Interlocked.CompareExchange(ref _chunkCreationLock, 1, 0) != 0) + { + Thread.SpinWait(1); + } + + try + { + chunk->memory = memory; + chunk->size = totalSize; + chunk->used = totalSize; + chunk->next = _chunks; + _chunks = chunk; + } + finally + { + Interlocked.Exchange(ref _chunkCreationLock, 0); + } + + for (nuint i = 0; i < blocksToCreate; i++) + { + var blockStartPtr = memory + (i * blockSize); + PushToBucket(cache, bucketIndex, blockStartPtr, chunk, blockSize); + } + + return true; + } + finally + { + Interlocked.Exchange(ref bucket->creationLock, 0); + } + } + + private void* AllocateFromChunk(int cacheIndex, nuint size, nuint alignment) + { + while (Interlocked.CompareExchange(ref _chunkCreationLock, 1, 0) != 0) + { + Thread.SpinWait(1); + } + + try + { + var chunk = _chunks; + while (chunk != null) + { + var alignedOffset = (chunk->used + alignment - 1) & ~(alignment - 1); + var totalNeeded = alignedOffset - chunk->used + size; + var available = chunk->size - chunk->used; + + if (totalNeeded <= available) + { + var blockStartPtr = chunk->memory + alignedOffset; + chunk->used = alignedOffset + size; + AssignBlockHeader((BlockHeader*)blockStartPtr, chunk, size, cacheIndex); + return blockStartPtr; + } + + chunk = chunk->next; + } + + var newChunkSize = Math.Max(_chunkSize, size + alignment); + var newMemory = (byte*)AlignedAlloc(newChunkSize, alignment); + if (newMemory == null) + { + return null; + } + + var newChunk = (MemoryChunk*)_chunkArena.Allocate(SizeOf(), AlignOf(), AllocationOption.None); + if (newChunk == null) + { + AlignedFree(newMemory); + return null; + } + + newChunk->memory = newMemory; + newChunk->size = newChunkSize; + newChunk->used = size; + newChunk->next = _chunks; + _chunks = newChunk; + + AssignBlockHeader((BlockHeader*)newMemory, newChunk, size, cacheIndex); + return newMemory; + } + finally + { + Interlocked.Exchange(ref _chunkCreationLock, 0); + } } /// - /// Allocates a memory block of the specified size. Thread-safe using lock-free algorithms. + /// Allocates a memory block of the specified size. /// - /// Size of memory to allocate in bytes. - /// Alignment requirement (0 = use default). - /// Options for allocation (e.g., clear memory). - /// MemoryBlock containing allocated memory, or Invalid if allocation fails. [MethodImpl(MethodImplOptions.AggressiveInlining)] public void* Allocate(nuint size, nuint alignment, AllocationOption allocationOption = AllocationOption.None) { @@ -206,57 +513,69 @@ public unsafe struct FreeList : IDisposable throw new ArgumentException("Alignment must be a power of two.", nameof(alignment)); } - // Align size to alignment boundary var alignedSize = (size + alignment - 1) & ~(alignment - 1); alignedSize = Math.Max(alignedSize, _MIN_BLOCK_SIZE); var totalSize = alignedSize + (nuint)sizeof(BlockHeader); - var bucketIndex = FindBucket(totalSize); - void* ptr = null; + var cache = GetCurrentCache(); + var cacheIndex = t_cacheIndex; + var requiresOverflowLock = cacheIndex == _OVERFLOW_CACHE_INDEX; - if (bucketIndex >= 0) + if (requiresOverflowLock) { - // Try to allocate from bucket - ptr = TryPopFromBucket(bucketIndex); + while (Interlocked.CompareExchange(ref _overflowLock, 1, 0) != 0) + { + Thread.SpinWait(1); + } + } + + try + { + DrainRemoteFrees(cache); + + void* ptr = null; + if (bucketIndex >= 0) + { + ptr = TryPopFromBucket(cache, cacheIndex, bucketIndex); + if (ptr == null && TryCreateBlocksForBucket(cache, cacheIndex, bucketIndex)) + { + ptr = TryPopFromBucket(cache, cacheIndex, bucketIndex); + } + } if (ptr == null) { - // Create new blocks for this bucket - if (TryCreateBlocksForBucket(bucketIndex)) - { - ptr = TryPopFromBucket(bucketIndex); - } + ptr = AllocateFromChunk(cacheIndex, totalSize, alignment); + } + if (ptr == null) + { + return null; } - } - if (ptr == null) - { - // Fallback to direct allocation from chunk - ptr = AllocateFromChunk(totalSize, alignment); - } - - if (ptr != null) - { var header = (BlockHeader*)ptr; - Interlocked.Add(ref _totalAllocatedBytes, (long)header->blockSize); + header->ownerCacheIndex = cacheIndex; - var pUserData = (byte*)ptr + sizeof(BlockHeader); + var userPtr = (byte*)ptr + sizeof(BlockHeader); if (allocationOption.HasFlag(AllocationOption.Clear)) { - MemClear(pUserData, alignedSize); + MemClear(userPtr, alignedSize); } - return pUserData; + return userPtr; + } + finally + { + if (requiresOverflowLock) + { + Interlocked.Exchange(ref _overflowLock, 0); + } } - - return null; } /// - /// Frees a previously allocated memory block. Thread-safe using lock-free algorithms. + /// Frees a previously allocated memory block. /// - /// MemoryBlock to free. [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Free(void* ptr) { @@ -267,226 +586,121 @@ public unsafe struct FreeList : IDisposable var blockStartPtr = (byte*)ptr - sizeof(BlockHeader); var header = (BlockHeader*)blockStartPtr; - if (header->magicNumber != _MAGIC_NUMBER) { return; } - var chuck = header->ownerChunk; - if (chuck == null) + var chunk = header->ownerChunk; + if (chunk == null) { return; } - var bucketIndex = FindBucket(header->blockSize); - if (bucketIndex >= 0) + var blockSize = header->blockSize; + var ownerCacheIndex = header->ownerCacheIndex; + var bucketIndex = FindBucket(blockSize); + + if (bucketIndex < 0) { - PushToBucket(bucketIndex, blockStartPtr, header->blockSize); + header->ownerChunk = null; + header->blockSize = 0; + header->magicNumber = 0; + header->ownerCacheIndex = 0; + return; } - Interlocked.Add(ref _totalAllocatedBytes, -(long)header->blockSize); - header->ownerChunk = null; - header->blockSize = 0; - header->magicNumber = 0; - } - - // FIX: This may introduce ABA problem. Consider adding a version counter to the free list nodes if this becomes an issue. - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private readonly void* TryPopFromBucket(int bucketIndex) - { - var buckets = GetBuckets(); - var bucket = &buckets[bucketIndex]; - - nint head, newHead; - FreeNode* headPtr; - - do + var sameThread = t_ownerId == _instanceId && t_cacheIndex == ownerCacheIndex; + var targetCache = ownerCacheIndex >= 0 && ownerCacheIndex <= _cacheCount ? _caches[ownerCacheIndex] : null; + if (targetCache == null) { - head = bucket->freeHead; - if (head == 0) + targetCache = GetOverflowCache(); + ownerCacheIndex = _OVERFLOW_CACHE_INDEX; + sameThread = t_ownerId == _instanceId && t_cacheIndex == ownerCacheIndex; + } + + if (sameThread) + { + if (ownerCacheIndex == _OVERFLOW_CACHE_INDEX) { - return null; + while (Interlocked.CompareExchange(ref _overflowLock, 1, 0) != 0) + { + Thread.SpinWait(1); + } + + try + { + PushToBucket(targetCache, bucketIndex, blockStartPtr, chunk, blockSize); + } + finally + { + Interlocked.Exchange(ref _overflowLock, 0); + } + } + else + { + PushToBucket(targetCache, bucketIndex, blockStartPtr, chunk, blockSize); } - headPtr = (FreeNode*)head; - newHead = (nint)headPtr->next; + return; + } - } while (Interlocked.CompareExchange(ref bucket->freeHead, newHead, head) != head); - - Interlocked.Decrement(ref bucket->freeCount); - return (void*)head; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private readonly void PushToBucket(int bucketIndex, void* ptr, nuint size) - { - var buckets = GetBuckets(); - var bucket = &buckets[bucketIndex]; - var node = (FreeNode*)ptr; - - node->size = size; + var remoteNode = (FreeNode*)blockStartPtr; + remoteNode->ownerChunk = chunk; + remoteNode->blockSize = blockSize; + remoteNode->bucketIndex = bucketIndex; nint head; do { - head = bucket->freeHead; - node->next = (FreeNode*)head; - - } while (Interlocked.CompareExchange(ref bucket->freeHead, (nint)node, head) != head); - - Interlocked.Increment(ref bucket->freeCount); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void AssignBlockHeader(BlockHeader* header, MemoryChunk* ownerChunk, nuint blockSize) - { - header->ownerChunk = ownerChunk; - header->blockSize = blockSize; - header->magicNumber = _MAGIC_NUMBER; - } - - private bool TryCreateBlocksForBucket(int bucketIndex) - { - var buckets = GetBuckets(); - var bucket = &buckets[bucketIndex]; - - while (Interlocked.CompareExchange(ref bucket->creationLock, 1, 0) != 0) - { - Thread.SpinWait(1); - } - - try - { - if (bucket->freeHead != 0) - { - return true; // Another thread did the work for us! - } - - var blockSize = bucket->blockSize; - var blocksToCreate = Math.Min(_chunkSize / blockSize, 256); // Limit number of blocks - - if (blocksToCreate == 0) - { - return false; - } - - var totalSize = blocksToCreate * blockSize; - var memory = (byte*)AlignedAlloc(totalSize, _alignment); - if (memory == null) - { - return false; - } - - var chunk = (MemoryChunk*)_chunkArena.Allocate(SizeOf(), AlignOf(), AllocationOption.None); - if (chunk == null) - { - AlignedFree(memory); - return false; - } - - chunk->memory = memory; - chunk->size = totalSize; - chunk->used = totalSize; - chunk->next = _chunks; - _chunks = chunk; - - // Add all blocks to the bucket's free list - for (nuint i = 0; i < blocksToCreate; i++) - { - var blockStartPtr = memory + (i * blockSize); - - AssignBlockHeader((BlockHeader*)blockStartPtr, chunk, blockSize); - PushToBucket(bucketIndex, blockStartPtr, blockSize); - } - - return true; - } - finally - { - Interlocked.Exchange(ref bucket->creationLock, 0); - } - } - - private void* AllocateFromChunk(nuint size, nuint alignment) - { - while (Interlocked.CompareExchange(ref _chunkCreationLock, 1, 0) != 0) - { - Thread.SpinWait(1); - } - - try - { - // Try to find space in existing chunks first - var chunk = _chunks; - while (chunk != null) - { - var available = chunk->size - chunk->used; - var alignedOffset = (chunk->used + alignment - 1) & ~(alignment - 1); - var totalNeeded = alignedOffset - chunk->used + size; - - if (totalNeeded <= available) - { - var blockStartPtr = chunk->memory + alignedOffset; - - // Write the header and return the pointer WITH the header - AssignBlockHeader((BlockHeader*)blockStartPtr, chunk, size); - return blockStartPtr; - } - - chunk = chunk->next; - } - - // Create new chunk - var newChunkSize = Math.Max(_chunkSize, size + alignment); - var newMemory = (byte*)AlignedAlloc(newChunkSize, alignment); - if (newMemory == null) - { - return null; - } - - var newChunk = (MemoryChunk*)_chunkArena.Allocate(SizeOf(), AlignOf(), AllocationOption.None); - if (newChunk == null) - { - AlignedFree(newMemory); - return null; - } - - newChunk->memory = newMemory; - newChunk->size = newChunkSize; - newChunk->used = size; - newChunk->next = _chunks; - _chunks = newChunk; - - // Write the header and return the pointer WITH the header - AssignBlockHeader((BlockHeader*)newMemory, newChunk, size); - return newMemory; - } - finally - { - Interlocked.Exchange(ref _chunkCreationLock, 0); - } + head = targetCache->remoteFreeHead; + remoteNode->next = (FreeNode*)head; + } while (Interlocked.CompareExchange(ref targetCache->remoteFreeHead, (nint)remoteNode, head) != head); } public void Dispose() { - if (Interlocked.CompareExchange(ref _disposed, 1, 0) == 0) + if (Interlocked.CompareExchange(ref _disposed, 1, 0) != 0) { - // Free all memory chunks - var chunk = _chunks; - while (chunk != null) - { - var next = chunk->next; - AlignedFree(chunk->memory); - chunk = next; - } - - _chunkArena.Dispose(); - - _chunks = null; - _totalAllocatedBytes = 0; - _totalFreeBytes = 0; + return; } + + if (_caches != null) + { + for (var i = 0; i <= _cacheCount; i++) + { + var cache = _caches[i]; + if (cache != null) + { + DrainRemoteFrees(cache); + cache->active = 0; + } + } + } + + var chunk = _chunks; + while (chunk != null) + { + var next = chunk->next; + AlignedFree(chunk->memory); + chunk = next; + } + + _chunkArena.Dispose(); + + if (_caches != null) + { + Free(_caches); + _caches = null; + } + + if (_instanceId != null) + { + Free(_instanceId); + _instanceId = null; + } + + _chunks = null; + _cacheCount = 0; } -} \ No newline at end of file +} diff --git a/Misaki.HighPerformance.LowLevel/Buffer/Stack.cs b/Misaki.HighPerformance.LowLevel/Buffer/Stack.cs index beae9dc..91d8e70 100644 --- a/Misaki.HighPerformance.LowLevel/Buffer/Stack.cs +++ b/Misaki.HighPerformance.LowLevel/Buffer/Stack.cs @@ -87,12 +87,19 @@ public unsafe partial struct Stack : IDisposable private void Init(nuint size) { + ArgumentOutOfRangeException.ThrowIfNegative(size); + if (_buffer != null) { Free(_buffer); } _buffer = (byte*)Malloc(size); + if (_buffer == null) + { + throw new OutOfMemoryException("Failed to allocate memory for the stack."); + } + _size = size; _offset = 0; _activeScopeCount = 0; @@ -103,15 +110,15 @@ public unsafe partial struct Stack : IDisposable s_locker.Enter(ref token); if (s_pStackBuffers == null) { - s_pStackBuffers = (void**)Malloc((nuint)sizeof(void*) * 4u); - s_stackCapacity = 4; + s_pStackBuffers = (void**)Malloc((nuint)(sizeof(void*) * Environment.ProcessorCount)); + s_stackCapacity = Environment.ProcessorCount; } if (s_stackCount >= s_stackCapacity) { var pOld = s_pStackBuffers; var newCapacity = s_stackCapacity * 2; - var pNew = (void**)Realloc(pOld, (nuint)sizeof(void*) * (nuint)newCapacity); + var pNew = (void**)Realloc(pOld, (nuint)(sizeof(void*) * newCapacity)); s_pStackBuffers = pNew; s_stackCapacity = newCapacity; diff --git a/Misaki.HighPerformance.LowLevel/Collections/HashMapHelper.cs b/Misaki.HighPerformance.LowLevel/Collections/HashMapHelper.cs index 515eead..891d9ef 100644 --- a/Misaki.HighPerformance.LowLevel/Collections/HashMapHelper.cs +++ b/Misaki.HighPerformance.LowLevel/Collections/HashMapHelper.cs @@ -131,15 +131,9 @@ public unsafe struct HashMapHelper : IDisposable public HashMapHelper(int capacity, int sizeOfTValue, int alignOfTValue, uint minGrowth, AllocationHandle handle, AllocationOption allocationOption) { - if (capacity <= 0) - { - throw new ArgumentOutOfRangeException(nameof(capacity), "Capacity must be greater than zero."); - } - - if (sizeOfTValue < 0 || alignOfTValue < 0) - { - throw new ArgumentOutOfRangeException(nameof(sizeOfTValue), "Size or alignment of TValue can not be less than zero."); - } + ArgumentOutOfRangeException.ThrowIfNegative(capacity); + ArgumentOutOfRangeException.ThrowIfNegative(sizeOfTValue); + ArgumentOutOfRangeException.ThrowIfNegative(alignOfTValue); _capacity = CalcCapacityCeilPow2(capacity); _bucketCapacity = _capacity * 2; diff --git a/Misaki.HighPerformance.LowLevel/Collections/UnsafeArray.cs b/Misaki.HighPerformance.LowLevel/Collections/UnsafeArray.cs index d062f39..15ad8c6 100644 --- a/Misaki.HighPerformance.LowLevel/Collections/UnsafeArray.cs +++ b/Misaki.HighPerformance.LowLevel/Collections/UnsafeArray.cs @@ -3,6 +3,7 @@ using Misaki.HighPerformance.LowLevel.Collections.Contracts; using Misaki.HighPerformance.LowLevel.Utilities; using System.Collections; using System.Diagnostics; +using System.Drawing; using System.Runtime.CompilerServices; namespace Misaki.HighPerformance.LowLevel.Collections; @@ -154,10 +155,7 @@ public unsafe struct UnsafeArray : IUnsafeCollection /// Thrown when the specified number of elements is less than or equal to zero. public UnsafeArray(int count, AllocationHandle handle, AllocationOption allocationOption = AllocationOption.None) { - if (count < 0) - { - throw new ArgumentOutOfRangeException(nameof(count), "Count can not be less than zero."); - } + ArgumentOutOfRangeException.ThrowIfNegative(count); if (handle.Alloc == null) { diff --git a/Misaki.HighPerformance.LowLevel/Misaki.HighPerformance.LowLevel.csproj b/Misaki.HighPerformance.LowLevel/Misaki.HighPerformance.LowLevel.csproj index 7f0d5e5..d2b03a5 100644 --- a/Misaki.HighPerformance.LowLevel/Misaki.HighPerformance.LowLevel.csproj +++ b/Misaki.HighPerformance.LowLevel/Misaki.HighPerformance.LowLevel.csproj @@ -7,7 +7,7 @@ true true Misaki - 1.4.4 + 1.4.5 $(AssemblyVersion) https://git.personalnas.com/Misaki/Misaki.HighPerformance.git https://git.personalnas.com/Misaki/Misaki.HighPerformance.git diff --git a/Misaki.HighPerformance.LowLevel/Utilities/MemoryUtility.cs b/Misaki.HighPerformance.LowLevel/Utilities/MemoryUtility.cs index 7e71aa8..699288b 100644 --- a/Misaki.HighPerformance.LowLevel/Utilities/MemoryUtility.cs +++ b/Misaki.HighPerformance.LowLevel/Utilities/MemoryUtility.cs @@ -21,18 +21,11 @@ public static unsafe partial class MemoryUtility [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void* Malloc(nuint size) { - try - { #if NET6_0_OR_GREATER - return NativeMemory.Alloc(size); + return NativeMemory.Alloc(size); #else - return Marshal.AllocHGlobal((IntPtr)size).ToPointer(); + return Marshal.AllocHGlobal((IntPtr)size).ToPointer(); #endif - } - catch (Exception) - { - return null; - } } /// @@ -43,20 +36,13 @@ public static unsafe partial class MemoryUtility [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void* Calloc(nuint size) { - try - { #if NET6_0_OR_GREATER - return NativeMemory.AllocZeroed(size); + return NativeMemory.AllocZeroed(size); #else - var ptr = Marshal.AllocHGlobal((IntPtr)size).ToPointer(); - Unsafe.InitBlock(ptr, 0, (uint)size); - return ptr; + var ptr = Marshal.AllocHGlobal((IntPtr)size).ToPointer(); + Unsafe.InitBlock(ptr, 0, (uint)size); + return ptr; #endif - } - catch (Exception) - { - return null; - } } /// @@ -68,18 +54,11 @@ public static unsafe partial class MemoryUtility [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void* AlignedAlloc(nuint size, nuint alignment) { - try - { #if NET6_0_OR_GREATER - return NativeMemory.AlignedAlloc(size, alignment); + return NativeMemory.AlignedAlloc(size, alignment); #else - return Marshal.AllocHGlobal((IntPtr)(size + alignment - 1)).ToPointer(); + return Marshal.AllocHGlobal((IntPtr)(size + alignment - 1)).ToPointer(); #endif - } - catch (Exception) - { - return null; - } } /// @@ -91,18 +70,11 @@ public static unsafe partial class MemoryUtility [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void* Realloc(void* ptr, nuint size) { - try - { #if NET6_0_OR_GREATER - return NativeMemory.Realloc(ptr, size); + return NativeMemory.Realloc(ptr, size); #else - return Marshal.ReAllocHGlobal((IntPtr)ptr, (IntPtr)size).ToPointer(); + return Marshal.ReAllocHGlobal((IntPtr)ptr, (IntPtr)size).ToPointer(); #endif - } - catch (Exception) - { - return null; - } } /// @@ -116,30 +88,23 @@ public static unsafe partial class MemoryUtility [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void* AlignedRealloc(void* ptr, nuint size, nuint alignment) { - try - { #if NET6_0_OR_GREATER - return NativeMemory.AlignedRealloc(ptr, size, alignment); + return NativeMemory.AlignedRealloc(ptr, size, alignment); #else - var newPtr = Marshal.AllocHGlobal((IntPtr)(size + alignment - 1)).ToPointer(); - if (newPtr == null) - { - return null; - } - - if (ptr != null) - { - Unsafe.CopyBlock(newPtr, ptr, (uint)size); - Marshal.FreeHGlobal((IntPtr)ptr); - } - - return newPtr; -#endif - } - catch (Exception) + var newPtr = Marshal.AllocHGlobal((IntPtr)(size + alignment - 1)).ToPointer(); + if (newPtr == null) { return null; } + + if (ptr != null) + { + Unsafe.CopyBlock(newPtr, ptr, (uint)size); + Marshal.FreeHGlobal((IntPtr)ptr); + } + + return newPtr; +#endif } /// diff --git a/Misaki.HighPerformance.Test/UnitTest/Buffer/TestAllocationManager.cs b/Misaki.HighPerformance.Test/UnitTest/Buffer/TestAllocationManager.cs index d0bad54..8f1b7e7 100644 --- a/Misaki.HighPerformance.Test/UnitTest/Buffer/TestAllocationManager.cs +++ b/Misaki.HighPerformance.Test/UnitTest/Buffer/TestAllocationManager.cs @@ -7,12 +7,6 @@ namespace Misaki.HighPerformance.Test.UnitTest.Buffer; [TestClass] public class TestAllocationManager { - [TestInitialize] - public void Initialize() - { - AllocationManager.EnableDebugLayer(); - } - [TestMethod] public void ShouldNotLeakTest() { diff --git a/Misaki.HighPerformance.Test/UnitTest/Buffer/TestFreeList.cs b/Misaki.HighPerformance.Test/UnitTest/Buffer/TestFreeList.cs new file mode 100644 index 0000000..43e5d31 --- /dev/null +++ b/Misaki.HighPerformance.Test/UnitTest/Buffer/TestFreeList.cs @@ -0,0 +1,151 @@ +using Misaki.HighPerformance.LowLevel.Buffer; +using System.Diagnostics; + +namespace Misaki.HighPerformance.Test.UnitTest.Buffer; + +[TestClass] +public unsafe class TestFreeList +{ + [TestMethod] + public void SingleThreadedAllocFreeTest() + { + using var freeList = new FreeList(8, 1024); + + // Allocate various sizes + void* p1 = freeList.Allocate(16, 8); + void* p2 = freeList.Allocate(32, 8); + void* p3 = freeList.Allocate(64, 8); + + Assert.IsTrue(p1 != null); + Assert.IsTrue(p2 != null); + Assert.IsTrue(p3 != null); + + // Free them + freeList.Free(p1); + freeList.Free(p2); + freeList.Free(p3); + + // Allocate again - should reuse from buckets (or at least succeed) + void* p4 = freeList.Allocate(16, 8); + void* p5 = freeList.Allocate(32, 8); + + Assert.IsTrue(p4 != null); + Assert.IsTrue(p5 != null); + + freeList.Free(p4); + freeList.Free(p5); + } + + [TestMethod] + public void MultiThreadedAllocSameThreadFreeTest() + { + const int threadCount = 8; + const int iterations = 1000; + using var freeList = new FreeList(8, 64 * 1024, threadCount); + + var threads = new Thread[threadCount]; + for (int i = 0; i < threadCount; i++) + { + threads[i] = new Thread(() => + { + for (int j = 0; j < iterations; j++) + { + void* ptr = freeList.Allocate(16, 8); + Assert.IsTrue(ptr != null); + freeList.Free(ptr); + } + }); + } + + foreach (var t in threads) t.Start(); + foreach (var t in threads) t.Join(); + } + + [TestMethod] + public void MultiThreadedCrossThreadFreeTest() + { + const int producerCount = 4; + const int consumerCount = 4; + const int iterations = 5000; + using var freeList = new FreeList(8, 64 * 1024, producerCount + consumerCount); + + var queue = new System.Collections.Concurrent.ConcurrentQueue(); + var producers = new Thread[producerCount]; + var consumers = new Thread[consumerCount]; + + bool producing = true; + + for (int i = 0; i < producerCount; i++) + { + producers[i] = new Thread(() => + { + for (int j = 0; j < iterations; j++) + { + void* ptr = freeList.Allocate(32, 8); + Assert.IsTrue(ptr != null); + queue.Enqueue((IntPtr)ptr); + } + }); + } + + for (int i = 0; i < consumerCount; i++) + { + consumers[i] = new Thread(() => + { + while (Volatile.Read(ref producing) || !queue.IsEmpty) + { + if (queue.TryDequeue(out var ptr)) + { + freeList.Free((void*)ptr); + } + else + { + Thread.Yield(); + } + } + }); + } + + foreach (var t in producers) t.Start(); + foreach (var t in consumers) t.Start(); + + foreach (var t in producers) t.Join(); + Volatile.Write(ref producing, false); + foreach (var t in consumers) t.Join(); + } + + [TestMethod] + public void OverflowCacheTest() + { + // Set maxConcurrencyLevel to 1, but use more threads + const int threadCount = 5; + using var freeList = new FreeList(8, 1024, 1); + + var threads = new Thread[threadCount]; + for (int i = 0; i < threadCount; i++) + { + threads[i] = new Thread(() => + { + void* ptr = freeList.Allocate(16, 8); + Assert.IsTrue(ptr != null); + freeList.Free(ptr); + }); + } + + foreach (var t in threads) t.Start(); + foreach (var t in threads) t.Join(); + } + + [TestMethod] + public void LargeAllocationTest() + { + using var freeList = new FreeList(8, 1024); + + // Allocate larger than default chunk size + nuint largeSize = 2048; + void* ptr = freeList.Allocate(largeSize, 8); + Assert.IsTrue(ptr != null); + + freeList.Free(ptr); + } +} diff --git a/Misaki.HighPerformance/Utilities/CollectionUtility.cs b/Misaki.HighPerformance/Utilities/CollectionUtility.cs index a023226..01f52f6 100644 --- a/Misaki.HighPerformance/Utilities/CollectionUtility.cs +++ b/Misaki.HighPerformance/Utilities/CollectionUtility.cs @@ -54,7 +54,7 @@ public static class CollectionUtility /// The zero-based index of the element to retrieve. /// A reference to the element at the specified index in the span. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ref readonly T GetElementUnsafe(this Span span, int index) + public static ref T GetElementUnsafe(this Span span, int index) { return ref Unsafe.Add(ref MemoryMarshal.GetReference(span), index); }