diff --git a/Misaki.HighPerformance.Jobs/JobScheduler.cs b/Misaki.HighPerformance.Jobs/JobScheduler.cs index 5728e96..abe7f20 100644 --- a/Misaki.HighPerformance.Jobs/JobScheduler.cs +++ b/Misaki.HighPerformance.Jobs/JobScheduler.cs @@ -270,14 +270,15 @@ public sealed unsafe partial class JobScheduler : IJobScheduler, IDisposable /// The number of worker threads to create. If less than 1, at least one thread will be created. public JobScheduler(int threadCount) { - _jobDataAllocator = new(8); + var workerCount = Math.Max(1, threadCount); + + _jobDataAllocator = new(8, maxConcurrencyLevel: workerCount + 1); _jobInfoPool = new(); _jobQueue = new(); _workSignal = new(0); _cts = new(); - var workerCount = Math.Max(1, threadCount); _workerThreads = new WorkerThread[workerCount]; for (var i = 0; i < workerCount; i++) diff --git a/Misaki.HighPerformance.Jobs/Misaki.HighPerformance.Jobs.csproj b/Misaki.HighPerformance.Jobs/Misaki.HighPerformance.Jobs.csproj index c6548ad..69a77b5 100644 --- a/Misaki.HighPerformance.Jobs/Misaki.HighPerformance.Jobs.csproj +++ b/Misaki.HighPerformance.Jobs/Misaki.HighPerformance.Jobs.csproj @@ -6,7 +6,7 @@ enable True true - 1.5.1 + 1.5.2 $(AssemblyVersion) Misaki https://git.personalnas.com/Misaki/Misaki.HighPerformance.git diff --git a/Misaki.HighPerformance.LowLevel/Buffer/AllocationManager.cs b/Misaki.HighPerformance.LowLevel/Buffer/AllocationManager.cs index 3b49915..da426ae 100644 --- a/Misaki.HighPerformance.LowLevel/Buffer/AllocationManager.cs +++ b/Misaki.HighPerformance.LowLevel/Buffer/AllocationManager.cs @@ -33,16 +33,17 @@ public readonly struct AllocationInfo /// public static unsafe class AllocationManager { - // === Intrusive allocation tracking (enabled when debug layer is on) === +#if ENABLE_DEBUG_LAYER [StructLayout(LayoutKind.Sequential)] private struct AllocationHeader { public AllocationHeader* prev; public AllocationHeader* next; - public void* basePtr; // pointer returned by underlying allocator - public nuint userSize; // requested size from the user - public GCHandle stackHandle; // GCHandle to managed StackTrace + public void* basePtr; // pointer returned by underlying allocator + public nuint userSize; // requested size from the user + public GCHandle stackHandle; // GCHandle to managed StackTrace } +#endif private struct ArenaAllocator : IAllocator, IDisposable { @@ -270,11 +271,12 @@ public static unsafe class AllocationManager private static readonly HeapAllocator* s_pHeapAllocator; private static readonly StackAllocator* s_pStackAllocator; - private static bool s_debugLayer; private static bool s_disposed; - private static AllocationHeader* s_pLiveHead; +#if ENABLE_DEBUG_LAYER private static SpinLock s_liveLock; + private static AllocationHeader* s_pLiveHead; +#endif private static readonly ConcurrentSlotMap s_allocations; @@ -285,31 +287,28 @@ public static unsafe class AllocationManager /// public static int LiveAllocationCount => s_allocations.Count; - /// - /// Gets a value indicating whether the debug layer is currently enabled. - /// - public static bool IsDebugLayerEnabled => s_debugLayer; - - static AllocationManager() { var allocatorTotalSize = (nuint)(sizeof(ArenaAllocator) + sizeof(HeapAllocator) + sizeof(StackAllocator)); - var basePtr = NativeMemory.Alloc(allocatorTotalSize); + var basePtr = Malloc(allocatorTotalSize); + s_pArenaAllocator = (ArenaAllocator*)basePtr; s_pHeapAllocator = (HeapAllocator*)((byte*)basePtr + (nuint)sizeof(ArenaAllocator)); s_pStackAllocator = (StackAllocator*)((byte*)basePtr + (nuint)(sizeof(ArenaAllocator) + sizeof(HeapAllocator))); +#if ENABLE_DEBUG_LAYER s_liveLock = new SpinLock(false); + s_pLiveHead = null; +#endif s_allocations = new ConcurrentSlotMap(256); s_pArenaAllocator->Init(_DEFAULT_MEMORY_POOL_SIZE); s_pHeapAllocator->Init(); s_pStackAllocator->Init(); - - s_pLiveHead = null; } +#if ENABLE_DEBUG_LAYER [MethodImpl(MethodImplOptions.AggressiveInlining)] private static byte* AlignUp(byte* p, nuint alignment) { @@ -462,22 +461,7 @@ public static unsafe class AllocationManager return newUser; } - - /// - /// Enables the debug layer, allowing additional diagnostic information to be collected. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void EnableDebugLayer() - { - // To avoid ambiguity between pointers allocated before/after enabling, this must be called - // before any heap allocations are live. - if (s_allocations.Count != 0) - { - throw new InvalidOperationException("EnableDebugLayer must be called before any allocations are active."); - } - - s_debugLayer = true; - } +#endif /// /// Gets a reference to the allocation pHandle for the specified allocator type. @@ -499,10 +483,6 @@ public static unsafe class AllocationManager /// /// Allocates a block of memory from the heap with the specified size and alignment, using the given allocation options. /// - /// - /// This will allocate memory from the heap. If the debug layer is enabled, additional tracking information will be recorded. - /// The memory handle is always tracked unless the flag is specified. - /// /// The number of bytes to allocate. Must be greater than zero. /// The alignment, in bytes, for the allocated memory block. Must be a power of two. /// An optional set of flags that control allocation behavior, such as whether the memory should be cleared or @@ -511,17 +491,11 @@ public static unsafe class AllocationManager /// Thrown if the allocation fails. public static void* HeapAlloc(nuint size, nuint alignment, AllocationOption allocationOption, MemoryHandle* pHandle) { - var isUntrack = allocationOption.HasFlag(AllocationOption.Untrack); - - void* ptr; - if (s_debugLayer && !isUntrack) - { - ptr = DebugAllocate(size, alignment); - } - else - { - ptr = AlignedAlloc(size, alignment); - } +#if ENABLE_DEBUG_LAYER + var ptr = DebugAllocate(size, alignment); +#else + var ptr = AlignedAlloc(size, alignment); +#endif if (ptr == null) { @@ -534,15 +508,7 @@ public static unsafe class AllocationManager MemClear(ptr, size); } - if (isUntrack) - { - *pHandle = MagicHandle; - } - else - { - *pHandle = AddAllocation((IntPtr)ptr); - } - + *pHandle = AddAllocation((IntPtr)ptr); return ptr; } @@ -554,11 +520,13 @@ public static unsafe class AllocationManager /// The handle representing the memory allocation to free. The handle must be valid and previously allocated. public static void HeapFree(void* ptr, MemoryHandle handle) { - if (s_debugLayer && handle != MagicHandle) +#if ENABLE_DEBUG_LAYER + if (handle != MagicHandle) { DebugFree(ptr); } else +#endif { AlignedFree(ptr); } @@ -662,53 +630,53 @@ public static unsafe class AllocationManager return; } +#if ENABLE_DEBUG_LAYER // In debug mode, walk the intrusive list to surface any leaks. - if (s_debugLayer) + var snapshot = new List(); + var taken = false; + try { - var snapshot = new List(); - var taken = false; - try + s_liveLock.Enter(ref taken); + if (s_pLiveHead != null) { - s_liveLock.Enter(ref taken); - if (s_pLiveHead != null) + snapshot.Capacity = 128; + for (var p = s_pLiveHead; p != null; p = p->next) { - snapshot.Capacity = 128; - for (var p = s_pLiveHead; p != null; p = p->next) + var trace = (StackTrace)HeaderGetHandle(p).Target!; + snapshot.Add(new AllocationInfo { - var trace = (StackTrace)HeaderGetHandle(p).Target!; - snapshot.Add(new AllocationInfo - { - Size = p->userSize, - StackTrace = trace - }); - } + Size = p->userSize, + StackTrace = trace + }); } } - finally - { - if (taken) - { - s_liveLock.Exit(); - } - } - - nuint unfreeBytes = 0u; - foreach (var info in snapshot) - { - unfreeBytes += info.Size; - } - - if (unfreeBytes > 0u) - { - throw new MemoryLeakException(CollectionsMarshal.AsSpan(snapshot)); - } - - Debug.Assert(LiveAllocationCount == 0); } - else if (LiveAllocationCount != 0) + finally + { + if (taken) + { + s_liveLock.Exit(); + } + } + + nuint unfreeBytes = 0u; + foreach (var info in snapshot) + { + unfreeBytes += info.Size; + } + + if (unfreeBytes > 0u) + { + throw new MemoryLeakException(CollectionsMarshal.AsSpan(snapshot)); + } + + Debug.Assert(LiveAllocationCount == 0); +#else + if (LiveAllocationCount != 0) { throw new MemoryLeakException($"Found {LiveAllocationCount} memory lakes! Please enable debug layer for more informations."); } +#endif // NOTE: Arena allocator holds the base ptr for all allocators, heap and stack allocators do not own any memory themselves. if (s_pArenaAllocator != null) diff --git a/Misaki.HighPerformance.LowLevel/Buffer/AllocationOption.cs b/Misaki.HighPerformance.LowLevel/Buffer/AllocationOption.cs index f02bd22..3acf6b4 100644 --- a/Misaki.HighPerformance.LowLevel/Buffer/AllocationOption.cs +++ b/Misaki.HighPerformance.LowLevel/Buffer/AllocationOption.cs @@ -10,11 +10,7 @@ public enum AllocationOption : byte /// /// Clear the memory to zero upon allocation. /// - Clear = 1 << 0, - /// - /// Specify that this memory allocation should not been tracked competly, which will not perform any safty check like use after free and leack detection. - /// - Untrack = 1 << 1, + Clear = 1 << 0 } public enum Allocator : byte diff --git a/Misaki.HighPerformance.LowLevel/Buffer/Arena.cs b/Misaki.HighPerformance.LowLevel/Buffer/Arena.cs index cfe80c4..633f056 100644 --- a/Misaki.HighPerformance.LowLevel/Buffer/Arena.cs +++ b/Misaki.HighPerformance.LowLevel/Buffer/Arena.cs @@ -1,4 +1,4 @@ -using System.Runtime.InteropServices; +using System.Runtime.InteropServices; namespace Misaki.HighPerformance.LowLevel.Buffer; @@ -17,6 +17,8 @@ public unsafe struct Arena : IDisposable public Arena(nuint size) { + ArgumentOutOfRangeException.ThrowIfNegative(size); + if (_buffer != null) { return; diff --git a/Misaki.HighPerformance.LowLevel/Buffer/DynamicArena.cs b/Misaki.HighPerformance.LowLevel/Buffer/DynamicArena.cs index 43706bc..8fc8c45 100644 --- a/Misaki.HighPerformance.LowLevel/Buffer/DynamicArena.cs +++ b/Misaki.HighPerformance.LowLevel/Buffer/DynamicArena.cs @@ -1,4 +1,4 @@ -using System.Runtime.InteropServices; +using System.Runtime.InteropServices; namespace Misaki.HighPerformance.LowLevel.Buffer; diff --git a/Misaki.HighPerformance.LowLevel/Buffer/FreeList.cs b/Misaki.HighPerformance.LowLevel/Buffer/FreeList.cs index aca13f6..2ae8c32 100644 --- a/Misaki.HighPerformance.LowLevel/Buffer/FreeList.cs +++ b/Misaki.HighPerformance.LowLevel/Buffer/FreeList.cs @@ -4,17 +4,19 @@ using System.Runtime.InteropServices; namespace Misaki.HighPerformance.LowLevel.Buffer; /// -/// A lock-free, thread-safe variable-size allocator that manages memory blocks of different sizes. -/// Optimized for high-performance scenarios with frequent allocations and deallocations. +/// A thread-safe variable-size allocator that uses per-thread caches for the hot path and +/// a remote-free queue for cross-thread deallocation. /// -[StructLayout(LayoutKind.Explicit, Size = 256)] // Cache line aligned to prevent false sharing +[StructLayout(LayoutKind.Sequential)] public unsafe struct FreeList : IDisposable { [StructLayout(LayoutKind.Sequential)] private struct FreeNode { public FreeNode* next; - public nuint size; + public MemoryChunk* ownerChunk; + public nuint blockSize; + public int bucketIndex; } [StructLayout(LayoutKind.Sequential)] @@ -23,81 +25,75 @@ public unsafe struct FreeList : IDisposable public MemoryChunk* next; public byte* memory; public nuint size; - public nuint used; // Amount of memory used in this chunk + public nuint used; } [StructLayout(LayoutKind.Explicit, Size = 32)] private struct SizeBucket { [FieldOffset(0)] - public long freeCount; // Number of free blocks + public long freeCount; [FieldOffset(8)] - public nint freeHead; // Free list head for this size + public nint freeHead; [FieldOffset(16)] - public nuint blockSize; // Fixed size for this bucket + public nuint blockSize; [FieldOffset(24)] public int creationLock; } - [StructLayout(LayoutKind.Explicit, Size = 24)] + [StructLayout(LayoutKind.Sequential)] + private struct ThreadCache + { + public fixed byte buckets[_MAX_BUCKETS * 32]; + public nint remoteFreeHead; + public int threadId; + public int active; + } + + [StructLayout(LayoutKind.Explicit, Size = 32)] private struct BlockHeader { - // Ensure the size is fixed across x86 and x64 [FieldOffset(0)] public MemoryChunk* ownerChunk; [FieldOffset(8)] public nuint blockSize; [FieldOffset(16)] public ulong magicNumber; + [FieldOffset(24)] + public int ownerCacheIndex; } - private const int _MAX_BUCKETS = 16; // Number of size buckets - private const nuint _MIN_BLOCK_SIZE = 16; // Minimum block size - private const nuint _DEFAULT_CHUNK_SIZE = 64 * 1024; // 64KB chunks - private const ulong _MAGIC_NUMBER = 0xDEADBEEFDEADBEEF; // For validating blocks + private const int _MAX_BUCKETS = 16; + private const int _DEFAULT_MAX_CONCURRENCY_LEVEL = 16; + private const int _OVERFLOW_CACHE_INDEX = 0; + private const nuint _MIN_BLOCK_SIZE = 16; + private const nuint _DEFAULT_CHUNK_SIZE = 64 * 1024; + private const ulong _MAGIC_NUMBER = 0xDEADBEEFDEADBEEF; - [FieldOffset(0)] - private fixed byte _buckets[_MAX_BUCKETS * 32]; // SizeBucket array (32 bytes per bucket) + [ThreadStatic] + private static int t_cacheIndex; - [FieldOffset(512)] - private DynamicArena _chunkArena; // 128 + [ThreadStatic] + private static void* t_ownerId; - [FieldOffset(640)] - private MemoryChunk* _chunks; // 8 - - [FieldOffset(648)] - private readonly nuint _chunkSize; // 8 - - [FieldOffset(656)] - private readonly nuint _alignment; // 8 - - [FieldOffset(664)] - private long _totalAllocatedBytes; // 8 - - [FieldOffset(672)] - private long _totalFreeBytes; // 8 - - [FieldOffset(676)] - private volatile int _disposed; // 4 - - [FieldOffset(680)] - private volatile int _chunkCreationLock; // 4 + private void* _instanceId; + private ThreadCache** _caches; + private DynamicArena _chunkArena; + private MemoryChunk* _chunks; + private readonly nuint _chunkSize; + private readonly nuint _alignment; + private readonly int _maxConcurrencyLevel; + private int _cacheCount; + private volatile int _disposed; + private volatile int _chunkCreationLock; + private volatile int _cacheRegistrationLock; + private volatile int _overflowLock; /// /// Gets the alignment requirement for allocations. /// public readonly nuint Alignment => _alignment; - /// - /// Gets the total number of allocated bytes. - /// - public readonly long TotalAllocatedBytes => Interlocked.Read(ref Unsafe.AsRef(in _totalAllocatedBytes)); - - /// - /// Gets the total number of free bytes available. - /// - public readonly long TotalFreeBytes => Interlocked.Read(ref Unsafe.AsRef(in _totalFreeBytes)); - /// /// Gets whether the allocator has been disposed. /// @@ -108,12 +104,18 @@ public unsafe struct FreeList : IDisposable /// public readonly nuint ChunkSize => _chunkSize; + /// + /// Gets the maximum number of dedicated thread caches. + /// + public readonly int MaxConcurrencyLevel => _maxConcurrencyLevel; + /// /// Initializes a new variable-size FreeList allocator with the specified parameters. /// /// Alignment requirement for blocks (must be power of 2). /// Size of memory chunks to allocate (default: 64KB). - public FreeList(nuint alignment, nuint chunkSize = _DEFAULT_CHUNK_SIZE) + /// Maximum number of dedicated thread caches. + public FreeList(nuint alignment, nuint chunkSize = _DEFAULT_CHUNK_SIZE, int maxConcurrencyLevel = _DEFAULT_MAX_CONCURRENCY_LEVEL) { if (alignment == 0 || (alignment & (alignment - 1)) != 0) { @@ -125,21 +127,72 @@ public unsafe struct FreeList : IDisposable throw new ArgumentException("Chunk size must be at least 1KB", nameof(chunkSize)); } + if (maxConcurrencyLevel < 1) + { + throw new ArgumentOutOfRangeException(nameof(maxConcurrencyLevel), "Max concurrency level must be greater than zero."); + } + _alignment = alignment; _chunkSize = chunkSize; - _chunks = null; - _totalAllocatedBytes = 0; - _totalFreeBytes = 0; - _disposed = 0; - _chunkCreationLock = 0; + _maxConcurrencyLevel = maxConcurrencyLevel; - _chunkArena = new DynamicArena(1024); - InitializeBuckets(); + try + { + _instanceId = Malloc((nuint)sizeof(nint)); + + _chunks = null; + _cacheCount = 0; + _disposed = 0; + _chunkCreationLock = 0; + _cacheRegistrationLock = 0; + _overflowLock = 0; + _chunkArena = new DynamicArena(1024); + _caches = (ThreadCache**)Malloc((nuint)sizeof(ThreadCache*) * (nuint)(maxConcurrencyLevel + 1)); + + for (var i = 0; i <= maxConcurrencyLevel; i++) + { + _caches[i] = null; + } + + var overflowCache = CreateCacheForThread(0); + if (overflowCache == null) + { + throw new OutOfMemoryException("Failed to initialize free list overflow cache."); + } + + _caches[_OVERFLOW_CACHE_INDEX] = overflowCache; + + } + catch + { + if (_instanceId != null) + { + Free(_instanceId); + _instanceId = null; + } + + if (_caches != null) + { + Free(_caches); + _caches = null; + } + + _chunkArena.Dispose(); + + throw; + } } - private readonly void InitializeBuckets() + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static SizeBucket* GetBuckets(ThreadCache* cache) { - var buckets = GetBuckets(); + return (SizeBucket*)cache->buckets; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void InitializeBuckets(ThreadCache* cache) + { + var buckets = GetBuckets(cache); var size = _MIN_BLOCK_SIZE; for (var i = 0; i < _MAX_BUCKETS; i++) @@ -147,42 +200,296 @@ public unsafe struct FreeList : IDisposable buckets[i].blockSize = size; buckets[i].freeHead = 0; buckets[i].freeCount = 0; - size *= 2; // Exponential size increase + buckets[i].creationLock = 0; + size *= 2; } - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private readonly SizeBucket* GetBuckets() - { - fixed (byte* ptr = _buckets) - { - return (SizeBucket*)ptr; - } + cache->remoteFreeHead = 0; } [MethodImpl(MethodImplOptions.AggressiveInlining)] private readonly int FindBucket(nuint size) { - var buckets = GetBuckets(); - + var blockSize = _MIN_BLOCK_SIZE; for (var i = 0; i < _MAX_BUCKETS; i++) { - if (size <= buckets[i].blockSize) + if (size <= blockSize) { return i; } + + blockSize <<= 1; } - return -1; // Size too large for buckets + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private ThreadCache* CreateCacheForThread(int threadId) + { + var cache = (ThreadCache*)_chunkArena.Allocate(SizeOf(), AlignOf(), AllocationOption.Clear); + if (cache == null) + { + return null; + } + + InitializeBuckets(cache); + cache->threadId = threadId; + cache->active = 1; + return cache; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void DrainRemoteFrees(ThreadCache* cache) + { + var head = (FreeNode*)Interlocked.Exchange(ref cache->remoteFreeHead, 0); + while (head != null) + { + var next = head->next; + PushToBucket(cache, head->bucketIndex, head, head->ownerChunk, head->blockSize); + head = next; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private readonly ThreadCache* GetOverflowCache() + { + return _caches[_OVERFLOW_CACHE_INDEX]; + } + + private ThreadCache* RegisterThreadCache() + { + while (Interlocked.CompareExchange(ref _cacheRegistrationLock, 1, 0) != 0) + { + Thread.SpinWait(1); + } + + try + { + if (t_ownerId == _instanceId && t_cacheIndex > 0 && t_cacheIndex <= _cacheCount) + { + return _caches[t_cacheIndex]; + } + + if (_cacheCount >= _maxConcurrencyLevel) + { + t_ownerId = _instanceId; + t_cacheIndex = _OVERFLOW_CACHE_INDEX; + return GetOverflowCache(); + } + + var threadId = Environment.CurrentManagedThreadId; + var cache = CreateCacheForThread(threadId); + if (cache == null) + { + t_ownerId = _instanceId; + t_cacheIndex = _OVERFLOW_CACHE_INDEX; + return GetOverflowCache(); + } + + _cacheCount++; + _caches[_cacheCount] = cache; + + t_ownerId = _instanceId; + t_cacheIndex = _cacheCount; + return cache; + } + finally + { + Interlocked.Exchange(ref _cacheRegistrationLock, 0); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private ThreadCache* GetCurrentCache() + { + if (t_ownerId == _instanceId) + { + var index = t_cacheIndex; + if ((uint)index <= (uint)_cacheCount) + { + var cache = _caches[index]; + if (cache != null) + { + return cache; + } + } + } + + return RegisterThreadCache(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void* TryPopFromBucket(ThreadCache* cache, int cacheIndex, int bucketIndex) + { + var buckets = GetBuckets(cache); + var bucket = &buckets[bucketIndex]; + var head = (FreeNode*)bucket->freeHead; + if (head == null) + { + return null; + } + + bucket->freeHead = (nint)head->next; + bucket->freeCount--; + + AssignBlockHeader((BlockHeader*)head, head->ownerChunk, head->blockSize, cacheIndex); + return head; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void PushToBucket(ThreadCache* cache, int bucketIndex, void* ptr, MemoryChunk* ownerChunk, nuint blockSize) + { + var buckets = GetBuckets(cache); + var bucket = &buckets[bucketIndex]; + var node = (FreeNode*)ptr; + node->ownerChunk = ownerChunk; + node->blockSize = blockSize; + node->bucketIndex = bucketIndex; + node->next = (FreeNode*)bucket->freeHead; + bucket->freeHead = (nint)node; + bucket->freeCount++; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void AssignBlockHeader(BlockHeader* header, MemoryChunk* ownerChunk, nuint blockSize, int ownerCacheIndex) + { + header->ownerChunk = ownerChunk; + header->blockSize = blockSize; + header->magicNumber = _MAGIC_NUMBER; + header->ownerCacheIndex = ownerCacheIndex; + } + + private bool TryCreateBlocksForBucket(ThreadCache* cache, int cacheIndex, int bucketIndex) + { + var buckets = GetBuckets(cache); + var bucket = &buckets[bucketIndex]; + + while (Interlocked.CompareExchange(ref bucket->creationLock, 1, 0) != 0) + { + Thread.SpinWait(1); + } + + try + { + DrainRemoteFrees(cache); + if (bucket->freeHead != 0) + { + return true; + } + + var blockSize = bucket->blockSize; + var blocksToCreate = Math.Min(_chunkSize / blockSize, 256); + if (blocksToCreate == 0) + { + return false; + } + + var totalSize = blocksToCreate * blockSize; + var memory = (byte*)AlignedAlloc(totalSize, _alignment); + if (memory == null) + { + return false; + } + + var chunk = (MemoryChunk*)_chunkArena.Allocate(SizeOf(), AlignOf(), AllocationOption.None); + if (chunk == null) + { + AlignedFree(memory); + return false; + } + + while (Interlocked.CompareExchange(ref _chunkCreationLock, 1, 0) != 0) + { + Thread.SpinWait(1); + } + + try + { + chunk->memory = memory; + chunk->size = totalSize; + chunk->used = totalSize; + chunk->next = _chunks; + _chunks = chunk; + } + finally + { + Interlocked.Exchange(ref _chunkCreationLock, 0); + } + + for (nuint i = 0; i < blocksToCreate; i++) + { + var blockStartPtr = memory + (i * blockSize); + PushToBucket(cache, bucketIndex, blockStartPtr, chunk, blockSize); + } + + return true; + } + finally + { + Interlocked.Exchange(ref bucket->creationLock, 0); + } + } + + private void* AllocateFromChunk(int cacheIndex, nuint size, nuint alignment) + { + while (Interlocked.CompareExchange(ref _chunkCreationLock, 1, 0) != 0) + { + Thread.SpinWait(1); + } + + try + { + var chunk = _chunks; + while (chunk != null) + { + var alignedOffset = (chunk->used + alignment - 1) & ~(alignment - 1); + var totalNeeded = alignedOffset - chunk->used + size; + var available = chunk->size - chunk->used; + + if (totalNeeded <= available) + { + var blockStartPtr = chunk->memory + alignedOffset; + chunk->used = alignedOffset + size; + AssignBlockHeader((BlockHeader*)blockStartPtr, chunk, size, cacheIndex); + return blockStartPtr; + } + + chunk = chunk->next; + } + + var newChunkSize = Math.Max(_chunkSize, size + alignment); + var newMemory = (byte*)AlignedAlloc(newChunkSize, alignment); + if (newMemory == null) + { + return null; + } + + var newChunk = (MemoryChunk*)_chunkArena.Allocate(SizeOf(), AlignOf(), AllocationOption.None); + if (newChunk == null) + { + AlignedFree(newMemory); + return null; + } + + newChunk->memory = newMemory; + newChunk->size = newChunkSize; + newChunk->used = size; + newChunk->next = _chunks; + _chunks = newChunk; + + AssignBlockHeader((BlockHeader*)newMemory, newChunk, size, cacheIndex); + return newMemory; + } + finally + { + Interlocked.Exchange(ref _chunkCreationLock, 0); + } } /// - /// Allocates a memory block of the specified size. Thread-safe using lock-free algorithms. + /// Allocates a memory block of the specified size. /// - /// Size of memory to allocate in bytes. - /// Alignment requirement (0 = use default). - /// Options for allocation (e.g., clear memory). - /// MemoryBlock containing allocated memory, or Invalid if allocation fails. [MethodImpl(MethodImplOptions.AggressiveInlining)] public void* Allocate(nuint size, nuint alignment, AllocationOption allocationOption = AllocationOption.None) { @@ -206,57 +513,69 @@ public unsafe struct FreeList : IDisposable throw new ArgumentException("Alignment must be a power of two.", nameof(alignment)); } - // Align size to alignment boundary var alignedSize = (size + alignment - 1) & ~(alignment - 1); alignedSize = Math.Max(alignedSize, _MIN_BLOCK_SIZE); var totalSize = alignedSize + (nuint)sizeof(BlockHeader); - var bucketIndex = FindBucket(totalSize); - void* ptr = null; + var cache = GetCurrentCache(); + var cacheIndex = t_cacheIndex; + var requiresOverflowLock = cacheIndex == _OVERFLOW_CACHE_INDEX; - if (bucketIndex >= 0) + if (requiresOverflowLock) { - // Try to allocate from bucket - ptr = TryPopFromBucket(bucketIndex); + while (Interlocked.CompareExchange(ref _overflowLock, 1, 0) != 0) + { + Thread.SpinWait(1); + } + } + + try + { + DrainRemoteFrees(cache); + + void* ptr = null; + if (bucketIndex >= 0) + { + ptr = TryPopFromBucket(cache, cacheIndex, bucketIndex); + if (ptr == null && TryCreateBlocksForBucket(cache, cacheIndex, bucketIndex)) + { + ptr = TryPopFromBucket(cache, cacheIndex, bucketIndex); + } + } if (ptr == null) { - // Create new blocks for this bucket - if (TryCreateBlocksForBucket(bucketIndex)) - { - ptr = TryPopFromBucket(bucketIndex); - } + ptr = AllocateFromChunk(cacheIndex, totalSize, alignment); + } + if (ptr == null) + { + return null; } - } - if (ptr == null) - { - // Fallback to direct allocation from chunk - ptr = AllocateFromChunk(totalSize, alignment); - } - - if (ptr != null) - { var header = (BlockHeader*)ptr; - Interlocked.Add(ref _totalAllocatedBytes, (long)header->blockSize); + header->ownerCacheIndex = cacheIndex; - var pUserData = (byte*)ptr + sizeof(BlockHeader); + var userPtr = (byte*)ptr + sizeof(BlockHeader); if (allocationOption.HasFlag(AllocationOption.Clear)) { - MemClear(pUserData, alignedSize); + MemClear(userPtr, alignedSize); } - return pUserData; + return userPtr; + } + finally + { + if (requiresOverflowLock) + { + Interlocked.Exchange(ref _overflowLock, 0); + } } - - return null; } /// - /// Frees a previously allocated memory block. Thread-safe using lock-free algorithms. + /// Frees a previously allocated memory block. /// - /// MemoryBlock to free. [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Free(void* ptr) { @@ -267,226 +586,121 @@ public unsafe struct FreeList : IDisposable var blockStartPtr = (byte*)ptr - sizeof(BlockHeader); var header = (BlockHeader*)blockStartPtr; - if (header->magicNumber != _MAGIC_NUMBER) { return; } - var chuck = header->ownerChunk; - if (chuck == null) + var chunk = header->ownerChunk; + if (chunk == null) { return; } - var bucketIndex = FindBucket(header->blockSize); - if (bucketIndex >= 0) + var blockSize = header->blockSize; + var ownerCacheIndex = header->ownerCacheIndex; + var bucketIndex = FindBucket(blockSize); + + if (bucketIndex < 0) { - PushToBucket(bucketIndex, blockStartPtr, header->blockSize); + header->ownerChunk = null; + header->blockSize = 0; + header->magicNumber = 0; + header->ownerCacheIndex = 0; + return; } - Interlocked.Add(ref _totalAllocatedBytes, -(long)header->blockSize); - header->ownerChunk = null; - header->blockSize = 0; - header->magicNumber = 0; - } - - // FIX: This may introduce ABA problem. Consider adding a version counter to the free list nodes if this becomes an issue. - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private readonly void* TryPopFromBucket(int bucketIndex) - { - var buckets = GetBuckets(); - var bucket = &buckets[bucketIndex]; - - nint head, newHead; - FreeNode* headPtr; - - do + var sameThread = t_ownerId == _instanceId && t_cacheIndex == ownerCacheIndex; + var targetCache = ownerCacheIndex >= 0 && ownerCacheIndex <= _cacheCount ? _caches[ownerCacheIndex] : null; + if (targetCache == null) { - head = bucket->freeHead; - if (head == 0) + targetCache = GetOverflowCache(); + ownerCacheIndex = _OVERFLOW_CACHE_INDEX; + sameThread = t_ownerId == _instanceId && t_cacheIndex == ownerCacheIndex; + } + + if (sameThread) + { + if (ownerCacheIndex == _OVERFLOW_CACHE_INDEX) { - return null; + while (Interlocked.CompareExchange(ref _overflowLock, 1, 0) != 0) + { + Thread.SpinWait(1); + } + + try + { + PushToBucket(targetCache, bucketIndex, blockStartPtr, chunk, blockSize); + } + finally + { + Interlocked.Exchange(ref _overflowLock, 0); + } + } + else + { + PushToBucket(targetCache, bucketIndex, blockStartPtr, chunk, blockSize); } - headPtr = (FreeNode*)head; - newHead = (nint)headPtr->next; + return; + } - } while (Interlocked.CompareExchange(ref bucket->freeHead, newHead, head) != head); - - Interlocked.Decrement(ref bucket->freeCount); - return (void*)head; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private readonly void PushToBucket(int bucketIndex, void* ptr, nuint size) - { - var buckets = GetBuckets(); - var bucket = &buckets[bucketIndex]; - var node = (FreeNode*)ptr; - - node->size = size; + var remoteNode = (FreeNode*)blockStartPtr; + remoteNode->ownerChunk = chunk; + remoteNode->blockSize = blockSize; + remoteNode->bucketIndex = bucketIndex; nint head; do { - head = bucket->freeHead; - node->next = (FreeNode*)head; - - } while (Interlocked.CompareExchange(ref bucket->freeHead, (nint)node, head) != head); - - Interlocked.Increment(ref bucket->freeCount); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void AssignBlockHeader(BlockHeader* header, MemoryChunk* ownerChunk, nuint blockSize) - { - header->ownerChunk = ownerChunk; - header->blockSize = blockSize; - header->magicNumber = _MAGIC_NUMBER; - } - - private bool TryCreateBlocksForBucket(int bucketIndex) - { - var buckets = GetBuckets(); - var bucket = &buckets[bucketIndex]; - - while (Interlocked.CompareExchange(ref bucket->creationLock, 1, 0) != 0) - { - Thread.SpinWait(1); - } - - try - { - if (bucket->freeHead != 0) - { - return true; // Another thread did the work for us! - } - - var blockSize = bucket->blockSize; - var blocksToCreate = Math.Min(_chunkSize / blockSize, 256); // Limit number of blocks - - if (blocksToCreate == 0) - { - return false; - } - - var totalSize = blocksToCreate * blockSize; - var memory = (byte*)AlignedAlloc(totalSize, _alignment); - if (memory == null) - { - return false; - } - - var chunk = (MemoryChunk*)_chunkArena.Allocate(SizeOf(), AlignOf(), AllocationOption.None); - if (chunk == null) - { - AlignedFree(memory); - return false; - } - - chunk->memory = memory; - chunk->size = totalSize; - chunk->used = totalSize; - chunk->next = _chunks; - _chunks = chunk; - - // Add all blocks to the bucket's free list - for (nuint i = 0; i < blocksToCreate; i++) - { - var blockStartPtr = memory + (i * blockSize); - - AssignBlockHeader((BlockHeader*)blockStartPtr, chunk, blockSize); - PushToBucket(bucketIndex, blockStartPtr, blockSize); - } - - return true; - } - finally - { - Interlocked.Exchange(ref bucket->creationLock, 0); - } - } - - private void* AllocateFromChunk(nuint size, nuint alignment) - { - while (Interlocked.CompareExchange(ref _chunkCreationLock, 1, 0) != 0) - { - Thread.SpinWait(1); - } - - try - { - // Try to find space in existing chunks first - var chunk = _chunks; - while (chunk != null) - { - var available = chunk->size - chunk->used; - var alignedOffset = (chunk->used + alignment - 1) & ~(alignment - 1); - var totalNeeded = alignedOffset - chunk->used + size; - - if (totalNeeded <= available) - { - var blockStartPtr = chunk->memory + alignedOffset; - - // Write the header and return the pointer WITH the header - AssignBlockHeader((BlockHeader*)blockStartPtr, chunk, size); - return blockStartPtr; - } - - chunk = chunk->next; - } - - // Create new chunk - var newChunkSize = Math.Max(_chunkSize, size + alignment); - var newMemory = (byte*)AlignedAlloc(newChunkSize, alignment); - if (newMemory == null) - { - return null; - } - - var newChunk = (MemoryChunk*)_chunkArena.Allocate(SizeOf(), AlignOf(), AllocationOption.None); - if (newChunk == null) - { - AlignedFree(newMemory); - return null; - } - - newChunk->memory = newMemory; - newChunk->size = newChunkSize; - newChunk->used = size; - newChunk->next = _chunks; - _chunks = newChunk; - - // Write the header and return the pointer WITH the header - AssignBlockHeader((BlockHeader*)newMemory, newChunk, size); - return newMemory; - } - finally - { - Interlocked.Exchange(ref _chunkCreationLock, 0); - } + head = targetCache->remoteFreeHead; + remoteNode->next = (FreeNode*)head; + } while (Interlocked.CompareExchange(ref targetCache->remoteFreeHead, (nint)remoteNode, head) != head); } public void Dispose() { - if (Interlocked.CompareExchange(ref _disposed, 1, 0) == 0) + if (Interlocked.CompareExchange(ref _disposed, 1, 0) != 0) { - // Free all memory chunks - var chunk = _chunks; - while (chunk != null) - { - var next = chunk->next; - AlignedFree(chunk->memory); - chunk = next; - } - - _chunkArena.Dispose(); - - _chunks = null; - _totalAllocatedBytes = 0; - _totalFreeBytes = 0; + return; } + + if (_caches != null) + { + for (var i = 0; i <= _cacheCount; i++) + { + var cache = _caches[i]; + if (cache != null) + { + DrainRemoteFrees(cache); + cache->active = 0; + } + } + } + + var chunk = _chunks; + while (chunk != null) + { + var next = chunk->next; + AlignedFree(chunk->memory); + chunk = next; + } + + _chunkArena.Dispose(); + + if (_caches != null) + { + Free(_caches); + _caches = null; + } + + if (_instanceId != null) + { + Free(_instanceId); + _instanceId = null; + } + + _chunks = null; + _cacheCount = 0; } -} \ No newline at end of file +} diff --git a/Misaki.HighPerformance.LowLevel/Buffer/Stack.cs b/Misaki.HighPerformance.LowLevel/Buffer/Stack.cs index beae9dc..91d8e70 100644 --- a/Misaki.HighPerformance.LowLevel/Buffer/Stack.cs +++ b/Misaki.HighPerformance.LowLevel/Buffer/Stack.cs @@ -87,12 +87,19 @@ public unsafe partial struct Stack : IDisposable private void Init(nuint size) { + ArgumentOutOfRangeException.ThrowIfNegative(size); + if (_buffer != null) { Free(_buffer); } _buffer = (byte*)Malloc(size); + if (_buffer == null) + { + throw new OutOfMemoryException("Failed to allocate memory for the stack."); + } + _size = size; _offset = 0; _activeScopeCount = 0; @@ -103,15 +110,15 @@ public unsafe partial struct Stack : IDisposable s_locker.Enter(ref token); if (s_pStackBuffers == null) { - s_pStackBuffers = (void**)Malloc((nuint)sizeof(void*) * 4u); - s_stackCapacity = 4; + s_pStackBuffers = (void**)Malloc((nuint)(sizeof(void*) * Environment.ProcessorCount)); + s_stackCapacity = Environment.ProcessorCount; } if (s_stackCount >= s_stackCapacity) { var pOld = s_pStackBuffers; var newCapacity = s_stackCapacity * 2; - var pNew = (void**)Realloc(pOld, (nuint)sizeof(void*) * (nuint)newCapacity); + var pNew = (void**)Realloc(pOld, (nuint)(sizeof(void*) * newCapacity)); s_pStackBuffers = pNew; s_stackCapacity = newCapacity; diff --git a/Misaki.HighPerformance.LowLevel/Collections/HashMapHelper.cs b/Misaki.HighPerformance.LowLevel/Collections/HashMapHelper.cs index 515eead..891d9ef 100644 --- a/Misaki.HighPerformance.LowLevel/Collections/HashMapHelper.cs +++ b/Misaki.HighPerformance.LowLevel/Collections/HashMapHelper.cs @@ -131,15 +131,9 @@ public unsafe struct HashMapHelper : IDisposable public HashMapHelper(int capacity, int sizeOfTValue, int alignOfTValue, uint minGrowth, AllocationHandle handle, AllocationOption allocationOption) { - if (capacity <= 0) - { - throw new ArgumentOutOfRangeException(nameof(capacity), "Capacity must be greater than zero."); - } - - if (sizeOfTValue < 0 || alignOfTValue < 0) - { - throw new ArgumentOutOfRangeException(nameof(sizeOfTValue), "Size or alignment of TValue can not be less than zero."); - } + ArgumentOutOfRangeException.ThrowIfNegative(capacity); + ArgumentOutOfRangeException.ThrowIfNegative(sizeOfTValue); + ArgumentOutOfRangeException.ThrowIfNegative(alignOfTValue); _capacity = CalcCapacityCeilPow2(capacity); _bucketCapacity = _capacity * 2; diff --git a/Misaki.HighPerformance.LowLevel/Collections/UnsafeArray.cs b/Misaki.HighPerformance.LowLevel/Collections/UnsafeArray.cs index d062f39..15ad8c6 100644 --- a/Misaki.HighPerformance.LowLevel/Collections/UnsafeArray.cs +++ b/Misaki.HighPerformance.LowLevel/Collections/UnsafeArray.cs @@ -3,6 +3,7 @@ using Misaki.HighPerformance.LowLevel.Collections.Contracts; using Misaki.HighPerformance.LowLevel.Utilities; using System.Collections; using System.Diagnostics; +using System.Drawing; using System.Runtime.CompilerServices; namespace Misaki.HighPerformance.LowLevel.Collections; @@ -154,10 +155,7 @@ public unsafe struct UnsafeArray : IUnsafeCollection /// Thrown when the specified number of elements is less than or equal to zero. public UnsafeArray(int count, AllocationHandle handle, AllocationOption allocationOption = AllocationOption.None) { - if (count < 0) - { - throw new ArgumentOutOfRangeException(nameof(count), "Count can not be less than zero."); - } + ArgumentOutOfRangeException.ThrowIfNegative(count); if (handle.Alloc == null) { diff --git a/Misaki.HighPerformance.LowLevel/Misaki.HighPerformance.LowLevel.csproj b/Misaki.HighPerformance.LowLevel/Misaki.HighPerformance.LowLevel.csproj index 7f0d5e5..d2b03a5 100644 --- a/Misaki.HighPerformance.LowLevel/Misaki.HighPerformance.LowLevel.csproj +++ b/Misaki.HighPerformance.LowLevel/Misaki.HighPerformance.LowLevel.csproj @@ -7,7 +7,7 @@ true true Misaki - 1.4.4 + 1.4.5 $(AssemblyVersion) https://git.personalnas.com/Misaki/Misaki.HighPerformance.git https://git.personalnas.com/Misaki/Misaki.HighPerformance.git diff --git a/Misaki.HighPerformance.LowLevel/Utilities/MemoryUtility.cs b/Misaki.HighPerformance.LowLevel/Utilities/MemoryUtility.cs index 7e71aa8..699288b 100644 --- a/Misaki.HighPerformance.LowLevel/Utilities/MemoryUtility.cs +++ b/Misaki.HighPerformance.LowLevel/Utilities/MemoryUtility.cs @@ -21,18 +21,11 @@ public static unsafe partial class MemoryUtility [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void* Malloc(nuint size) { - try - { #if NET6_0_OR_GREATER - return NativeMemory.Alloc(size); + return NativeMemory.Alloc(size); #else - return Marshal.AllocHGlobal((IntPtr)size).ToPointer(); + return Marshal.AllocHGlobal((IntPtr)size).ToPointer(); #endif - } - catch (Exception) - { - return null; - } } /// @@ -43,20 +36,13 @@ public static unsafe partial class MemoryUtility [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void* Calloc(nuint size) { - try - { #if NET6_0_OR_GREATER - return NativeMemory.AllocZeroed(size); + return NativeMemory.AllocZeroed(size); #else - var ptr = Marshal.AllocHGlobal((IntPtr)size).ToPointer(); - Unsafe.InitBlock(ptr, 0, (uint)size); - return ptr; + var ptr = Marshal.AllocHGlobal((IntPtr)size).ToPointer(); + Unsafe.InitBlock(ptr, 0, (uint)size); + return ptr; #endif - } - catch (Exception) - { - return null; - } } /// @@ -68,18 +54,11 @@ public static unsafe partial class MemoryUtility [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void* AlignedAlloc(nuint size, nuint alignment) { - try - { #if NET6_0_OR_GREATER - return NativeMemory.AlignedAlloc(size, alignment); + return NativeMemory.AlignedAlloc(size, alignment); #else - return Marshal.AllocHGlobal((IntPtr)(size + alignment - 1)).ToPointer(); + return Marshal.AllocHGlobal((IntPtr)(size + alignment - 1)).ToPointer(); #endif - } - catch (Exception) - { - return null; - } } /// @@ -91,18 +70,11 @@ public static unsafe partial class MemoryUtility [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void* Realloc(void* ptr, nuint size) { - try - { #if NET6_0_OR_GREATER - return NativeMemory.Realloc(ptr, size); + return NativeMemory.Realloc(ptr, size); #else - return Marshal.ReAllocHGlobal((IntPtr)ptr, (IntPtr)size).ToPointer(); + return Marshal.ReAllocHGlobal((IntPtr)ptr, (IntPtr)size).ToPointer(); #endif - } - catch (Exception) - { - return null; - } } /// @@ -116,30 +88,23 @@ public static unsafe partial class MemoryUtility [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void* AlignedRealloc(void* ptr, nuint size, nuint alignment) { - try - { #if NET6_0_OR_GREATER - return NativeMemory.AlignedRealloc(ptr, size, alignment); + return NativeMemory.AlignedRealloc(ptr, size, alignment); #else - var newPtr = Marshal.AllocHGlobal((IntPtr)(size + alignment - 1)).ToPointer(); - if (newPtr == null) - { - return null; - } - - if (ptr != null) - { - Unsafe.CopyBlock(newPtr, ptr, (uint)size); - Marshal.FreeHGlobal((IntPtr)ptr); - } - - return newPtr; -#endif - } - catch (Exception) + var newPtr = Marshal.AllocHGlobal((IntPtr)(size + alignment - 1)).ToPointer(); + if (newPtr == null) { return null; } + + if (ptr != null) + { + Unsafe.CopyBlock(newPtr, ptr, (uint)size); + Marshal.FreeHGlobal((IntPtr)ptr); + } + + return newPtr; +#endif } /// diff --git a/Misaki.HighPerformance.Test/UnitTest/Buffer/TestAllocationManager.cs b/Misaki.HighPerformance.Test/UnitTest/Buffer/TestAllocationManager.cs index d0bad54..8f1b7e7 100644 --- a/Misaki.HighPerformance.Test/UnitTest/Buffer/TestAllocationManager.cs +++ b/Misaki.HighPerformance.Test/UnitTest/Buffer/TestAllocationManager.cs @@ -7,12 +7,6 @@ namespace Misaki.HighPerformance.Test.UnitTest.Buffer; [TestClass] public class TestAllocationManager { - [TestInitialize] - public void Initialize() - { - AllocationManager.EnableDebugLayer(); - } - [TestMethod] public void ShouldNotLeakTest() { diff --git a/Misaki.HighPerformance.Test/UnitTest/Buffer/TestFreeList.cs b/Misaki.HighPerformance.Test/UnitTest/Buffer/TestFreeList.cs new file mode 100644 index 0000000..43e5d31 --- /dev/null +++ b/Misaki.HighPerformance.Test/UnitTest/Buffer/TestFreeList.cs @@ -0,0 +1,151 @@ +using Misaki.HighPerformance.LowLevel.Buffer; +using System.Diagnostics; + +namespace Misaki.HighPerformance.Test.UnitTest.Buffer; + +[TestClass] +public unsafe class TestFreeList +{ + [TestMethod] + public void SingleThreadedAllocFreeTest() + { + using var freeList = new FreeList(8, 1024); + + // Allocate various sizes + void* p1 = freeList.Allocate(16, 8); + void* p2 = freeList.Allocate(32, 8); + void* p3 = freeList.Allocate(64, 8); + + Assert.IsTrue(p1 != null); + Assert.IsTrue(p2 != null); + Assert.IsTrue(p3 != null); + + // Free them + freeList.Free(p1); + freeList.Free(p2); + freeList.Free(p3); + + // Allocate again - should reuse from buckets (or at least succeed) + void* p4 = freeList.Allocate(16, 8); + void* p5 = freeList.Allocate(32, 8); + + Assert.IsTrue(p4 != null); + Assert.IsTrue(p5 != null); + + freeList.Free(p4); + freeList.Free(p5); + } + + [TestMethod] + public void MultiThreadedAllocSameThreadFreeTest() + { + const int threadCount = 8; + const int iterations = 1000; + using var freeList = new FreeList(8, 64 * 1024, threadCount); + + var threads = new Thread[threadCount]; + for (int i = 0; i < threadCount; i++) + { + threads[i] = new Thread(() => + { + for (int j = 0; j < iterations; j++) + { + void* ptr = freeList.Allocate(16, 8); + Assert.IsTrue(ptr != null); + freeList.Free(ptr); + } + }); + } + + foreach (var t in threads) t.Start(); + foreach (var t in threads) t.Join(); + } + + [TestMethod] + public void MultiThreadedCrossThreadFreeTest() + { + const int producerCount = 4; + const int consumerCount = 4; + const int iterations = 5000; + using var freeList = new FreeList(8, 64 * 1024, producerCount + consumerCount); + + var queue = new System.Collections.Concurrent.ConcurrentQueue(); + var producers = new Thread[producerCount]; + var consumers = new Thread[consumerCount]; + + bool producing = true; + + for (int i = 0; i < producerCount; i++) + { + producers[i] = new Thread(() => + { + for (int j = 0; j < iterations; j++) + { + void* ptr = freeList.Allocate(32, 8); + Assert.IsTrue(ptr != null); + queue.Enqueue((IntPtr)ptr); + } + }); + } + + for (int i = 0; i < consumerCount; i++) + { + consumers[i] = new Thread(() => + { + while (Volatile.Read(ref producing) || !queue.IsEmpty) + { + if (queue.TryDequeue(out var ptr)) + { + freeList.Free((void*)ptr); + } + else + { + Thread.Yield(); + } + } + }); + } + + foreach (var t in producers) t.Start(); + foreach (var t in consumers) t.Start(); + + foreach (var t in producers) t.Join(); + Volatile.Write(ref producing, false); + foreach (var t in consumers) t.Join(); + } + + [TestMethod] + public void OverflowCacheTest() + { + // Set maxConcurrencyLevel to 1, but use more threads + const int threadCount = 5; + using var freeList = new FreeList(8, 1024, 1); + + var threads = new Thread[threadCount]; + for (int i = 0; i < threadCount; i++) + { + threads[i] = new Thread(() => + { + void* ptr = freeList.Allocate(16, 8); + Assert.IsTrue(ptr != null); + freeList.Free(ptr); + }); + } + + foreach (var t in threads) t.Start(); + foreach (var t in threads) t.Join(); + } + + [TestMethod] + public void LargeAllocationTest() + { + using var freeList = new FreeList(8, 1024); + + // Allocate larger than default chunk size + nuint largeSize = 2048; + void* ptr = freeList.Allocate(largeSize, 8); + Assert.IsTrue(ptr != null); + + freeList.Free(ptr); + } +} diff --git a/Misaki.HighPerformance/Utilities/CollectionUtility.cs b/Misaki.HighPerformance/Utilities/CollectionUtility.cs index a023226..01f52f6 100644 --- a/Misaki.HighPerformance/Utilities/CollectionUtility.cs +++ b/Misaki.HighPerformance/Utilities/CollectionUtility.cs @@ -54,7 +54,7 @@ public static class CollectionUtility /// The zero-based index of the element to retrieve. /// A reference to the element at the specified index in the span. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ref readonly T GetElementUnsafe(this Span span, int index) + public static ref T GetElementUnsafe(this Span span, int index) { return ref Unsafe.Add(ref MemoryMarshal.GetReference(span), index); }