Misaki/Misaki.HighPerformance
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(allocator): add unified Reallocate to all allocators

Browse Source 
Introduce a unified Reallocate method to all memory allocator types (Arena, Stack, FreeList, VirtualArena, VirtualStack, DynamicArena) and require it in the IMemoryAllocator interface. This enables efficient resizing of memory blocks, with fast-path optimizations for stack-like allocators.

Update AllocationManager and MemoryPool to use the new Reallocate method, simplifying and optimizing memory resizing logic. Add public properties for buffer pointers, sizes, and offsets to allocator structs for easier diagnostics.

Set FreeList's default concurrency level to 1 and make its allocation method return null on dispose instead of throwing. Clean up vector types for formatting, fix UnsafeList's RemoveRangeSwapBack logic, and simplify RemoveAtSwapBack.

Simplify Program.cs to only run SPMDBenchmark. Add new unit tests for FixedString, UnsafeList, UnsafeHashMap, and UnsafeHashSet. Apply minor test code cleanups for consistency in TestUnsafeQueue.

BREAKING CHANGE: IMemoryAllocator now requires a Reallocate method, and allocator APIs have changed accordingly.
This commit is contained in:
Misaki
2026-04-04 14:16:52 +09:00
parent a95381e16d
commit 208e1aa975
19 changed files with 761 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
Misaki.HighPerformance.LowLevel/Buffer/AllocationManager.cs
View File

@@ -129,23 +129,8 @@ public static unsafe class AllocationManager
#endif
)
{
if (ptr == null)
{
return Allocate(instance, newSize, alignment, allocationOption
#if MHP_ENABLE_SAFETY_CHECKS
, pHandle
#endif
);
}
var selfPtr = (ArenaAllocator*)instance;
var newPtr = selfPtr->_arena.Allocate(newSize, alignment, allocationOption);
if (newPtr == null)
{
return null;
}
MemCpy(newPtr, ptr, Math.Min(oldSize, newSize));
var newPtr = selfPtr->_arena.Reallocate(ptr, oldSize, newSize, alignment, allocationOption);
#if MHP_ENABLE_SAFETY_CHECKS
*pHandle = new MemoryHandle(_ARENA_MAGIC_ID, selfPtr->_currentTick);
@@ -370,44 +355,8 @@ public static unsafe class AllocationManager
#endif
)
{
if (ptr == null)
{
return Allocate(instance, newSize, alignment, allocationOption
#if MHP_ENABLE_SAFETY_CHECKS
, pHandle
#endif
);
}
EnsureInitialize();
// Optimize for last allocation. Set offset directly.
var oldBase = s_stack.Buffer + s_stack.Offset - oldSize;
if (ptr == oldBase)
{
if (newSize > oldSize)
{
var diff = newSize - oldSize;
s_stack.Offset += diff;
if (allocationOption.HasFlag(AllocationOption.Clear))
{
MemClear(s_stack.Buffer + s_stack.Offset - diff, diff);
}
}
#if MHP_ENABLE_SAFETY_CHECKS
*pHandle = new MemoryHandle(_STACK_MAGIC_ID, (int)s_stack.Offset);
#endif
return ptr;
}
var newPtr = s_stack.Allocate(newSize, alignment, allocationOption);
if (newPtr == null)
{
return null;
}
MemCpy(newPtr, ptr, Math.Min(oldSize, newSize));
var newPtr = s_stack.Reallocate(ptr, oldSize, newSize, alignment, allocationOption);
#if MHP_ENABLE_SAFETY_CHECKS
*pHandle = new MemoryHandle(_STACK_MAGIC_ID, (int)s_stack.Offset);
@@ -491,25 +440,8 @@ public static unsafe class AllocationManager
#endif
)
{
if (ptr == null)
{
return Allocate(instance, newSize, alignment, allocationOption
#if MHP_ENABLE_SAFETY_CHECKS
, pHandle
#endif
);
}
var selfPtr = (FreeListAllocator*)instance;
var newPtr = selfPtr->_freeList.Allocate(newSize, alignment, allocationOption);
if (newPtr == null)
{
return null;
}
MemCpy(newPtr, ptr, Math.Min(oldSize, newSize));
selfPtr->_freeList.Free(ptr);
var newPtr = selfPtr->_freeList.Reallocate(ptr, oldSize, newSize, alignment, allocationOption);
#if MHP_ENABLE_SAFETY_CHECKS
RemoveAllocation(*pHandle);

47
Misaki.HighPerformance.LowLevel/Buffer/Arena.cs
View File

@@ -26,6 +26,10 @@ public unsafe struct Arena : IMemoryAllocator<Arena, Arena.CreationOptions>
[FieldOffset(16)]
private nuint _offset;
public readonly byte* Buffer => _buffer;
public readonly nuint Size => _size;
public readonly nuint Offset => _offset;
public Arena(nuint size)
{
ArgumentOutOfRangeException.ThrowIfNegative(size);
@@ -55,7 +59,7 @@ public unsafe struct Arena : IMemoryAllocator<Arena, Arena.CreationOptions>
{
if (_buffer == null)
{
throw new ObjectDisposedException(nameof(Arena));
return null;
}
if (size == 0)
@@ -92,6 +96,47 @@ public unsafe struct Arena : IMemoryAllocator<Arena, Arena.CreationOptions>
return ptr;
}
public void* Reallocate(void* ptr, nuint oldSize, nuint newSize, nuint alignment, AllocationOption allocationOption)
{
if (_buffer == null)
{
return null;
}
if (ptr == null)
{
return Allocate(newSize, alignment, allocationOption);
}
var additionalSize = newSize - oldSize;
var currentOffset = Volatile.Read(ref _offset);
if ((byte*)ptr + oldSize == _buffer + currentOffset)
{
if (currentOffset + additionalSize <= _size)
{
if (Interlocked.CompareExchange(ref _offset, currentOffset + additionalSize, currentOffset) == currentOffset)
{
if (allocationOption.HasFlag(AllocationOption.Clear) && additionalSize > 0)
{
MemClear((byte*)ptr + oldSize, additionalSize);
}
return ptr;
}
}
}
var newPtr = Allocate(newSize, alignment, allocationOption);
if (newPtr == null)
{
return null;
}
MemCpy(newPtr, ptr, Math.Min(oldSize, newSize));
return newPtr;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public readonly void Free(void* ptr)
{

27
Misaki.HighPerformance.LowLevel/Buffer/DynamicArena.cs
View File

@@ -7,14 +7,14 @@ namespace Misaki.HighPerformance.LowLevel.Buffer;
/// A dynamic memory management structure that automatically grows by creating linked arenas when more space is needed.
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 128)]
public unsafe struct DynamicArena : IMemoryAllocator<DynamicArena, DynamicArena.CreateOptions>
public unsafe struct DynamicArena : IMemoryAllocator<DynamicArena, DynamicArena.CreationOptions>
{
public struct CreateOptions
public struct CreationOptions
{
public uint initialSize;
}
public static DynamicArena Create(in CreateOptions options)
public static DynamicArena Create(in CreationOptions options)
{
return new DynamicArena(options.initialSize);
}
@@ -134,6 +134,27 @@ public unsafe struct DynamicArena : IMemoryAllocator<DynamicArena, DynamicArena.
return result;
}
public void* Reallocate(void* ptr, nuint oldSize, nuint newSize, nuint alignment, AllocationOption allocationOption)
{
if (ptr == null)
{
return Allocate(newSize, alignment, allocationOption);
}
var newPtr = Allocate(newSize, alignment, allocationOption);
if (newPtr == null)
{
return null;
}
if (newPtr != ptr)
{
MemCpy(newPtr, ptr, Math.Min(oldSize, newSize));
}
return newPtr;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public readonly void Free(void* ptr)
{

28
Misaki.HighPerformance.LowLevel/Buffer/FreeList.cs
View File

@@ -7,16 +7,16 @@ namespace Misaki.HighPerformance.LowLevel.Buffer;
/// A variable-size allocator that uses per-thread caches for the hot path and a remote-free queue for cross-thread deallocation.
/// </summary>
[StructLayout(LayoutKind.Sequential)]
public unsafe struct FreeList : IMemoryAllocator<FreeList, FreeList.CreationOpts>
public unsafe struct FreeList : IMemoryAllocator<FreeList, FreeList.CreationOptions>
{
public struct CreationOpts
public struct CreationOptions
{
public nuint alignment;
public nuint chunkSize;
public int maxConcurrencyLevel;
}
public static FreeList Create(in CreationOpts opts)
public static FreeList Create(in CreationOptions opts)
{
return new FreeList(opts.alignment, opts.chunkSize, opts.maxConcurrencyLevel);
}
@@ -75,7 +75,7 @@ public unsafe struct FreeList : IMemoryAllocator<FreeList, FreeList.CreationOpts
}
private const int _MAX_BUCKETS = 16;
private const int _DEFAULT_MAX_CONCURRENCY_LEVEL = 16;
private const int _DEFAULT_MAX_CONCURRENCY_LEVEL = 1;
private const int _OVERFLOW_CACHE_INDEX = 0;
private const nuint _MIN_BLOCK_SIZE = 16;
private const nuint _DEFAULT_CHUNK_SIZE = 64 * 1024;
@@ -506,7 +506,7 @@ public unsafe struct FreeList : IMemoryAllocator<FreeList, FreeList.CreationOpts
{
if (_disposed != 0)
{
throw new ObjectDisposedException(nameof(FreeList));
return null;
}
if (size == 0)
@@ -590,6 +590,24 @@ public unsafe struct FreeList : IMemoryAllocator<FreeList, FreeList.CreationOpts
}
}
public void* Reallocate(void* ptr, nuint oldSize, nuint newSize, nuint alignment, AllocationOption allocationOption = AllocationOption.None)
{
if (_disposed != 0)
{
return null;
}
var newPtr = Allocate(newSize, alignment, allocationOption);
if (newPtr != null && ptr != null)
{
var copySize = Math.Min(oldSize, newSize);
MemCpy(newPtr, ptr, copySize);
Free(ptr);
}
return newPtr;
}
/// <summary>
/// Frees a previously allocated memory block.
/// </summary>

1
Misaki.HighPerformance.LowLevel/Buffer/IAllocator.cs
View File

@@ -123,5 +123,6 @@ public unsafe interface IMemoryAllocator<TSelf, TOpts> : IDisposable
static abstract TSelf Create(in TOpts opts);
void* Allocate(nuint size, nuint alignment, AllocationOption option = AllocationOption.None);
void* Reallocate(void* ptr, nuint oldSize, nuint newSize, nuint alignment, AllocationOption allocationOption = AllocationOption.None);
void Free(void* ptr);
}

28
Misaki.HighPerformance.LowLevel/Buffer/MemoryPool.cs
View File

@@ -41,33 +41,7 @@ public unsafe struct MemoryPool<T, TOpts> : IDisposable
#endif
)
{
if (ptr == null)
{
return Allocate(pAllocator, newSize, alignment, allocationOption
#if MHP_ENABLE_SAFETY_CHECKS
, pHandle
#endif
);
}
var newPtr = Allocate(pAllocator, newSize, alignment, allocationOption
#if MHP_ENABLE_SAFETY_CHECKS
, pHandle
#endif
);
if (newPtr == null)
{
return null;
}
MemCpy(newPtr, ptr, Math.Min(oldSize, newSize));
Free(pAllocator, ptr
#if MHP_ENABLE_SAFETY_CHECKS
, *pHandle
#endif
);
return newPtr;
return ((T*)pAllocator)->Reallocate(ptr, oldSize, newSize, alignment, allocationOption);
}
private static void Free(void* pAllocator, void* ptr

55
Misaki.HighPerformance.LowLevel/Buffer/Stack.cs
View File

@@ -1,4 +1,4 @@
using System.Diagnostics;
using System.Collections;
using System.Runtime.CompilerServices;
namespace Misaki.HighPerformance.LowLevel.Buffer;
@@ -8,14 +8,14 @@ namespace Misaki.HighPerformance.LowLevel.Buffer;
/// blocks within a preallocated buffer.
/// </summary>
/// <remarks>This is not a thread-safe implementation.</remarks>
public unsafe partial struct Stack : IMemoryAllocator<Stack, Stack.CreationOpts>
public unsafe partial struct Stack : IMemoryAllocator<Stack, Stack.CreationOptions>
{
public struct CreationOpts
public struct CreationOptions
{
public nuint size;
}
public static Stack Create(in CreationOpts opts)
public static Stack Create(in CreationOptions opts)
{
return new Stack(opts.size);
}
@@ -57,12 +57,9 @@ public unsafe partial struct Stack : IMemoryAllocator<Stack, Stack.CreationOpts>
private uint _activeScopeCount;
#endif
internal readonly byte* Buffer => _buffer;
internal nuint Offset
{
readonly get => _offset;
set => _offset = value;
}
public readonly byte* Buffer => _buffer;
public readonly nuint Size => _size;
public readonly nuint Offset => _offset;
/// <summary>
/// Initializes a new instance of the StackAllocator class with a buffer of the specified size.
@@ -141,6 +138,44 @@ public unsafe partial struct Stack : IMemoryAllocator<Stack, Stack.CreationOpts>
return ptr;
}
public void* Reallocate(void* ptr, nuint oldSize, nuint newSize, nuint alignment, AllocationOption allocationOption)
{
if (_buffer == null)
{
return null;
}
if (ptr == null)
{
return Allocate(newSize, alignment, allocationOption);
}
var oldBase = _buffer + _offset - oldSize;
if (ptr == oldBase)
{
if (newSize > oldSize)
{
var diff = newSize - oldSize;
_offset += diff;
if (allocationOption.HasFlag(AllocationOption.Clear))
{
MemClear(_buffer + _offset - diff, diff);
}
}
return ptr;
}
var newPtr = Allocate(newSize, alignment, allocationOption);
if (newPtr == null)
{
return null;
}
MemCpy(newPtr, ptr, Math.Min(oldSize, newSize));
return newPtr;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public readonly void Free(void* ptr)
{

170
Misaki.HighPerformance.LowLevel/Buffer/VirtualArena.cs
View File

@@ -12,6 +12,7 @@ public unsafe struct VirtualArena : IMemoryAllocator<VirtualArena, VirtualArena.
{
public nuint reserveCapacity;
}
public static VirtualArena Create(in CreationOptions opts)
{
return new VirtualArena(opts.reserveCapacity);
@@ -24,7 +25,12 @@ public unsafe struct VirtualArena : IMemoryAllocator<VirtualArena, VirtualArena.
private nuint _committedSize;
private nuint _allocatedOffset;
private volatile int _allocationLock;
private int _allocationLock;
public readonly byte* Buffer => _baseAddress;
public readonly nuint Reserved => _reserveCapacity;
public readonly nuint Committed => _committedSize;
public readonly nuint Allocated => _allocatedOffset;
public VirtualArena(nuint reserveCapacity)
{
@@ -52,56 +58,162 @@ public unsafe struct VirtualArena : IMemoryAllocator<VirtualArena, VirtualArena.
/// <returns>A pointer to the allocated memory block if the allocation succeeds, otherwise null.</returns>
public void* Allocate(nuint size, nuint alignment, AllocationOption allocationOption)
{
while (Interlocked.CompareExchange(ref _allocationLock, 1, 0) != 0)
if (_baseAddress == null || size == 0)
{
Thread.SpinWait(1);
return null;
}
void* ptr;
var spinWait = new SpinWait();
try
while (true)
{
// Align the requested offset
var alignedOffset = (_allocatedOffset + alignment - 1) & ~(alignment - 1);
var newAllocatedOffset = alignedOffset + size;
var currentOffset = Volatile.Read(ref _allocatedOffset);
if (newAllocatedOffset > _reserveCapacity)
// Align the requested offset
var alignedOffset = (currentOffset + alignment - 1) & ~(alignment - 1);
var newOffset = alignedOffset + size;
if (newOffset > _reserveCapacity)
{
return null; // Out of reserved space
return null;
}
if (newAllocatedOffset > _committedSize)
if (newOffset <= Volatile.Read(ref _committedSize))
{
var sizeToCommit = newAllocatedOffset - _committedSize;
// Align the commit size to the 64KB OS Page Size
sizeToCommit = (sizeToCommit + _PAGE_SIZE - 1) & ~(_PAGE_SIZE - 1);
var commitAddress = _baseAddress + _committedSize;
var result = Mmap(commitAddress, sizeToCommit, VirtualAllocationFlags.Commit);
if (result == null)
// Try to atomically claim this space.
if (Interlocked.CompareExchange(ref _allocatedOffset, newOffset, currentOffset) == currentOffset)
{
return null; // Out of physical RAM
var ptr = _baseAddress + alignedOffset;
if (allocationOption.HasFlag(AllocationOption.Clear))
{
MemClear(ptr, size);
}
return ptr;
}
_committedSize += sizeToCommit;
spinWait.SpinOnce();
continue;
}
ptr = _baseAddress + alignedOffset;
_allocatedOffset = newAllocatedOffset;
var lockWait = new SpinWait();
while (Interlocked.CompareExchange(ref _allocationLock, 1, 0) != 0)
{
lockWait.SpinOnce();
}
try
{
// DOUBLE-CHECK: Did another thread commit enough memory while we were waiting for the lock?
var currentCommitted = _committedSize;
if (newOffset > currentCommitted)
{
var sizeToCommit = newOffset - currentCommitted;
sizeToCommit = (sizeToCommit + _PAGE_SIZE - 1) & ~(_PAGE_SIZE - 1);
var commitAddress = _baseAddress + currentCommitted;
var result = Mmap(commitAddress, sizeToCommit, VirtualAllocationFlags.Commit);
if (result == null)
{
return null;
}
Volatile.Write(ref _committedSize, currentCommitted + sizeToCommit);
}
}
finally
{
Volatile.Write(ref _allocationLock, 0);
}
// We committed the memory (or realized someone else did).
// Loop back up to try the lock-free allocation again!
}
finally
}
public void* Reallocate(void* ptr, nuint oldSize, nuint newSize, nuint alignment, AllocationOption allocationOption)
{
if (_baseAddress == null || newSize == 0)
{
Interlocked.Exchange(ref _allocationLock, 0);
return null;
}
if (allocationOption.HasFlag(AllocationOption.Clear))
if (newSize <= oldSize)
{
MemClear(ptr, size);
return ptr;
}
return ptr;
if (ptr == null)
{
return Allocate(newSize, alignment, allocationOption);
}
var additionalSize = newSize - oldSize;
var currentOffset = Volatile.Read(ref _allocatedOffset);
// Fast-path: Check if it's the last allocated block
if ((byte*)ptr + oldSize == _baseAddress + currentOffset)
{
var newOffset = currentOffset + additionalSize;
// Check if we need to commit more physical memory
if (newOffset > Volatile.Read(ref _committedSize))
{
var spinWait = new SpinWait();
while (Interlocked.CompareExchange(ref _allocationLock, 1, 0) != 0)
{
spinWait.SpinOnce();
}
try
{
// DOUBLE CHECK: Did another thread commit the memory while we waited?
var currentCommitted = _committedSize;
if (newOffset > currentCommitted)
{
var sizeToCommit = newOffset - currentCommitted;
sizeToCommit = (sizeToCommit + _PAGE_SIZE - 1) & ~(_PAGE_SIZE - 1);
var commitAddress = _baseAddress + currentCommitted;
var result = Mmap(commitAddress, sizeToCommit, VirtualAllocationFlags.Commit);
if (result == null)
{
return null; // OOM or mapping failure
}
Volatile.Write(ref _committedSize, currentCommitted + sizeToCommit);
}
}
finally
{
Volatile.Write(ref _allocationLock, 0);
}
}
// Try to atomically extend the block
if (Interlocked.CompareExchange(ref _allocatedOffset, newOffset, currentOffset) == currentOffset)
{
// Safe to clear: we own the space between oldSize and newOffset
if (allocationOption.HasFlag(AllocationOption.Clear) && additionalSize > 0)
{
MemClear((byte*)ptr + oldSize, additionalSize);
}
return ptr;
}
}
var newPtr = Allocate(newSize, alignment, allocationOption);
if (newPtr == null)
{
return null;
}
MemCpy(newPtr, ptr, oldSize);
return newPtr;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]

86
Misaki.HighPerformance.LowLevel/Buffer/VirtualStack.cs
View File

@@ -1,19 +1,18 @@
using Misaki.HighPerformance.LowLevel.Utilities;
using System.Diagnostics;
using System.Runtime.CompilerServices;
namespace Misaki.HighPerformance.LowLevel.Buffer;
public unsafe struct VirtualStack : IMemoryAllocator<VirtualStack, VirtualStack.CreationOpts>
public unsafe struct VirtualStack : IMemoryAllocator<VirtualStack, VirtualStack.CreationOptions>
{
private const nuint _PAGE_SIZE = 64 * 1024;
public struct CreationOpts
public struct CreationOptions
{
public nuint reserveCapacity;
}
public static VirtualStack Create(in CreationOpts opts)
public static VirtualStack Create(in CreationOptions opts)
{
return new VirtualStack(opts.reserveCapacity);
}
@@ -56,12 +55,10 @@ public unsafe struct VirtualStack : IMemoryAllocator<VirtualStack, VirtualStack.
private uint _activeScopeCount;
#endif
internal readonly byte* Buffer => _baseAddress;
internal nuint Offset
{
readonly get => _allocatedOffset;
set => _allocatedOffset = value;
}
public readonly byte* Buffer => _baseAddress;
public readonly nuint Reserved => _reserveCapacity;
public readonly nuint Committed => _committedSize;
public readonly nuint Allocated => _allocatedOffset;
public VirtualStack(nuint reserveCapacity)
{
@@ -120,7 +117,7 @@ public unsafe struct VirtualStack : IMemoryAllocator<VirtualStack, VirtualStack.
if (newAllocatedOffset > _reserveCapacity)
{
return null; // Out of reserved space
return null;
}
if (newAllocatedOffset > _committedSize)
@@ -152,6 +149,68 @@ public unsafe struct VirtualStack : IMemoryAllocator<VirtualStack, VirtualStack.
return userPtr;
}
public void* Reallocate(void* ptr, nuint oldSize, nuint newSize, nuint alignment, AllocationOption allocationOption)
{
if (_baseAddress == null)
{
return null;
}
if (newSize < oldSize)
{
return ptr;
}
if (ptr == null)
{
return Allocate(newSize, alignment, allocationOption);
}
if ((byte*)ptr + oldSize == _baseAddress + _allocatedOffset)
{
var diff = newSize - oldSize;
_allocatedOffset += diff;
if (_allocatedOffset > _committedSize)
{
var sizeToCommit = _allocatedOffset - _committedSize;
// Align the commit size to the 64KB OS Page Size
sizeToCommit = (sizeToCommit + _PAGE_SIZE - 1) & ~(_PAGE_SIZE - 1);
var commitAddress = _baseAddress + _committedSize;
var result = Mmap(commitAddress, sizeToCommit, VirtualAllocationFlags.Commit);
if (result == null)
{
return null;
}
_committedSize += sizeToCommit;
}
if (allocationOption.HasFlag(AllocationOption.Clear))
{
MemClear(_baseAddress + _allocatedOffset - diff, diff);
}
}
var newPtr = Allocate(newSize, alignment, allocationOption);
if (newPtr == null)
{
return null;
}
MemCpy(newPtr, ptr, Math.Min(oldSize, newSize));
return newPtr;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public readonly void Free(void* ptr)
{
}
/// <summary>
/// Resets the internal offset to its initial position, keeping the committed physical memory intact for future reuse.
/// </summary>
@@ -161,11 +220,6 @@ public unsafe struct VirtualStack : IMemoryAllocator<VirtualStack, VirtualStack.
_allocatedOffset = 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public readonly void Free(void* ptr)
{
}
public void Dispose()
{
if (_baseAddress != null)