using Misaki.HighPerformance.Collections;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace Misaki.HighPerformance.LowLevel.Buffer;
///
/// Holds information about a memory allocation.
///
public readonly struct AllocationInfo
{
///
/// Get the size of the allocation in bytes.
///
public nuint Size
{
get; init;
}
///
/// Get the stack trace at the time of allocation for debugging purposes.
///
public StackTrace StackTrace
{
get; init;
}
}
///
/// Provides memory allocation management for native memory allocations, with support for tracking,
/// debugging, and custom allocation strategies.
///
public static unsafe class AllocationManager
{
#if ENABLE_DEBUG_LAYER
[StructLayout(LayoutKind.Sequential)]
private struct AllocationHeader
{
public AllocationHeader* prev;
public AllocationHeader* next;
public void* basePtr; // pointer returned by underlying allocator
public nuint userSize; // requested size from the user
public GCHandle stackHandle; // GCHandle to managed StackTrace
}
#endif
private struct ArenaAllocator : IAllocator, IDisposable
{
private const int _ARENA_MAGIC_ID = -3941029;
private DynamicArena _arena;
private AllocationHandle _handle;
private int _currentTick;
public readonly AllocationHandle Handle => _handle;
public readonly int CurrentTick => _currentTick;
public void Init(uint initialSize)
{
_arena = new DynamicArena(initialSize);
_handle = new AllocationHandle
{
State = Unsafe.AsPointer(ref this),
Alloc = &Allocate,
Realloc = &Reallocate,
Free = null,
IsValid = &IsValid
};
_currentTick = 0;
}
private static void* Allocate(void* instance, nuint size, nuint alignment, AllocationOption allocationOption, MemoryHandle* pHandle)
{
var selfPtr = (ArenaAllocator*)instance;
var ptr = selfPtr->_arena.Allocate(size, alignment, allocationOption);
if (ptr == null)
{
*pHandle = MemoryHandle.Invalid;
return null;
}
*pHandle = new MemoryHandle(_ARENA_MAGIC_ID, selfPtr->_currentTick);
return ptr;
}
private static void* Reallocate(void* instance, void* ptr, nuint oldSize, nuint newSize, nuint alignment, AllocationOption allocationOption, MemoryHandle* pHandle)
{
if (ptr == null)
{
return Allocate(instance, newSize, alignment, allocationOption, pHandle);
}
var selfPtr = (ArenaAllocator*)instance;
var newPtr = selfPtr->_arena.Allocate(newSize, alignment, allocationOption);
if (newPtr == null)
{
return null;
}
MemCpy(newPtr, ptr, Math.Min(oldSize, newSize));
*pHandle = new MemoryHandle(_ARENA_MAGIC_ID, selfPtr->_currentTick);
return newPtr;
}
private static bool IsValid(void* instance, MemoryHandle handle)
{
var selfPtr = (ArenaAllocator*)instance;
return handle.id == _ARENA_MAGIC_ID && handle.generation == selfPtr->_currentTick;
}
public void Reset()
{
_arena.Reset();
_currentTick++;
}
public void Dispose()
{
_arena.Dispose();
}
}
private struct HeapAllocator : IAllocator
{
private AllocationHandle _handle;
public readonly AllocationHandle Handle => _handle;
public void Init()
{
_handle = new AllocationHandle
{
State = null,
Alloc = &Allocate,
Realloc = &Reallocate,
Free = &Free,
IsValid = &IsValid
};
}
private static void* Allocate(void* _, nuint size, nuint alignment, AllocationOption allocationOption, MemoryHandle* pHandle)
{
return HeapAlloc(size, alignment, allocationOption, pHandle);
}
private static void* Reallocate(void* _, void* ptr, nuint oldSize, nuint newSize, nuint alignment, AllocationOption allocationOption, MemoryHandle* pHandle)
{
if (ptr == null)
{
return Allocate(null, newSize, alignment, allocationOption, pHandle);
}
MemoryHandle newHandle;
var newPtr = HeapAlloc(newSize, alignment, allocationOption, &newHandle);
if (newPtr == null)
{
return null;
}
MemCpy(newPtr, ptr, Math.Min(oldSize, newSize));
HeapFree(ptr, *pHandle);
*pHandle = newHandle;
return newPtr;
}
private static void Free(void* _, void* ptr, MemoryHandle handle)
{
HeapFree(ptr, handle);
}
private static bool IsValid(void* _, MemoryHandle handle)
{
return ContainsAllocation(handle);
}
}
private struct StackAllocator : IAllocator, IDisposable
{
private const int _STACK_MAGIC_ID = -6843541;
[ThreadStatic]
private static Stack s_stack;
private AllocationHandle _handle;
public readonly AllocationHandle Handle => _handle;
public void Init()
{
_handle = new AllocationHandle
{
State = Unsafe.AsPointer(ref this),
Alloc = &Allocate,
Realloc = &Reallocate,
Free = null,
IsValid = &IsValid
};
}
private static void* Allocate(void* instance, nuint size, nuint alignment, AllocationOption allocationOption, MemoryHandle* pHandle)
{
var ptr = s_stack.Allocate(size, alignment, allocationOption);
if (ptr == null)
{
*pHandle = MemoryHandle.Invalid;
return null;
}
*pHandle = new MemoryHandle(_STACK_MAGIC_ID, (int)s_stack.Offset);
return ptr;
}
private static void* Reallocate(void* instance, void* ptr, nuint oldSize, nuint newSize, nuint alignment, AllocationOption allocationOption, MemoryHandle* pHandle)
{
if (ptr == null)
{
return Allocate(instance, newSize, alignment, allocationOption, pHandle);
}
// Optimize for last allocation. Set offset directly.
var oldBase = s_stack.Buffer + s_stack.Offset - oldSize;
if (ptr == oldBase)
{
if (newSize > oldSize)
{
var diff = newSize - oldSize;
s_stack.Offset += diff;
if (allocationOption.HasFlag(AllocationOption.Clear))
{
MemClear(s_stack.Buffer + s_stack.Offset - diff, diff);
}
}
*pHandle = new MemoryHandle(_STACK_MAGIC_ID, (int)s_stack.Offset);
return ptr;
}
var newPtr = s_stack.Allocate(newSize, alignment, allocationOption);
if (newPtr == null)
{
return null;
}
MemCpy(newPtr, ptr, Math.Min(oldSize, newSize));
*pHandle = new MemoryHandle(_STACK_MAGIC_ID, (int)s_stack.Offset);
return newPtr;
}
private static bool IsValid(void* instance, MemoryHandle handle)
{
return handle.id == _STACK_MAGIC_ID && handle.generation <= (int)s_stack.Offset;
}
public static Stack.Scope CreateScope(StackAllocator* pSelf)
{
return s_stack.CreateScope(pSelf->_handle);
}
public readonly void Dispose()
{
Stack.DisposeAll();
}
}
private const uint _DEFAULT_MEMORY_POOL_SIZE = 1024 * 1024; // 1 MB
private static readonly ArenaAllocator* s_pArenaAllocator;
private static readonly HeapAllocator* s_pHeapAllocator;
private static readonly StackAllocator* s_pStackAllocator;
private static bool s_disposed;
#if ENABLE_DEBUG_LAYER
private static SpinLock s_liveLock;
private static AllocationHeader* s_pLiveHead;
#endif
private static readonly ConcurrentSlotMap s_allocations;
public static readonly MemoryHandle MagicHandle = new MemoryHandle(int.MinValue, int.MinValue);
///
/// Gets the number of live persistent heap allocations when the debug layer is disabled.
///
public static int LiveAllocationCount => s_allocations.Count;
static AllocationManager()
{
var allocatorTotalSize = (nuint)(sizeof(ArenaAllocator) + sizeof(HeapAllocator) + sizeof(StackAllocator));
var basePtr = Malloc(allocatorTotalSize);
s_pArenaAllocator = (ArenaAllocator*)basePtr;
s_pHeapAllocator = (HeapAllocator*)((byte*)basePtr + (nuint)sizeof(ArenaAllocator));
s_pStackAllocator = (StackAllocator*)((byte*)basePtr + (nuint)(sizeof(ArenaAllocator) + sizeof(HeapAllocator)));
#if ENABLE_DEBUG_LAYER
s_liveLock = new SpinLock(false);
s_pLiveHead = null;
#endif
s_allocations = new ConcurrentSlotMap(256);
s_pArenaAllocator->Init(_DEFAULT_MEMORY_POOL_SIZE);
s_pHeapAllocator->Init();
s_pStackAllocator->Init();
}
#if ENABLE_DEBUG_LAYER
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static byte* AlignUp(byte* p, nuint alignment)
{
var a = alignment == 0 ? (nuint)IntPtr.Size : alignment;
return (byte*)(((nuint)p + (a - 1)) & ~(a - 1));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static GCHandle HeaderGetHandle(AllocationHeader* header)
{
return header->stackHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void HeaderSetHandle(AllocationHeader* header, GCHandle handle)
{
header->stackHandle = handle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void HeaderFreeHandle(AllocationHeader* header)
{
if (header->stackHandle.IsAllocated)
{
header->stackHandle.Free();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void LinkHeader(AllocationHeader* header)
{
var taken = false;
try
{
s_liveLock.Enter(ref taken);
header->prev = null;
header->next = s_pLiveHead;
if (s_pLiveHead != null)
{
s_pLiveHead->prev = header;
}
s_pLiveHead = header;
}
finally
{
if (taken)
{
s_liveLock.Exit();
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void UnlinkHeader(AllocationHeader* header)
{
var taken = false;
try
{
s_liveLock.Enter(ref taken);
var prev = header->prev;
var next = header->next;
if (prev != null)
{
prev->next = next;
}
else
{
s_pLiveHead = next;
}
if (next != null)
{
next->prev = prev;
}
header->prev = header->next = null;
}
finally
{
if (taken)
{
s_liveLock.Exit();
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void* DebugAllocate(nuint size, nuint alignment)
{
// Over-allocate to fit header + alignment padding; we align the user pointer, header is placed just before it.
var pad = alignment == 0 ? (nuint)IntPtr.Size : alignment;
var total = size + (nuint)sizeof(AllocationHeader) + (pad - 1);
var basePtr = AlignedAlloc(total, pad);
var user = AlignUp((byte*)basePtr + (nuint)sizeof(AllocationHeader), pad);
var header = (AllocationHeader*)(user - (nuint)sizeof(AllocationHeader));
header->basePtr = basePtr;
header->userSize = size;
HeaderSetHandle(header, GCHandle.Alloc(new StackTrace(2, true)));
LinkHeader(header);
return user;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void DebugFree(void* userPtr)
{
var header = (AllocationHeader*)((byte*)userPtr - (nuint)sizeof(AllocationHeader));
UnlinkHeader(header);
HeaderFreeHandle(header);
AlignedFree(header->basePtr);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void* DebugReallocate(void* userPtr, nuint oldSize, nuint newSize, nuint alignment, AllocationOption allocationOption)
{
if (userPtr == null)
{
return DebugAllocate(newSize, alignment);
}
var oldHeader = (AllocationHeader*)((byte*)userPtr - (nuint)sizeof(AllocationHeader));
var handle = HeaderGetHandle(oldHeader); // preserve original allocation StackTrace
var pad = alignment == 0 ? (nuint)IntPtr.Size : alignment;
var total = newSize + (nuint)sizeof(AllocationHeader) + (pad - 1);
var newBase = AlignedAlloc(total, pad);
var newUser = AlignUp((byte*)newBase + (nuint)sizeof(AllocationHeader), pad);
var newHeader = (AllocationHeader*)(newUser - (nuint)sizeof(AllocationHeader));
newHeader->basePtr = newBase;
newHeader->userSize = newSize;
HeaderSetHandle(newHeader, handle); // transfer ownership to the new header
LinkHeader(newHeader);
// Mirror original behavior: copy newSize bytes
MemCpy(newUser, userPtr, newSize);
if (allocationOption.HasFlag(AllocationOption.Clear) && newSize > oldSize)
{
MemClear(newUser + oldSize, newSize - oldSize);
}
// Unlink and free the old block (without freeing the StackTrace pHandle again)
//oldHeader->stackHandle = GCHandle.FromIntPtr(0);
UnlinkHeader(oldHeader);
AlignedFree(oldHeader->basePtr);
return newUser;
}
#endif
///
/// Gets a reference to the allocation pHandle for the specified allocator type.
///
/// The allocator type for which to retrieve the allocation pHandle.
/// A reference to the allocation pHandle associated with the specified allocator type.
///
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static AllocationHandle GetAllocationHandle(Allocator allocator)
{
return allocator switch
{
Allocator.Temp => s_pArenaAllocator->Handle,
Allocator.Persistent => s_pHeapAllocator->Handle,
_ => throw new ArgumentException("Target allocator type does not support custom allocation.", nameof(allocator)),
};
}
///
/// Allocates a block of memory from the heap with the specified size and alignment, using the given allocation options.
///
/// The number of bytes to allocate. Must be greater than zero.
/// The alignment, in bytes, for the allocated memory block. Must be a power of two.
/// An optional set of flags that control allocation behavior, such as whether the memory should be cleared or
/// tracked. The default is .
/// A pointer to the beginning of the allocated memory block.
/// Thrown if the allocation fails.
public static void* HeapAlloc(nuint size, nuint alignment, AllocationOption allocationOption, MemoryHandle* pHandle)
{
#if ENABLE_DEBUG_LAYER
var ptr = DebugAllocate(size, alignment);
#else
var ptr = AlignedAlloc(size, alignment);
#endif
if (ptr == null)
{
*pHandle = MemoryHandle.Invalid;
return null;
}
if (allocationOption.HasFlag(AllocationOption.Clear))
{
MemClear(ptr, size);
}
*pHandle = AddAllocation((IntPtr)ptr);
return ptr;
}
///
/// Releases a block of unmanaged memory previously allocated by the heap allocator.
///
/// A pointer to the memory block to be freed. The pointer must have been returned by a compatible heap allocation
/// method and must not be null.
/// The handle representing the memory allocation to free. The handle must be valid and previously allocated.
public static void HeapFree(void* ptr, MemoryHandle handle)
{
#if ENABLE_DEBUG_LAYER
if (handle != MagicHandle)
{
DebugFree(ptr);
}
else
#endif
{
AlignedFree(ptr);
}
RemoveAllocation(handle);
}
///
/// Releases a block of unmanaged memory previously allocated by the heap allocator.
///
/// The handle representing the memory allocation to free. The handle must be valid and previously allocated.
public static void HeapFree(MemoryHandle handle)
{
if (TryGetAllocation(handle, out var ptr))
{
HeapFree((void*)ptr, handle);
}
}
///
/// Resets the temporary memory allocator, clearing all allocated memory.
///
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ResetTempAllocator()
{
s_pArenaAllocator->Reset();
}
///
/// Creates a new thread local stack scope for managing temporary allocations within the current context.
///
/// A instance representing the newly created stack scope. The scope must be disposed when no longer needed to release allocated resources.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Stack.Scope CreateStackScope()
{
return StackAllocator.CreateScope(s_pStackAllocator);
}
///
/// Registers a memory allocation and returns a handle that can be used to manage or reference the allocated memory.
///
/// A pointer to the memory block to be registered. The pointer must reference a valid, allocated memory region.
/// A MemoryHandle representing the registered allocation.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static MemoryHandle AddAllocation(IntPtr ptr)
{
var id = s_allocations.Add(ptr, out var generation);
return new MemoryHandle(id, generation);
}
///
/// Removes the memory allocation associated with the specified handle.
///
/// The handle representing the memory allocation to remove. The handle must be valid and previously allocated.
/// true if the allocation was successfully removed; otherwise, false.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool RemoveAllocation(MemoryHandle handle)
{
return s_allocations.Remove(handle.id, handle.generation);
}
///
/// Attempts to retrieve the memory allocation pointer associated with the specified handle.
///
/// The memory handle identifying the allocation to retrieve allocation.
/// When this method returns, contains the pointer to the memory allocation if found; otherwise, .
/// true if the allocation was found and contains a valid pointer; otherwise, false.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool TryGetAllocation(MemoryHandle handle, out IntPtr ptr)
{
return s_allocations.TryGetElement(handle.id, handle.generation, out ptr);
}
///
/// Determines whether the specified memory handle refers to a currently tracked allocation.
///
///
/// This only validates the memory when you added the allocation via .
/// For validating memory from , use instead.
///
/// The memory handle to check for an associated allocation.
/// true if the allocation corresponding to the handle exists; otherwise, false.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool ContainsAllocation(MemoryHandle handle)
{
if (handle == MagicHandle)
{
return true;
}
return s_allocations.Contains(handle.id, handle.generation);
}
///
/// Disposes of the AllocationManager, freeing all allocated memory and resources.
///
public static void Dispose()
{
if (s_disposed)
{
return;
}
#if ENABLE_DEBUG_LAYER
// In debug mode, walk the intrusive list to surface any leaks.
var snapshot = new List();
var taken = false;
try
{
s_liveLock.Enter(ref taken);
if (s_pLiveHead != null)
{
snapshot.Capacity = 128;
for (var p = s_pLiveHead; p != null; p = p->next)
{
var trace = (StackTrace)HeaderGetHandle(p).Target!;
snapshot.Add(new AllocationInfo
{
Size = p->userSize,
StackTrace = trace
});
}
}
}
finally
{
if (taken)
{
s_liveLock.Exit();
}
}
nuint unfreeBytes = 0u;
foreach (var info in snapshot)
{
unfreeBytes += info.Size;
}
if (unfreeBytes > 0u)
{
throw new MemoryLeakException(CollectionsMarshal.AsSpan(snapshot));
}
Debug.Assert(LiveAllocationCount == 0);
#else
if (LiveAllocationCount != 0)
{
throw new MemoryLeakException($"Found {LiveAllocationCount} memory lakes! Please enable debug layer for more informations.");
}
#endif
// NOTE: Arena allocator holds the base ptr for all allocators, heap and stack allocators do not own any memory themselves.
if (s_pArenaAllocator != null)
{
s_pArenaAllocator->Dispose();
s_pStackAllocator->Dispose();
NativeMemory.Free(s_pArenaAllocator);
}
s_disposed = true;
}
}