Misaki/Misaki.HighPerformance
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add TLSF allocator and refactor allocation API

Browse Source 
- Introduced TLSF allocator with thread-safe wrapper and integrated into AllocationManager.
- Extended AllocationManagerDesc for TLSF config; made properties settable.
- Refactored AllocationHandle to encapsulate function pointers and state, replacing direct field access with methods.
- Updated all memory-related structs to use new AllocationHandle API.
- Added ReplaceIfZeros utility to MemoryUtility.
- Improved IndexOfNullByte performance.
- Minor fix in MemoryLeakException output order.
- FreeList now uses a fixed 64KB refill budget.
- Bumped version to 1.6.21; removed MHP_ENABLE_STACKTRACE from Debug.
- Updated Program.cs to test TLSF allocator and manage allocation lifecycle.
This commit is contained in:
Misaki
2026-05-05 22:13:58 +09:00
parent 627c1da928
commit d3e497c7d8
14 changed files with 303 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
Misaki.HighPerformance.LowLevel/Buffer/AllocationManager.cs
View File

@@ -3,6 +3,7 @@ using Misaki.HighPerformance.Collections;
#endif
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace Misaki.HighPerformance.LowLevel.Buffer;
@@ -20,7 +21,7 @@ public readonly struct AllocationInfo
}
/// <summary>
/// Gets the newSize of the allocation in bytes.
/// Gets the size of the allocation in bytes.
/// </summary>
public nuint Size
{
@@ -38,40 +39,46 @@ public readonly struct AllocationInfo
#endif
}
public readonly struct AllocationManagerDesc
public struct AllocationManagerDesc
{
public required nuint ArenaCapacity
public nuint ArenaCapacity
{
get; init;
get; set;
}
public required nuint StackCapacity
public nuint StackCapacity
{
get; init;
get; set;
}
public required nuint FreeListChunkSize
public nuint FreeListChunkSize
{
get; init;
get; set;
}
public required nuint FreeListDefaultAlignment
public nuint FreeListDefaultAlignment
{
get; init;
get; set;
}
[Obsolete("FreeList concurrency level is no longer used and will be ignored. FreeList is now designed to be thread-safe without a fixed concurrency level.")]
public int FreeListConcurrencyLevel
public nuint TLSFAlignment
{
get; init;
get; set;
}
public nuint TLSFInitialChunkSize
{
get; set;
}
public static AllocationManagerDesc Default => new AllocationManagerDesc
{
ArenaCapacity = 1024 * 1024 * 1024, // 1 GB
StackCapacity = 16 * 1024 * 1024, // 16 MB per thread
StackCapacity = 32 * 1024 * 1024, // 32 MB per thread
FreeListChunkSize = 64 * 1024 * 1024,
FreeListDefaultAlignment = 8,
TLSFAlignment = 16,
TLSFInitialChunkSize = 64 * 1024, // 64 KB
};
}
@@ -89,13 +96,7 @@ public static unsafe class AllocationManager
public void Init()
{
_handle = new AllocationHandle
{
State = null,
Alloc = &Allocate,
Realloc = &Reallocate,
Free = &Free
};
_handle = new AllocationHandle(null, &Allocate, &Reallocate, &Free);
}
private static void* Allocate(void* _, nuint size, nuint alignment, AllocationOption allocationOption)
@@ -138,6 +139,63 @@ public static unsafe class AllocationManager
}
}
// TODO: Lock-free implementation
internal struct TLSFAllocator : IAllocator, IDisposable
{
private TLSF _tlsf;
private GCHandle _lock;
private AllocationHandle _handle;
public readonly AllocationHandle Handle => _handle;
public void Init(nuint alignment, nuint initialChunkSize)
{
_tlsf = new TLSF(alignment, initialChunkSize);
#pragma warning disable CS9216 // A value of type 'System.Threading.Lock' converted to a different type will use likely unintended monitor-based locking in 'lock' statement.
_lock = GCHandle.Alloc(new Lock(), GCHandleType.Normal);
#pragma warning restore CS9216 // A value of type 'System.Threading.Lock' converted to a different type will use likely unintended monitor-based locking in 'lock' statement.
_handle = new AllocationHandle(Unsafe.AsPointer(in this), &Allocate, &Reallocate, &Free);
}
private static void* Allocate(void* state, nuint size, nuint alignment, AllocationOption allocationOption)
{
var allocator = (TLSFAllocator*)state;
var locker = (Lock)allocator->_lock.Target!;
lock (locker)
{
return allocator->_tlsf.Allocate(size, alignment, allocationOption);
}
}
private static void* Reallocate(void* state, void* ptr, nuint oldSize, nuint newSize, nuint alignment, AllocationOption allocationOption)
{
var allocator = (TLSFAllocator*)state;
var locker = (Lock)allocator->_lock.Target!;
lock (locker)
{
return allocator->_tlsf.Reallocate(ptr, oldSize, newSize, alignment, allocationOption);
}
}
private static void Free(void* state, void* ptr)
{
var allocator = (TLSFAllocator*)state;
var locker = (Lock)allocator->_lock.Target!;
lock (locker)
{
allocator->_tlsf.Free(ptr);
}
}
public void Dispose()
{
_lock.Free();
}
}
private class ThreadLocalStackPool
{
public MemoryPool<VirtualStack, VirtualStack.CreationOptions> pool;
@@ -159,6 +217,7 @@ public static unsafe class AllocationManager
internal static MemoryPool<VirtualArena, VirtualArena.CreationOptions> s_arenaAllocator;
internal static MemoryPool<FreeList, FreeList.CreationOptions> s_freeListAllocator;
internal static HeapAllocator* s_pHeapAllocator;
internal static TLSFAllocator* s_pTLSFAllocator;
[ThreadStatic]
private static ThreadLocalStackPool? t_stackAllocator;
@@ -189,9 +248,8 @@ public static unsafe class AllocationManager
private static volatile bool s_initialized;
private static nuint s_threadLocalStackSize;
private static SpinLock s_stackLocker = new SpinLock(false);
public static void Initialize(AllocationManagerDesc opts)
public static void Initialize(AllocationManagerDesc desc = default)
{
if (s_initialized)
{
@@ -202,21 +260,30 @@ public static unsafe class AllocationManager
s_allocations = new ConcurrentSlotMap<AllocationInfo>(256);
#endif
var defaultDesc = AllocationManagerDesc.Default;
var spanDesc = MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref desc, 1));
var spanDefault = MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref defaultDesc, 1));
ReplaceIfZeros(spanDesc, spanDefault);
s_arenaAllocator = new MemoryPool<VirtualArena, VirtualArena.CreationOptions>(new VirtualArena.CreationOptions
{
reserveCapacity = opts.ArenaCapacity
reserveCapacity = desc.ArenaCapacity
});
s_freeListAllocator = new MemoryPool<FreeList, FreeList.CreationOptions>(new FreeList.CreationOptions
{
alignment = opts.FreeListDefaultAlignment,
chunkSize = opts.FreeListChunkSize
alignment = desc.FreeListDefaultAlignment,
chunkSize = desc.FreeListChunkSize
});
s_pHeapAllocator = (HeapAllocator*)Malloc((nuint)(sizeof(HeapAllocator)));
s_pHeapAllocator = (HeapAllocator*)Malloc((nuint)sizeof(HeapAllocator));
s_pHeapAllocator->Init();
s_threadLocalStackSize = opts.StackCapacity;
s_pTLSFAllocator = (TLSFAllocator*)Malloc((nuint)sizeof(TLSFAllocator));
s_pTLSFAllocator->Init(desc.TLSFAlignment, desc.TLSFInitialChunkSize);
s_threadLocalStackSize = desc.StackCapacity;
s_initialized = true;
}
@@ -423,5 +490,12 @@ public static unsafe class AllocationManager
Free(s_pHeapAllocator);
s_pHeapAllocator = null;
}
if (s_pTLSFAllocator != null)
{
s_pTLSFAllocator->Dispose();
Free(s_pTLSFAllocator);
s_pTLSFAllocator = null;
}
}
}

4
Misaki.HighPerformance.LowLevel/Buffer/FreeList.cs
View File

@@ -422,8 +422,8 @@ public unsafe struct FreeList : IMemoryAllocator<FreeList, FreeList.CreationOpti
}
var blockSize = bucket->blockSize;
var blocksToCreate = Math.Max(1u, _chunkSize / blockSize);
blocksToCreate = Math.Min(blocksToCreate, 256);
const nuint REFILL_BUDGET = 64 * 1024; // 64KB per refill
var blocksToCreate = Math.Max(1u, REFILL_BUDGET / blockSize);
if (blocksToCreate == 0)
{

41
Misaki.HighPerformance.LowLevel/Buffer/IAllocator.cs
View File

@@ -1,3 +1,4 @@
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
namespace Misaki.HighPerformance.LowLevel.Buffer;
@@ -97,35 +98,39 @@ public readonly unsafe struct AllocationHandle
public static AllocationHandle Persistent => AllocationManager.s_pHeapAllocator->Handle;
/// <summary>
/// Gets a pointer to the state instance associated with this allocation handle.
/// Allocator for persistent allocations using a Two-Level Segregated Fit (TLSF) algorithm. Allocations are not automatically released after use, but can be reused to reduce fragmentation, system call and improve performance.
/// </summary>
public required void* State
public static AllocationHandle TLSF => AllocationManager.s_pTLSFAllocator->Handle;
private readonly void* _state;
private readonly AllocFunc _alloc;
private readonly ReallocFunc _realloc;
private readonly FreeFunc _free;
public AllocationHandle(void* state, AllocFunc alloc, ReallocFunc realloc, FreeFunc free)
{
get; init;
_state = state;
_alloc = alloc;
_realloc = realloc;
_free = free;
}
/// <summary>
/// Gets a function pointer for allocating memory.
/// </summary>
public required AllocFunc Alloc
public void* Alloc(nuint size, nuint alignment, AllocationOption option = AllocationOption.None)
{
get; init;
Debug.Assert(_alloc != null);
return _alloc(_state, size, alignment, option);
}
/// <summary>
/// Gets a function pointer for reallocating memory.
/// </summary>
public required ReallocFunc Realloc
public void* Realloc(void* ptr, nuint oldSize, nuint newSize, nuint alignment, AllocationOption allocationOption = AllocationOption.None)
{
get; init;
Debug.Assert(_realloc != null);
return _realloc(_state, ptr, oldSize, newSize, alignment, allocationOption);
}
/// <summary>
/// Gets a function pointer for freeing allocated memory.
/// </summary>
public required FreeFunc Free
public void Free(void* ptr)
{
get; init;
Debug.Assert(_free != null);
_free(_state, ptr);
}
}

6
Misaki.HighPerformance.LowLevel/Buffer/MemoryBlock.cs
View File

@@ -49,7 +49,7 @@ public unsafe struct MemoryBlock : IDisposable
throw new InvalidOperationException("Target allocation handle does not support allocation.");
}
_buffer = handle.Alloc(handle.State, size, alignment, allocationOption);
_buffer = handle.Alloc(size, alignment, allocationOption);
_size = size;
_alignment = alignment;
@@ -110,7 +110,7 @@ public unsafe struct MemoryBlock : IDisposable
return;
}
_buffer = _allocationHandle.Realloc(_allocationHandle.State, _buffer, _size, newSize, _alignment, option);
_buffer = _allocationHandle.Realloc(_buffer, _size, newSize, _alignment, option);
_size = newSize;
#if MHP_ENABLE_SAFETY_CHECKS
_memoryHandle.Update(_buffer, _size);
@@ -265,7 +265,7 @@ public unsafe struct MemoryBlock : IDisposable
if (_allocationHandle.Free != null)
{
_allocationHandle.Free(_allocationHandle.State, _buffer);
_allocationHandle.Free(_buffer);
}
#if MHP_ENABLE_SAFETY_CHECKS

8
Misaki.HighPerformance.LowLevel/Buffer/MemoryPool.cs
View File

@@ -19,13 +19,7 @@ public unsafe struct MemoryPool<TAllocator, TOpts> : IDisposable
_pAllocator = (TAllocator*)Malloc((nuint)sizeof(TAllocator));
*_pAllocator = allocator;
_allocationHandle = new AllocationHandle
{
State = _pAllocator,
Alloc = &Allocate,
Realloc = &Reallocate,
Free = &Free
};
_allocationHandle = new AllocationHandle(_pAllocator, &Allocate, &Reallocate, &Free);
}
private static void* Allocate(void* pAllocator, nuint size, nuint alignment, AllocationOption allocationOption)

8
Misaki.HighPerformance.LowLevel/Buffer/TLSF.cs
View File

@@ -86,8 +86,8 @@ public unsafe struct TLSF : IMemoryAllocator<TLSF, TLSF.CreationOptions>
private uint* _slBitmaps;
private BlockHeader** _blocks;
private MemoryChunk* _chunks;
private nuint _alignment;
private nuint _chunkSize;
private readonly nuint _alignment;
private readonly nuint _chunkSize;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static TLSF Create(in CreationOptions opts)
@@ -141,10 +141,6 @@ public unsafe struct TLSF : IMemoryAllocator<TLSF, TLSF.CreationOptions>
{
var totalSize = size + (nuint)sizeof(MemoryChunk);
var mem = (byte*)AlignedAlloc(totalSize, _alignment);
if (mem == null)
{
throw new OutOfMemoryException("Failed to allocate MemoryChunk for TlsfAllocator.");
}
MemoryChunk* chunk = (MemoryChunk*)mem;
chunk->next = _chunks;

6
Misaki.HighPerformance.LowLevel/Collections/HashMapHelper.cs
View File

@@ -251,7 +251,7 @@ public unsafe struct HashMapHelper<TKey> : IDisposable
throw new InvalidOperationException("Target allocation handle does not support allocation.");
}
var buf = (byte*)_allocationHandle.Alloc(_allocationHandle.State, (uint)totalSize, (nuint)_alignment, allocationOption);
var buf = (byte*)_allocationHandle.Alloc((uint)totalSize, (nuint)_alignment, allocationOption);
_buffer = buf;
_keys = (TKey*)(_buffer + keyOffset);
@@ -287,7 +287,7 @@ public unsafe struct HashMapHelper<TKey> : IDisposable
if (_allocationHandle.Free != null)
{
_allocationHandle.Free(_allocationHandle.State, oldBuffer);
_allocationHandle.Free(oldBuffer);
}
#if MHP_ENABLE_SAFETY_CHECKS
@@ -707,7 +707,7 @@ public unsafe struct HashMapHelper<TKey> : IDisposable
if (_allocationHandle.Free != null)
{
_allocationHandle.Free(_allocationHandle.State, _buffer);
_allocationHandle.Free(_buffer);
}
#if MHP_ENABLE_SAFETY_CHECKS

9
Misaki.HighPerformance.LowLevel/Collections/UnsafeArray.cs
View File

@@ -141,7 +141,7 @@ public unsafe struct UnsafeArray<T> : IUnsafeCollection<T>
throw new InvalidOperationException("Target allocation handle does not support allocation.");
}
_buffer = (T*)handle.Alloc(handle.State, (nuint)(count * sizeof(T)), AlignOf<T>(), allocationOption);
_buffer = (T*)handle.Alloc((nuint)(count * sizeof(T)), AlignOf<T>(), allocationOption);
#if MHP_ENABLE_SAFETY_CHECKS
_memoryHandle = MemoryHandle.Create(_buffer, (nuint)(count * sizeof(T)));
#endif
@@ -232,7 +232,7 @@ public unsafe struct UnsafeArray<T> : IUnsafeCollection<T>
}
var elemSize = SizeOf<T>();
_buffer = (T*)_allocationHandle.Realloc(_allocationHandle.State, _buffer, (nuint)Count * elemSize, (nuint)newSize * elemSize, AlignOf<T>(), option);
_buffer = (T*)_allocationHandle.Realloc(_buffer, (nuint)Count * elemSize, (nuint)newSize * elemSize, AlignOf<T>(), option);
_count = newSize;
#if MHP_ENABLE_SAFETY_CHECKS
_memoryHandle.Update(_buffer, (nuint)newSize * elemSize);
@@ -388,10 +388,7 @@ public unsafe struct UnsafeArray<T> : IUnsafeCollection<T>
return;
}
if (_allocationHandle.Free != null)
{
_allocationHandle.Free(_allocationHandle.State, _buffer);
}
_allocationHandle.Free(_buffer);
#if MHP_ENABLE_SAFETY_CHECKS
_memoryHandle.Dispose();

10
Misaki.HighPerformance.LowLevel/Collections/UnsafeParallelHashMap.cs
View File

@@ -70,7 +70,7 @@ public unsafe struct UnsafeParallelHashMap<TKey, TValue> : IDisposable
{
ArgumentOutOfRangeException.ThrowIfNegative(capacity);
_data = (UnsafeParallelHashMapData<TKey, TValue>*)handle.Alloc(handle.State, (uint)sizeof(UnsafeParallelHashMapData<TKey, TValue>), (nuint)AlignOf<UnsafeParallelHashMapData<TKey, TValue>>(), AllocationOption.Clear);
_data = (UnsafeParallelHashMapData<TKey, TValue>*)handle.Alloc((uint)sizeof(UnsafeParallelHashMapData<TKey, TValue>), (nuint)AlignOf<UnsafeParallelHashMapData<TKey, TValue>>(), AllocationOption.Clear);
if (_data == null)
throw new OutOfMemoryException("Failed to allocate UnsafeParallelHashMapData.");
@@ -110,13 +110,13 @@ public unsafe struct UnsafeParallelHashMap<TKey, TValue> : IDisposable
if (_data->buffer != null && _data->allocationHandle.Free != null)
{
_data->allocationHandle.Free(_data->allocationHandle.State, _data->buffer);
_data->allocationHandle.Free(_data->buffer);
_data->buffer = null;
}
if (_data != null && _data->allocationHandle.Free != null)
{
_data->allocationHandle.Free(_data->allocationHandle.State, _data);
_data->allocationHandle.Free(_data);
_data = null;
}
}
@@ -186,7 +186,7 @@ public unsafe struct UnsafeParallelHashMap<TKey, TValue> : IDisposable
throw new InvalidOperationException("Target allocation handle does not support allocation.");
}
var buf = (byte*)data->allocationHandle.Alloc(data->allocationHandle.State, (uint)totalSize, (nuint)data->alignment, allocationOption);
var buf = (byte*)data->allocationHandle.Alloc((uint)totalSize, (nuint)data->alignment, allocationOption);
data->buffer = buf;
data->keys = (TKey*)(buf + keyOffset);
@@ -399,7 +399,7 @@ public unsafe struct UnsafeParallelHashMap<TKey, TValue> : IDisposable
if (_data->allocationHandle.Free != null && oldBuffer != null)
{
_data->allocationHandle.Free(_data->allocationHandle.State, oldBuffer);
_data->allocationHandle.Free(oldBuffer);
}
#if MHP_ENABLE_SAFETY_CHECKS

8
Misaki.HighPerformance.LowLevel/Collections/UnsafeParallelQueue.cs
View File

@@ -88,7 +88,7 @@ public unsafe struct UnsafeParallelQueue<T> : IDisposable
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static DisposablePtr<UnsafeParallelQueue<T>> Allocate(int capacityPerChunk, AllocationHandle handle, AllocationOption allocationOption = AllocationOption.None)
{
var pQueue = (UnsafeParallelQueue<T>*)handle.Alloc(handle.State, SizeOf<DisposablePtr<UnsafeParallelQueue<T>>>(), AlignOf<DisposablePtr<UnsafeParallelQueue<T>>>(), AllocationOption.None);
var pQueue = (UnsafeParallelQueue<T>*)handle.Alloc(SizeOf<DisposablePtr<UnsafeParallelQueue<T>>>(), AlignOf<DisposablePtr<UnsafeParallelQueue<T>>>(), AllocationOption.None);
*pQueue = new UnsafeParallelQueue<T>(capacityPerChunk, handle, allocationOption);
return new DisposablePtr<UnsafeParallelQueue<T>>(pQueue);
}
@@ -289,7 +289,7 @@ public unsafe struct UnsafeParallelQueue<T> : IDisposable
private readonly ChunkHeader* AllocateNewChunk()
{
nuint byteSize = (nuint)sizeof(ChunkHeader) + (nuint)(_chunkCapacity * sizeof(ChunkSlot));
ChunkHeader* block = (ChunkHeader*)_allocHandle.Alloc(_allocHandle.State, byteSize, AlignOf<int>(), _allocOption);
ChunkHeader* block = (ChunkHeader*)_allocHandle.Alloc(byteSize, AlignOf<int>(), _allocOption);
block->next = null;
block->nextFree = null;
@@ -352,7 +352,7 @@ public unsafe struct UnsafeParallelQueue<T> : IDisposable
while (curr != null)
{
var next = curr->next;
_allocHandle.Free(_allocHandle.State, curr);
_allocHandle.Free(curr);
curr = next;
}
@@ -361,7 +361,7 @@ public unsafe struct UnsafeParallelQueue<T> : IDisposable
while (free != null)
{
var next = free->nextFree;
_allocHandle.Free(_allocHandle.State, free);
_allocHandle.Free(free);
free = next;
}

2
Misaki.HighPerformance.LowLevel/MemoryLeakException.cs
View File

@@ -55,7 +55,7 @@ public class MemoryLeakException : Exception
if (frame != null)
{
var methodInfo = DiagnosticMethodInfo.Create(frame);
stringBuilder.AppendLine($"File: {fileName}, Type: {methodInfo?.DeclaringTypeName}, Method: {methodInfo?.Name}, Line: {frame.GetFileLineNumber()}");
stringBuilder.AppendLine($"Type: {methodInfo?.DeclaringTypeName}, Method: {methodInfo?.Name}, File: {fileName}, Line: {frame.GetFileLineNumber()}");
}
}
}

4
Misaki.HighPerformance.LowLevel/Misaki.HighPerformance.LowLevel.csproj
View File

@@ -7,7 +7,7 @@
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
<Authors>Misaki</Authors>
<AssemblyVersion>1.6.20</AssemblyVersion>
<AssemblyVersion>1.6.21</AssemblyVersion>
<Version>$(AssemblyVersion)</Version>
<PackageProjectUrl>https://git.personalnas.com/Misaki/Misaki.HighPerformance.git</PackageProjectUrl>
<RepositoryUrl>https://git.personalnas.com/Misaki/Misaki.HighPerformance.git</RepositoryUrl>
@@ -24,7 +24,7 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
<IsAotCompatible>True</IsAotCompatible>
<DefineConstants>$(DefineConstants);MHP_ENABLE_SAFETY_CHECKS;MHP_ENABLE_STACKTRACE</DefineConstants>
<DefineConstants>$(DefineConstants);MHP_ENABLE_SAFETY_CHECKS</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">

108
Misaki.HighPerformance.LowLevel/Utilities/MemoryUtility.Byte.cs
View File

@@ -1,6 +1,7 @@
using System.Diagnostics.CodeAnalysis;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
namespace Misaki.HighPerformance.LowLevel.Utilities;
@@ -56,7 +57,7 @@ public static unsafe partial class MemoryUtility
/// <param name="searchSpace">A pointer to the byte array where the search will be performed.</param>
/// <returns>Returns the index of the first null byte found in the array..</returns>
/// <exception cref="ArgumentException">Thrown if the byte array is not null-terminated.</exception>"
public static unsafe int IndexOfNullByte(byte* searchSpace)
public static int IndexOfNullByte(byte* searchSpace)
{
const int Length = int.MaxValue;
const uint uValue = 0; // Use uint for comparisons to avoid unnecessary 8->32 extensions
@@ -75,21 +76,44 @@ public static unsafe partial class MemoryUtility
lengthToExamine -= 8;
if (uValue == searchSpace[offset])
{
goto Found;
}
if (uValue == searchSpace[offset + 1])
{
goto Found1;
}
if (uValue == searchSpace[offset + 2])
{
goto Found2;
}
if (uValue == searchSpace[offset + 3])
{
goto Found3;
}
if (uValue == searchSpace[offset + 4])
{
goto Found4;
}
if (uValue == searchSpace[offset + 5])
{
goto Found5;
}
if (uValue == searchSpace[offset + 6])
{
goto Found6;
}
if (uValue == searchSpace[offset + 7])
{
goto Found7;
}
offset += 8;
}
@@ -99,13 +123,24 @@ public static unsafe partial class MemoryUtility
lengthToExamine -= 4;
if (uValue == searchSpace[offset])
{
goto Found;
}
if (uValue == searchSpace[offset + 1])
{
goto Found1;
}
if (uValue == searchSpace[offset + 2])
{
goto Found2;
}
if (uValue == searchSpace[offset + 3])
{
goto Found3;
}
offset += 4;
}
@@ -115,7 +150,9 @@ public static unsafe partial class MemoryUtility
lengthToExamine -= 1;
if (uValue == searchSpace[offset])
{
goto Found;
}
offset += 1;
}
@@ -359,4 +396,73 @@ public static unsafe partial class MemoryUtility
Found7:
return (int)(offset + 7);
}
public static void ReplaceIfZeros(Span<byte> a, ReadOnlySpan<byte> b)
{
if (a.Length != b.Length)
{
throw new ArgumentException("Spans must be the same size.");
}
var i = 0;
if (Vector.IsHardwareAccelerated && a.Length >= Vector<byte>.Count)
{
ref var ptrA = ref MemoryMarshal.GetReference(a);
ref var ptrB = ref MemoryMarshal.GetReference(b);
var limit = a.Length - Vector<byte>.Count;
for (; i <= limit; i += Vector<byte>.Count)
{
var vecA = Vector.LoadUnsafe(ref ptrA, (nuint)i);
var vecB = Vector.LoadUnsafe(ref ptrB, (nuint)i);
var mask = Vector.Equals(vecA, Vector<byte>.Zero);
var result = Vector.ConditionalSelect(mask, vecB, vecA);
result.StoreUnsafe(ref ptrA, (nuint)i);
}
}
// Fallback standard loop for the remaining "tail" bytes (e.g., the last 15 bytes)
for (; i < a.Length; i++)
{
if (a[i] == 0)
{
a[i] = b[i];
}
}
}
public static void ReplaceIfZeros(void* a, void* b, nuint length)
{
var ptrA = (byte*)a;
var ptrB = (byte*)b;
nuint i = 0u;
if (Vector.IsHardwareAccelerated && length >= (nuint)Vector<byte>.Count)
{
var vectorSize = (nuint)Vector<byte>.Count;
var limit = length - vectorSize;
for (; i <= limit; i += vectorSize)
{
var vecA = Vector.Load(ptrA + i);
var vecB = Vector.Load(ptrB + i);
var mask = Vector.Equals(vecA, Vector<byte>.Zero);
var result = Vector.ConditionalSelect(mask, vecB, vecA);
result.Store(ptrA + i);
}
}
// Fallback standard loop for the remaining "tail" bytes (e.g., the last 15 bytes)
for (; i < length; i++)
{
if (ptrA[i] == 0)
{
ptrA[i] = ptrB[i];
}
}
}
}