Misaki/Misaki.HighPerformance
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SPMD API overhaul: gather/scatter, job & packaging updates

Browse Source 
- ISPMDLane: add MaskGather, MaskStore, Scatter, MaskScatter; update MaskLoad/Gather signatures for hardware parity
- WideLane/ScalarLane: implement new methods with HW/fallback logic
- MathV: gather/mask-gather now delegate to lane methods
- Vector2/3/4: add CompressStore, Scatter, MaskScatter
- SPMD jobs/tests/README: migrate to new APIs for correctness
- Use Unsafe.BitCast instead of Unsafe.As/AsRef
- Add SPMDUtility for gather index extraction
- Job system: add ICustomJob<TSelf>, ScheduleCustom overload
- FreeList concurrency obsolete; always thread-safe
- NuGet: include LICENSE/README, set license/readme in .csproj
- Docs: update SPMD usage, clarify safety notes
- Minor: doc fixes, CompressStore test improvements
This commit is contained in:
Misaki
2026-05-04 13:56:49 +09:00
parent 99fcbec753
commit 155d7b0fbd
32 changed files with 1463 additions and 2028 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1414
Misaki.HighPerformance.Mathematics.SPMD/Templates/MathV.Vector.gen.cs
View File

File diff suppressed because it is too large Load Diff

441
Misaki.HighPerformance.Mathematics.SPMD/Templates/MathV.Vector.tt
View File

@@ -67,22 +67,10 @@ public static unsafe partial class MathV
<#= TLaneRestrictions #>
<#= TNumberRestrictions #>
{
<# for (int i = 0; i < dimension; i++) { #>
Unsafe.SkipInit(out TLane <#= components[i] #>);
var p<#= components[i] #> = (<#= TNumber #>*)&<#= components[i] #>;
<# } #>
for (var i = 0; i < TLane.LaneWidth; i++)
{
<# for (int i = 0; i < dimension; i++) { #>
p<#= components[i] #>[i] = pSrc[i * <#= dimension #> + <#= i #>];
<# } #>
}
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = <#= components[i] #>,
<#= components[i] #> = <#= TLane #>.Load(pSrc + <#= i #>),
<# } #>
};
}
@@ -95,6 +83,27 @@ public static unsafe partial class MathV
return LoadVector<#= dimension #><<#= GenericParameters #>>((<#= TNumber #>*)Unsafe.AsPointer(ref src));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static <#= vectorType #> MaskLoadVector<#= dimension #><<#= GenericParameters #>>(<#= TNumber #>* pSrc, <#= TLane #> mask)
<#= TLaneRestrictions #>
<#= TNumberRestrictions #>
{
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = <#= TLane #>.MaskLoad(pSrc + <#= i #>, mask),
<# } #>
};
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static <#= vectorType #> MaskLoadVector<#= dimension #><<#= GenericParameters #>>(ref <#= TNumber #> src, <#= TLane #> mask)
<#= TLaneRestrictions #>
<#= TNumberRestrictions #>
{
return MaskLoadVector<#= dimension #><<#= GenericParameters #>>((<#= TNumber #>*)Unsafe.AsPointer(ref src), mask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static <#= vectorType #> Load<<#= GenericParameters #>>(<#= ForEachDimension(dimension, i => $"TNumber* p{components[i]}") #>)
<#= TLaneRestrictions #>
@@ -127,108 +136,10 @@ public static unsafe partial class MathV
<#= TLaneRestrictions #>
<#= TNumberRestrictions #>
{
if (Avx2.IsSupported)
{
if (TLane.LaneWidth == Vector128<TNumber>.Count)
{
if (sizeof(TNumber) == sizeof(uint))
{
ref var v = ref Unsafe.As<TLane, Vector128<TNumber>>(ref indices);
var vidx = v.AsInt32();
#pragma warning disable CA1857 // A constant is expected for the parameter
<# for (int i = 0; i < dimension; i++) { #>
var v<#= components[i] #> = Avx2.GatherVector128((uint*)(pData + <#= i #>), vidx, scale);
<# } #>
#pragma warning restore CA1857 // A constant is expected for the parameter
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = Unsafe.As<Vector128<uint>, TLane>(ref v<#= components[i] #>),
<# } #>
};
}
if (sizeof(TNumber) == sizeof(ulong))
{
ref var v = ref Unsafe.As<TLane, Vector128<TNumber>>(ref indices);
var vidx = v.AsInt64();
#pragma warning disable CA1857 // A constant is expected for the parameter
<# for (int i = 0; i < dimension; i++) { #>
var v<#= components[i] #> = Avx2.GatherVector128((ulong*)(pData + <#= i #>), vidx, scale);
<# } #>
#pragma warning restore CA1857 // A constant is expected for the parameter
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = Unsafe.As<Vector128<ulong>, TLane>(ref v<#= components[i] #>),
<# } #>
};
}
}
else if (TLane.LaneWidth == Vector256<TNumber>.Count)
{
if (sizeof(TNumber) == sizeof(uint))
{
ref var v = ref Unsafe.As<TLane, Vector256<TNumber>>(ref indices);
var vidx = v.AsInt32();
#pragma warning disable CA1857 // A constant is expected for the parameter
<# for (int i = 0; i < dimension; i++) { #>
var v<#= components[i] #> = Avx2.GatherVector256((uint*)(pData + <#= i #>), vidx, scale);
<# } #>
#pragma warning restore CA1857 // A constant is expected for the parameter
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = Unsafe.As<Vector256<uint>, TLane>(ref v<#= components[i] #>),
<# } #>
};
}
if (sizeof(TNumber) == sizeof(ulong))
{
ref var v = ref Unsafe.As<TLane, Vector256<TNumber>>(ref indices);
var vidx = v.AsInt64();
#pragma warning disable CA1857 // A constant is expected for the parameter
<# for (int i = 0; i < dimension; i++) { #>
var v<#= components[i] #> = Avx2.GatherVector256((ulong*)(pData + <#= i #>), vidx, scale);
<# } #>
#pragma warning restore CA1857 // A constant is expected for the parameter
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = Unsafe.As<Vector256<ulong>, TLane>(ref v<#= components[i] #>),
<# } #>
};
}
}
}
<# for (int i = 0; i < dimension; i++) { #>
Unsafe.SkipInit(out TLane <#= components[i] #>);
var p<#= components[i] #> = (<#= TNumber #>*)&<#= components[i] #>;
<# } #>
for (var i = 0; i < TLane.LaneWidth; i++)
{
var scalarIdx = int.CreateTruncating(indices[i]);
<# for (int i = 0; i < dimension; i++) { #>
p<#= components[i] #>[i] = *(TNumber*)((byte*)pData + ((scalarIdx + <#= i #>) * scale));
<# } #>
}
return new Vector<#= dimension #><TLane, TNumber>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = <#= components[i] #>,
<#= components[i] #> = <#= TLane #>.Gather(pData + <#= i #>, indices, scale),
<# } #>
};
}
@@ -238,104 +149,10 @@ public static unsafe partial class MathV
<#= TLaneRestrictions #>
<#= TNumberRestrictions #>
{
if (Avx2.IsSupported)
{
if (TLane.LaneWidth == Vector128<TNumber>.Count)
{
if (sizeof(TNumber) == sizeof(uint))
{
var vidx = Vector128.Load(pIndices);
#pragma warning disable CA1857 // A constant is expected for the parameter
<# for (int i = 0; i < dimension; i++) { #>
var v<#= components[i] #> = Avx2.GatherVector128((uint*)(pData + <#= i #>), vidx, scale);
<# } #>
#pragma warning restore CA1857 // A constant is expected for the parameter
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = Unsafe.As<Vector128<uint>, TLane>(ref v<#= components[i] #>),
<# } #>
};
}
if (sizeof(TNumber) == sizeof(ulong))
{
var vidx = Vector128.Load(pIndices);
#pragma warning disable CA1857 // A constant is expected for the parameter
<# for (int i = 0; i < dimension; i++) { #>
var v<#= components[i] #> = Avx2.GatherVector128((ulong*)(pData + <#= i #>), vidx, scale);
<# } #>
#pragma warning restore CA1857 // A constant is expected for the parameter
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = Unsafe.As<Vector128<ulong>, TLane>(ref v<#= components[i] #>),
<# } #>
};
}
}
else if (TLane.LaneWidth == Vector256<TNumber>.Count)
{
if (sizeof(TNumber) == sizeof(uint))
{
var vidx = Vector256.Load(pIndices);
#pragma warning disable CA1857 // A constant is expected for the parameter
<# for (int i = 0; i < dimension; i++) { #>
var v<#= components[i] #> = Avx2.GatherVector256((uint*)(pData + <#= i #>), vidx, scale);
<# } #>
#pragma warning restore CA1857 // A constant is expected for the parameter
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = Unsafe.As<Vector256<uint>, TLane>(ref v<#= components[i] #>),
<# } #>
};
}
if (sizeof(TNumber) == sizeof(ulong))
{
var vidx = Vector128.Load(pIndices);
#pragma warning disable CA1857 // A constant is expected for the parameter
<# for (int i = 0; i < dimension; i++) { #>
var v<#= components[i] #> = Avx2.GatherVector256((ulong*)(pData + <#= i #>), vidx, scale);
<# } #>
#pragma warning restore CA1857 // A constant is expected for the parameter
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = Unsafe.As<Vector256<ulong>, TLane>(ref v<#= components[i] #>),
<# } #>
};
}
}
}
<# for (int i = 0; i < dimension; i++) { #>
Unsafe.SkipInit(out TLane <#= components[i] #>);
var p<#= components[i] #> = (<#= TNumber #>*)&<#= components[i] #>;
<# } #>
for (var i = 0; i < TLane.LaneWidth; i++)
{
var scalerIdx = pIndices[i];
<# for (int i = 0; i < dimension; i++) { #>
p<#= components[i] #>[i] = *(TNumber*)((byte*)pData + ((scalerIdx + <#= i #>) * scale));
<# } #>
}
return new Vector<#= dimension #><TLane, TNumber>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = <#= components[i] #>,
<#= components[i] #> = <#= TLane #>.Gather(pData + <#= i #>, pIndices, scale),
<# } #>
};
}
@@ -361,113 +178,10 @@ public static unsafe partial class MathV
where TLane : unmanaged, ISPMDLane<TLane, TNumber>
where TNumber : unmanaged, INumber<TNumber>, IBinaryNumber<TNumber>, IMinMaxValue<TNumber>, IBitwiseOperators<TNumber, TNumber, TNumber>
{
if (Avx2.IsSupported)
{
if (TLane.LaneWidth == Vector128<TNumber>.Count)
{
if (sizeof(TNumber) == sizeof(uint))
{
ref var vidx = ref Unsafe.As<TLane, Vector128<int>>(ref indices);
ref var vmask = ref Unsafe.As<TLane, Vector128<uint>>(ref mask);
#pragma warning disable CA1857 // A constant is expected for the parameter
<# for (int i = 0; i < dimension; i++) { #>
var v<#= components[i] #> = Avx2.GatherMaskVector128(Vector128<uint>.Zero, (uint*)(pData + <#= i #>), vidx, vmask, scale);
<# } #>
#pragma warning restore CA1857 // A constant is expected for the parameter
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = Unsafe.As<Vector128<uint>, TLane>(ref v<#= components[i] #>),
<# } #>
};
}
if (sizeof(TNumber) == sizeof(ulong))
{
ref var vidx = ref Unsafe.As<TLane, Vector128<int>>(ref indices);
var vmask = Unsafe.As<TLane, Vector128<ulong>>(ref mask);
#pragma warning disable CA1857 // A constant is expected for the parameter
<# for (int i = 0; i < dimension; i++) { #>
var v<#= components[i] #> = Avx2.GatherMaskVector128(Vector128<ulong>.Zero, (ulong*)(pData + <#= i #>), vidx, vmask, scale);
<# } #>
#pragma warning restore CA1857 // A constant is expected for the parameter
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = Unsafe.As<Vector128<ulong>, TLane>(ref v<#= components[i] #>),
<# } #>
};
}
}
else if (TLane.LaneWidth == Vector256<TNumber>.Count)
{
if (sizeof(TNumber) == sizeof(uint))
{
ref var vidx = ref Unsafe.As<TLane, Vector256<int>>(ref indices);
var vmask = Unsafe.As<TLane, Vector256<uint>>(ref mask);
#pragma warning disable CA1857 // A constant is expected for the parameter
<# for (int i = 0; i < dimension; i++) { #>
var v<#= components[i] #> = Avx2.GatherMaskVector256(Vector256<uint>.Zero, (uint*)(pData + <#= i #>), vidx, vmask, scale);
<# } #>
#pragma warning restore CA1857 // A constant is expected for the parameter
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = Unsafe.As<Vector256<uint>, TLane>(ref v<#= components[i] #>),
<# } #>
};
}
if (sizeof(TNumber) == sizeof(ulong))
{
ref var vidx = ref Unsafe.As<TLane, Vector128<int>>(ref indices);
var vmask = Unsafe.As<TLane, Vector256<ulong>>(ref mask);
#pragma warning disable CA1857 // A constant is expected for the parameter
<# for (int i = 0; i < dimension; i++) { #>
var v<#= components[i] #> = Avx2.GatherMaskVector256(Vector256<ulong>.Zero, (ulong*)(pData + <#= i #>), vidx, vmask, scale);
<# } #>
#pragma warning restore CA1857 // A constant is expected for the parameter
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = Unsafe.As<Vector256<ulong>, TLane>(ref v<#= components[i] #>),
<# } #>
};
}
}
}
<# for (int i = 0; i < dimension; i++) { #>
Unsafe.SkipInit(out TLane <#= components[i] #>);
var p<#= components[i] #> = (<#= TNumber #>*)&<#= components[i] #>;
<# } #>
for (var i = 0; i < TLane.LaneWidth; i++)
{
if (mask[i] == TNumber.Zero)
{
continue;
}
var scalerIdx = int.CreateTruncating(indices[i]);
<# for (int i = 0; i < dimension; i++) { #>
p<#= components[i] #>[i] = *(TNumber*)((byte*)pData + ((scalerIdx + <#= i #>) * scale));
<# } #>
}
return new Vector<#= dimension #><TLane, TNumber>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = <#= components[i] #>,
<#= components[i] #> = <#= TLane #>.MaskGather(pData + <#= i #>, indices, mask, scale),
<# } #>
};
}
@@ -477,113 +191,10 @@ public static unsafe partial class MathV
where TLane : unmanaged, ISPMDLane<TLane, TNumber>
where TNumber : unmanaged, INumber<TNumber>, IBinaryNumber<TNumber>, IMinMaxValue<TNumber>, IBitwiseOperators<TNumber, TNumber, TNumber>
{
if (Avx2.IsSupported)
{
if (TLane.LaneWidth == Vector128<TNumber>.Count)
{
if (sizeof(TNumber) == sizeof(uint))
{
var vidx = Vector128.Load(pIndices);
ref var vmask = ref Unsafe.As<TLane, Vector128<uint>>(ref mask);
#pragma warning disable CA1857 // A constant is expected for the parameter
<# for (int i = 0; i < dimension; i++) { #>
var v<#= components[i] #> = Avx2.GatherMaskVector128(Vector128<uint>.Zero, (uint*)(pData + <#= i #>), vidx, vmask, scale);
<# } #>
#pragma warning restore CA1857 // A constant is expected for the parameter
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = Unsafe.As<Vector128<uint>, TLane>(ref v<#= components[i] #>),
<# } #>
};
}
if (sizeof(TNumber) == sizeof(ulong))
{
var vidx = Vector128.Load(pIndices);
var vmask = Unsafe.As<TLane, Vector128<ulong>>(ref mask);
#pragma warning disable CA1857 // A constant is expected for the parameter
<# for (int i = 0; i < dimension; i++) { #>
var v<#= components[i] #> = Avx2.GatherMaskVector128(Vector128<ulong>.Zero, (ulong*)(pData + <#= i #>), vidx, vmask, scale);
<# } #>
#pragma warning restore CA1857 // A constant is expected for the parameter
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = Unsafe.As<Vector128<ulong>, TLane>(ref v<#= components[i] #>),
<# } #>
};
}
}
else if (TLane.LaneWidth == Vector256<TNumber>.Count)
{
if (sizeof(TNumber) == sizeof(uint))
{
var vidx = Vector256.Load(pIndices);
var vmask = Unsafe.As<TLane, Vector256<uint>>(ref mask);
#pragma warning disable CA1857 // A constant is expected for the parameter
<# for (int i = 0; i < dimension; i++) { #>
var v<#= components[i] #> = Avx2.GatherMaskVector256(Vector256<uint>.Zero, (uint*)(pData + <#= i #>), vidx, vmask, scale);
<# } #>
#pragma warning restore CA1857 // A constant is expected for the parameter
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = Unsafe.As<Vector256<uint>, TLane>(ref v<#= components[i] #>),
<# } #>
};
}
if (sizeof(TNumber) == sizeof(ulong))
{
var vidx = Vector128.Load(pIndices);
var vmask = Unsafe.As<TLane, Vector256<ulong>>(ref mask);
#pragma warning disable CA1857 // A constant is expected for the parameter
<# for (int i = 0; i < dimension; i++) { #>
var v<#= components[i] #> = Avx2.GatherMaskVector256(Vector256<ulong>.Zero, (ulong*)(pData + <#= i #>), vidx, vmask, scale);
<# } #>
#pragma warning restore CA1857 // A constant is expected for the parameter
return new <#= vectorType #>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = Unsafe.As<Vector256<ulong>, TLane>(ref v<#= components[i] #>),
<# } #>
};
}
}
}
<# for (int i = 0; i < dimension; i++) { #>
Unsafe.SkipInit(out TLane <#= components[i] #>);
var p<#= components[i] #> = (<#= TNumber #>*)&<#= components[i] #>;
<# } #>
for (var i = 0; i < TLane.LaneWidth; i++)
{
if (mask[i] == TNumber.Zero)
{
continue;
}
var scalerIdx = pIndices[i];
<# for (int i = 0; i < dimension; i++) { #>
p<#= components[i] #>[i] = *(TNumber*)((byte*)pData + ((scalerIdx + <#= i #>) * scale));
<# } #>
}
return new Vector<#= dimension #><TLane, TNumber>
{
<# for (int i = 0; i < dimension; i++) { #>
<#= components[i] #> = <#= components[i] #>,
<#= components[i] #> = <#= TLane #>.MaskGather(pData + <#= i #>, pIndices, mask, scale),
<# } #>
};
}

88
Misaki.HighPerformance.Mathematics.SPMD/Templates/Vector2.gen.cs
View File

@@ -61,25 +61,15 @@ public unsafe struct Vector2<TLane, TNumber> : IEquatable<Vector2<TLane, TNumber
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Store(TNumber* pDst)
{
var width = TLane.LaneWidth;
var x = stackalloc TNumber[width];
var y = stackalloc TNumber[width];
this.x.Store(x);
this.y.Store(y);
for (var i = 0; i < width; i++)
{
pDst[i * 2 + 0] = x[i];
pDst[i * 2 + 1] = y[i];
}
x.Store(pDst + 0 * TLane.LaneWidth);
y.Store(pDst + 1 * TLane.LaneWidth);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Store(ref TNumber dst)
{
Store((TNumber*)Unsafe.AsPointer(ref dst));
x.Store(ref Unsafe.Add(ref dst, 0 * TLane.LaneWidth));
y.Store(ref Unsafe.Add(ref dst, 1 * TLane.LaneWidth));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -96,6 +86,76 @@ public unsafe struct Vector2<TLane, TNumber> : IEquatable<Vector2<TLane, TNumber
this.y.Store(ref y);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void CompressStore(TNumber* pDst, Vector2<TLane, TNumber> mask)
{
x.CompressStore(pDst + 0 * TLane.LaneWidth, mask.x);
y.CompressStore(pDst + 1 * TLane.LaneWidth, mask.y);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void CompressStore(ref TNumber dst, Vector2<TLane, TNumber> mask)
{
x.CompressStore(ref Unsafe.Add(ref dst, 0 * TLane.LaneWidth), mask.x);
y.CompressStore(ref Unsafe.Add(ref dst, 1 * TLane.LaneWidth), mask.y);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Scatter(TNumber* pDst, TLane indices)
{
x.Scatter(pDst + 0, indices);
y.Scatter(pDst + 1, indices);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Scatter(TNumber* pDst, int* pIndices)
{
x.Scatter(pDst + 0, pIndices);
y.Scatter(pDst + 1, pIndices);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Scatter(ref TNumber dst, TLane indices)
{
x.Scatter(ref Unsafe.Add(ref dst, 0), indices);
y.Scatter(ref Unsafe.Add(ref dst, 1), indices);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Scatter(ref TNumber dst, int* pIndices)
{
x.Scatter(ref Unsafe.Add(ref dst, 0), pIndices);
y.Scatter(ref Unsafe.Add(ref dst, 1), pIndices);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MaskScatter(TNumber* pDst, TLane indices, TLane mask)
{
x.MaskScatter(pDst + 0, indices, mask);
y.MaskScatter(pDst + 1, indices, mask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MaskScatter(TNumber* pDst, int* pIndices, TLane mask)
{
x.MaskScatter(pDst + 0, pIndices, mask);
y.MaskScatter(pDst + 1, pIndices, mask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MaskScatter(ref TNumber dst, TLane indices, TLane mask)
{
x.MaskScatter(ref Unsafe.Add(ref dst, 0), indices, mask);
y.MaskScatter(ref Unsafe.Add(ref dst, 1), indices, mask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MaskScatter(ref TNumber dst, int* pIndices, TLane mask)
{
x.MaskScatter(ref Unsafe.Add(ref dst, 0), pIndices, mask);
y.MaskScatter(ref Unsafe.Add(ref dst, 1), pIndices, mask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector2<TLane, TNumber> operator -(in Vector2<TLane, TNumber> vector)
{

103
Misaki.HighPerformance.Mathematics.SPMD/Templates/Vector3.gen.cs
View File

@@ -64,28 +64,17 @@ public unsafe struct Vector3<TLane, TNumber> : IEquatable<Vector3<TLane, TNumber
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Store(TNumber* pDst)
{
var width = TLane.LaneWidth;
var x = stackalloc TNumber[width];
var y = stackalloc TNumber[width];
var z = stackalloc TNumber[width];
this.x.Store(x);
this.y.Store(y);
this.z.Store(z);
for (var i = 0; i < width; i++)
{
pDst[i * 3 + 0] = x[i];
pDst[i * 3 + 1] = y[i];
pDst[i * 3 + 2] = z[i];
}
x.Store(pDst + 0 * TLane.LaneWidth);
y.Store(pDst + 1 * TLane.LaneWidth);
z.Store(pDst + 2 * TLane.LaneWidth);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Store(ref TNumber dst)
{
Store((TNumber*)Unsafe.AsPointer(ref dst));
x.Store(ref Unsafe.Add(ref dst, 0 * TLane.LaneWidth));
y.Store(ref Unsafe.Add(ref dst, 1 * TLane.LaneWidth));
z.Store(ref Unsafe.Add(ref dst, 2 * TLane.LaneWidth));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -104,6 +93,86 @@ public unsafe struct Vector3<TLane, TNumber> : IEquatable<Vector3<TLane, TNumber
this.z.Store(ref z);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void CompressStore(TNumber* pDst, Vector3<TLane, TNumber> mask)
{
x.CompressStore(pDst + 0 * TLane.LaneWidth, mask.x);
y.CompressStore(pDst + 1 * TLane.LaneWidth, mask.y);
z.CompressStore(pDst + 2 * TLane.LaneWidth, mask.z);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void CompressStore(ref TNumber dst, Vector3<TLane, TNumber> mask)
{
x.CompressStore(ref Unsafe.Add(ref dst, 0 * TLane.LaneWidth), mask.x);
y.CompressStore(ref Unsafe.Add(ref dst, 1 * TLane.LaneWidth), mask.y);
z.CompressStore(ref Unsafe.Add(ref dst, 2 * TLane.LaneWidth), mask.z);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Scatter(TNumber* pDst, TLane indices)
{
x.Scatter(pDst + 0, indices);
y.Scatter(pDst + 1, indices);
z.Scatter(pDst + 2, indices);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Scatter(TNumber* pDst, int* pIndices)
{
x.Scatter(pDst + 0, pIndices);
y.Scatter(pDst + 1, pIndices);
z.Scatter(pDst + 2, pIndices);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Scatter(ref TNumber dst, TLane indices)
{
x.Scatter(ref Unsafe.Add(ref dst, 0), indices);
y.Scatter(ref Unsafe.Add(ref dst, 1), indices);
z.Scatter(ref Unsafe.Add(ref dst, 2), indices);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Scatter(ref TNumber dst, int* pIndices)
{
x.Scatter(ref Unsafe.Add(ref dst, 0), pIndices);
y.Scatter(ref Unsafe.Add(ref dst, 1), pIndices);
z.Scatter(ref Unsafe.Add(ref dst, 2), pIndices);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MaskScatter(TNumber* pDst, TLane indices, TLane mask)
{
x.MaskScatter(pDst + 0, indices, mask);
y.MaskScatter(pDst + 1, indices, mask);
z.MaskScatter(pDst + 2, indices, mask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MaskScatter(TNumber* pDst, int* pIndices, TLane mask)
{
x.MaskScatter(pDst + 0, pIndices, mask);
y.MaskScatter(pDst + 1, pIndices, mask);
z.MaskScatter(pDst + 2, pIndices, mask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MaskScatter(ref TNumber dst, TLane indices, TLane mask)
{
x.MaskScatter(ref Unsafe.Add(ref dst, 0), indices, mask);
y.MaskScatter(ref Unsafe.Add(ref dst, 1), indices, mask);
z.MaskScatter(ref Unsafe.Add(ref dst, 2), indices, mask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MaskScatter(ref TNumber dst, int* pIndices, TLane mask)
{
x.MaskScatter(ref Unsafe.Add(ref dst, 0), pIndices, mask);
y.MaskScatter(ref Unsafe.Add(ref dst, 1), pIndices, mask);
z.MaskScatter(ref Unsafe.Add(ref dst, 2), pIndices, mask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector3<TLane, TNumber> operator -(in Vector3<TLane, TNumber> vector)
{

118
Misaki.HighPerformance.Mathematics.SPMD/Templates/Vector4.gen.cs
View File

@@ -67,31 +67,19 @@ public unsafe struct Vector4<TLane, TNumber> : IEquatable<Vector4<TLane, TNumber
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Store(TNumber* pDst)
{
var width = TLane.LaneWidth;
var x = stackalloc TNumber[width];
var y = stackalloc TNumber[width];
var z = stackalloc TNumber[width];
var w = stackalloc TNumber[width];
this.x.Store(x);
this.y.Store(y);
this.z.Store(z);
this.w.Store(w);
for (var i = 0; i < width; i++)
{
pDst[i * 4 + 0] = x[i];
pDst[i * 4 + 1] = y[i];
pDst[i * 4 + 2] = z[i];
pDst[i * 4 + 3] = w[i];
}
x.Store(pDst + 0 * TLane.LaneWidth);
y.Store(pDst + 1 * TLane.LaneWidth);
z.Store(pDst + 2 * TLane.LaneWidth);
w.Store(pDst + 3 * TLane.LaneWidth);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Store(ref TNumber dst)
{
Store((TNumber*)Unsafe.AsPointer(ref dst));
x.Store(ref Unsafe.Add(ref dst, 0 * TLane.LaneWidth));
y.Store(ref Unsafe.Add(ref dst, 1 * TLane.LaneWidth));
z.Store(ref Unsafe.Add(ref dst, 2 * TLane.LaneWidth));
w.Store(ref Unsafe.Add(ref dst, 3 * TLane.LaneWidth));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -112,6 +100,96 @@ public unsafe struct Vector4<TLane, TNumber> : IEquatable<Vector4<TLane, TNumber
this.w.Store(ref w);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void CompressStore(TNumber* pDst, Vector4<TLane, TNumber> mask)
{
x.CompressStore(pDst + 0 * TLane.LaneWidth, mask.x);
y.CompressStore(pDst + 1 * TLane.LaneWidth, mask.y);
z.CompressStore(pDst + 2 * TLane.LaneWidth, mask.z);
w.CompressStore(pDst + 3 * TLane.LaneWidth, mask.w);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void CompressStore(ref TNumber dst, Vector4<TLane, TNumber> mask)
{
x.CompressStore(ref Unsafe.Add(ref dst, 0 * TLane.LaneWidth), mask.x);
y.CompressStore(ref Unsafe.Add(ref dst, 1 * TLane.LaneWidth), mask.y);
z.CompressStore(ref Unsafe.Add(ref dst, 2 * TLane.LaneWidth), mask.z);
w.CompressStore(ref Unsafe.Add(ref dst, 3 * TLane.LaneWidth), mask.w);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Scatter(TNumber* pDst, TLane indices)
{
x.Scatter(pDst + 0, indices);
y.Scatter(pDst + 1, indices);
z.Scatter(pDst + 2, indices);
w.Scatter(pDst + 3, indices);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Scatter(TNumber* pDst, int* pIndices)
{
x.Scatter(pDst + 0, pIndices);
y.Scatter(pDst + 1, pIndices);
z.Scatter(pDst + 2, pIndices);
w.Scatter(pDst + 3, pIndices);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Scatter(ref TNumber dst, TLane indices)
{
x.Scatter(ref Unsafe.Add(ref dst, 0), indices);
y.Scatter(ref Unsafe.Add(ref dst, 1), indices);
z.Scatter(ref Unsafe.Add(ref dst, 2), indices);
w.Scatter(ref Unsafe.Add(ref dst, 3), indices);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Scatter(ref TNumber dst, int* pIndices)
{
x.Scatter(ref Unsafe.Add(ref dst, 0), pIndices);
y.Scatter(ref Unsafe.Add(ref dst, 1), pIndices);
z.Scatter(ref Unsafe.Add(ref dst, 2), pIndices);
w.Scatter(ref Unsafe.Add(ref dst, 3), pIndices);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MaskScatter(TNumber* pDst, TLane indices, TLane mask)
{
x.MaskScatter(pDst + 0, indices, mask);
y.MaskScatter(pDst + 1, indices, mask);
z.MaskScatter(pDst + 2, indices, mask);
w.MaskScatter(pDst + 3, indices, mask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MaskScatter(TNumber* pDst, int* pIndices, TLane mask)
{
x.MaskScatter(pDst + 0, pIndices, mask);
y.MaskScatter(pDst + 1, pIndices, mask);
z.MaskScatter(pDst + 2, pIndices, mask);
w.MaskScatter(pDst + 3, pIndices, mask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MaskScatter(ref TNumber dst, TLane indices, TLane mask)
{
x.MaskScatter(ref Unsafe.Add(ref dst, 0), indices, mask);
y.MaskScatter(ref Unsafe.Add(ref dst, 1), indices, mask);
z.MaskScatter(ref Unsafe.Add(ref dst, 2), indices, mask);
w.MaskScatter(ref Unsafe.Add(ref dst, 3), indices, mask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MaskScatter(ref TNumber dst, int* pIndices, TLane mask)
{
x.MaskScatter(ref Unsafe.Add(ref dst, 0), pIndices, mask);
y.MaskScatter(ref Unsafe.Add(ref dst, 1), pIndices, mask);
z.MaskScatter(ref Unsafe.Add(ref dst, 2), pIndices, mask);
w.MaskScatter(ref Unsafe.Add(ref dst, 3), pIndices, mask);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector4<TLane, TNumber> operator -(in Vector4<TLane, TNumber> vector)
{

73
Misaki.HighPerformance.Mathematics.SPMD/Templates/Vector{T}Helper.ttinclude
View File

@@ -95,22 +95,13 @@ public unsafe struct {typeName} : IEquatable<{typeName}>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Store({TNumber}* pDst)
{{
var width = {TLane}.LaneWidth;
{ForEachDimension(dimension, 8, Environment.NewLine, (dim, sb) => sb.Append($"var {components[dim]} = stackalloc {TNumber}[width];"))}
{ForEachDimension(dimension, 8, Environment.NewLine, (dim, sb) => sb.Append($"this.{components[dim]}.Store({components[dim]});"))}
for (var i = 0; i < width; i++)
{{
{ForEachDimension(dimension, 12, Environment.NewLine, (dim, sb) => sb.Append($"pDst[i * {dimension} + {dim}] = {components[dim]}[i];"))}
}}
{ForEachDimension(dimension, 8, Environment.NewLine, (dim, sb) => sb.Append($"{components[dim]}.Store(pDst + {dim} * TLane.LaneWidth);"))}
}}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Store(ref {TNumber} dst)
{{
Store(({TNumber}*)Unsafe.AsPointer(ref dst));
{ForEachDimension(dimension, 8, Environment.NewLine, (dim, sb) => sb.Append($"{components[dim]}.Store(ref Unsafe.Add(ref dst, {dim} * TLane.LaneWidth));"))}
}}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -125,6 +116,66 @@ public unsafe struct {typeName} : IEquatable<{typeName}>
{ForEachDimension(dimension, 8, Environment.NewLine, (dim, sb) => sb.Append($"this.{components[dim]}.Store(ref {components[dim]});"))}
}}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void CompressStore({TNumber}* pDst, {typeName} mask)
{{
{ForEachDimension(dimension, 8, Environment.NewLine, (dim, sb) => sb.Append($"{components[dim]}.CompressStore(pDst + {dim} * TLane.LaneWidth, mask.{components[dim]});"))}
}}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void CompressStore(ref {TNumber} dst, {typeName} mask)
{{
{ForEachDimension(dimension, 8, Environment.NewLine, (dim, sb) => sb.Append($"{components[dim]}.CompressStore(ref Unsafe.Add(ref dst, {dim} * TLane.LaneWidth), mask.{components[dim]});"))}
}}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Scatter({TNumber}* pDst, {TLane} indices)
{{
{ForEachDimension(dimension, 8, Environment.NewLine, (dim, sb) => sb.Append($"{components[dim]}.Scatter(pDst + {dim}, indices);"))}
}}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Scatter({TNumber}* pDst, int* pIndices)
{{
{ForEachDimension(dimension, 8, Environment.NewLine, (dim, sb) => sb.Append($"{components[dim]}.Scatter(pDst + {dim}, pIndices);"))}
}}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Scatter(ref {TNumber} dst, {TLane} indices)
{{
{ForEachDimension(dimension, 8, Environment.NewLine, (dim, sb) => sb.Append($"{components[dim]}.Scatter(ref Unsafe.Add(ref dst, {dim}), indices);"))}
}}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Scatter(ref {TNumber} dst, int* pIndices)
{{
{ForEachDimension(dimension, 8, Environment.NewLine, (dim, sb) => sb.Append($"{components[dim]}.Scatter(ref Unsafe.Add(ref dst, {dim}), pIndices);"))}
}}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MaskScatter({TNumber}* pDst, {TLane} indices, {TLane} mask)
{{
{ForEachDimension(dimension, 8, Environment.NewLine, (dim, sb) => sb.Append($"{components[dim]}.MaskScatter(pDst + {dim}, indices, mask);"))}
}}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MaskScatter({TNumber}* pDst, int* pIndices, {TLane} mask)
{{
{ForEachDimension(dimension, 8, Environment.NewLine, (dim, sb) => sb.Append($"{components[dim]}.MaskScatter(pDst + {dim}, pIndices, mask);"))}
}}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MaskScatter(ref {TNumber} dst, {TLane} indices, {TLane} mask)
{{
{ForEachDimension(dimension, 8, Environment.NewLine, (dim, sb) => sb.Append($"{components[dim]}.MaskScatter(ref Unsafe.Add(ref dst, {dim}), indices, mask);"))}
}}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MaskScatter(ref {TNumber} dst, int* pIndices, {TLane} mask)
{{
{ForEachDimension(dimension, 8, Environment.NewLine, (dim, sb) => sb.Append($"{components[dim]}.MaskScatter(ref Unsafe.Add(ref dst, {dim}), pIndices, mask);"))}
}}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static {typeName} operator -(in {typeName} vector)
{{

32
Misaki.HighPerformance.Mathematics.SPMD/Templates/WideLane.gen.cs
View File

@@ -13,58 +13,42 @@ public readonly unsafe partial struct WideLane<TNumber> : ISPMDLane<WideLane<TNu
{
if (typeof(TNumber) == typeof(float) && typeof(TOtherNumber) == typeof(int))
{
ref var vFrom = ref Unsafe.As<Vector<TNumber>, Vector<float>>(ref Unsafe.AsRef(in value));
var vTo = Vector.ConvertToInt32(vFrom);
return Unsafe.As<Vector<int>, TOther>(ref vTo);
return Unsafe.BitCast<Vector<int>, TOther>(Vector.ConvertToInt32(Unsafe.BitCast<Vector<TNumber>, Vector<float>>(value)));
}
if (typeof(TNumber) == typeof(float) && typeof(TOtherNumber) == typeof(uint))
{
ref var vFrom = ref Unsafe.As<Vector<TNumber>, Vector<float>>(ref Unsafe.AsRef(in value));
var vTo = Vector.ConvertToUInt32(vFrom);
return Unsafe.As<Vector<uint>, TOther>(ref vTo);
return Unsafe.BitCast<Vector<uint>, TOther>(Vector.ConvertToUInt32(Unsafe.BitCast<Vector<TNumber>, Vector<float>>(value)));
}
if (typeof(TNumber) == typeof(double) && typeof(TOtherNumber) == typeof(long))
{
ref var vFrom = ref Unsafe.As<Vector<TNumber>, Vector<double>>(ref Unsafe.AsRef(in value));
var vTo = Vector.ConvertToInt64(vFrom);
return Unsafe.As<Vector<long>, TOther>(ref vTo);
return Unsafe.BitCast<Vector<long>, TOther>(Vector.ConvertToInt64(Unsafe.BitCast<Vector<TNumber>, Vector<double>>(value)));
}
if (typeof(TNumber) == typeof(double) && typeof(TOtherNumber) == typeof(ulong))
{
ref var vFrom = ref Unsafe.As<Vector<TNumber>, Vector<double>>(ref Unsafe.AsRef(in value));
var vTo = Vector.ConvertToUInt64(vFrom);
return Unsafe.As<Vector<ulong>, TOther>(ref vTo);
return Unsafe.BitCast<Vector<ulong>, TOther>(Vector.ConvertToUInt64(Unsafe.BitCast<Vector<TNumber>, Vector<double>>(value)));
}
if (typeof(TNumber) == typeof(int) && typeof(TOtherNumber) == typeof(float))
{
ref var vFrom = ref Unsafe.As<Vector<TNumber>, Vector<int>>(ref Unsafe.AsRef(in value));
var vTo = Vector.ConvertToSingle(vFrom);
return Unsafe.As<Vector<float>, TOther>(ref vTo);
return Unsafe.BitCast<Vector<float>, TOther>(Vector.ConvertToSingle(Unsafe.BitCast<Vector<TNumber>, Vector<int>>(value)));
}
if (typeof(TNumber) == typeof(uint) && typeof(TOtherNumber) == typeof(float))
{
ref var vFrom = ref Unsafe.As<Vector<TNumber>, Vector<uint>>(ref Unsafe.AsRef(in value));
var vTo = Vector.ConvertToSingle(vFrom);
return Unsafe.As<Vector<float>, TOther>(ref vTo);
return Unsafe.BitCast<Vector<float>, TOther>(Vector.ConvertToSingle(Unsafe.BitCast<Vector<TNumber>, Vector<uint>>(value)));
}
if (typeof(TNumber) == typeof(long) && typeof(TOtherNumber) == typeof(double))
{
ref var vFrom = ref Unsafe.As<Vector<TNumber>, Vector<long>>(ref Unsafe.AsRef(in value));
var vTo = Vector.ConvertToDouble(vFrom);
return Unsafe.As<Vector<double>, TOther>(ref vTo);
return Unsafe.BitCast<Vector<double>, TOther>(Vector.ConvertToDouble(Unsafe.BitCast<Vector<TNumber>, Vector<long>>(value)));
}
if (typeof(TNumber) == typeof(ulong) && typeof(TOtherNumber) == typeof(double))
{
ref var vFrom = ref Unsafe.As<Vector<TNumber>, Vector<ulong>>(ref Unsafe.AsRef(in value));
var vTo = Vector.ConvertToDouble(vFrom);
return Unsafe.As<Vector<double>, TOther>(ref vTo);
return Unsafe.BitCast<Vector<double>, TOther>(Vector.ConvertToDouble(Unsafe.BitCast<Vector<TNumber>, Vector<ulong>>(value)));
}
var casted = stackalloc TOtherNumber[LaneWidth];

4
Misaki.HighPerformance.Mathematics.SPMD/Templates/WideLane.tt
View File

@@ -33,9 +33,7 @@ public readonly unsafe partial struct WideLane<TNumber> : ISPMDLane<WideLane<TNu
<# foreach (var c in conversions) { #>
if (typeof(TNumber) == typeof(<#= c.From #>) && typeof(TOtherNumber) == typeof(<#= c.To #>))
{
ref var vFrom = ref Unsafe.As<Vector<TNumber>, Vector<<#= c.From #>>>(ref Unsafe.AsRef(in value));
var vTo = <#= c.Method #>(vFrom);
return Unsafe.As<Vector<<#= c.To #>>, TOther>(ref vTo);
return Unsafe.BitCast<Vector<<#= c.To #>>, TOther>(<#= c.Method #>(Unsafe.BitCast<Vector<TNumber>, Vector<<#= c.From #>>>(value)));
}
<# } #>