103 lines
2.3 KiB
ArmAsm
103 lines
2.3 KiB
ArmAsm
|
// The AVX/AVX-512 gap families: fused scalar multiply-add, carries through
|
||
|
|
// GF(2^8) affine transforms, population counts, non-temporal stores, mask
|
||
|
|
// moves and the KMOV widths. Every result is folded back so no instruction
|
||
|
|
// is dead.
|
||
|
|
|
||
|
|
#include "textflag.h"
|
||
|
|
|
||
|
|
// func avxblend(a, b []float64) float64
|
||
|
|
TEXT ·avxblend(SB), NOSPLIT, $0-56
|
||
|
|
MOVQ a_base+0(FP), SI
|
||
|
|
MOVQ b_base+24(FP), DI
|
||
|
|
VMOVUPD (SI), Y0
|
||
|
|
VMOVUPD (DI), Y1
|
||
|
|
VXORPS Y2, Y2, Y2
|
||
|
|
VSHUFPD $5, Y0, Y1, Y3
|
||
|
|
VMOVUPD Y3, (SI)
|
||
|
|
VPBLENDD $3, Y0, Y1, Y4
|
||
|
|
VPERM2F128 $1, Y4, Y0, Y0
|
||
|
|
VEXTRACTF128 $1, Y0, X1
|
||
|
|
VZEROALL
|
||
|
|
VMOVSD X1, ret+48(FP)
|
||
|
|
RET
|
||
|
|
|
||
|
|
// func avxint(p *byte, n int) uint64
|
||
|
|
TEXT ·avxint(SB), NOSPLIT, $0-24
|
||
|
|
MOVQ p+0(FP), SI
|
||
|
|
VMOVDQU (SI), Y0
|
||
|
|
VPCMPEQB Y0, Y0, Y1
|
||
|
|
VPSLLDQ $2, X0, X0
|
||
|
|
VPSRLDQ $4, Y0, Y0
|
||
|
|
VPALIGNR $3, X0, X1, X1
|
||
|
|
VPCLMULQDQ $0, X0, X1, X2
|
||
|
|
VGF2P8AFFINEQB $7, X2, X0, X3
|
||
|
|
VPOPCNTB X3, X4
|
||
|
|
VPOPCNTD Y0, Y5
|
||
|
|
VPERMI2B X0, X1, X2
|
||
|
|
VPTEST X0, X0
|
||
|
|
VPMOVMSKB X1, AX
|
||
|
|
VZEROUPPER
|
||
|
|
MOVQ AX, ret+16(FP)
|
||
|
|
RET
|
||
|
|
|
||
|
|
// func avxnt(p *float64)
|
||
|
|
TEXT ·avxnt(SB), NOSPLIT, $0-8
|
||
|
|
MOVQ p+0(FP), DI
|
||
|
|
VMOVUPD (DI), Y0
|
||
|
|
VADDPD Y0, Y0, Y0
|
||
|
|
VMOVNTDQ Y0, (DI)
|
||
|
|
VMOVNTDQ X0, 16(DI)
|
||
|
|
VZEROALL
|
||
|
|
RET
|
||
|
|
|
||
|
|
// func avxmas(a, b []float64) float64
|
||
|
|
TEXT ·avxmas(SB), NOSPLIT, $0-56
|
||
|
|
MOVQ a_base+0(FP), SI
|
||
|
|
MOVQ b_base+24(FP), DI
|
||
|
|
VMOVSD (SI), X0
|
||
|
|
VMOVSD (DI), X1
|
||
|
|
VFMADD213SD X1, X0, X0
|
||
|
|
VFNMADD231SD X1, X0, X0
|
||
|
|
VADDSD X1, X0, X0
|
||
|
|
VMOVSD X0, ret+48(FP)
|
||
|
|
RET
|
||
|
|
|
||
|
|
// func avxgpr(x, y uint64) uint64
|
||
|
|
TEXT ·avxgpr(SB), NOSPLIT, $0-24
|
||
|
|
MOVQ x+0(FP), AX
|
||
|
|
MOVQ y+8(FP), BX
|
||
|
|
ANDNL BX, AX, CX
|
||
|
|
MULXQ BX, DX, SI
|
||
|
|
RORXL $3, AX, CX
|
||
|
|
RORXQ $7, BX, SI
|
||
|
|
MOVQ CX, ret+16(FP)
|
||
|
|
RET
|
||
|
|
|
||
|
|
// func avxmask(kin uint8, p *byte) uint8
|
||
|
|
TEXT ·avxmask(SB), NOSPLIT, $0-17
|
||
|
|
MOVQ p+8(FP), SI
|
||
|
|
KMOVB kin+0(FP), K1
|
||
|
|
KMOVB K1, K2
|
||
|
|
KMOVW K2, K1
|
||
|
|
KMOVD K1, K3
|
||
|
|
KMOVQ K3, K4
|
||
|
|
KMOVB K4, K1
|
||
|
|
KMOVB K1, AX
|
||
|
|
KMOVD K1, (SI)
|
||
|
|
MOVB AL, ret+8(FP)
|
||
|
|
RET
|
||
|
|
|
||
|
|
// func avx512(p *uint64, n int) uint64
|
||
|
|
TEXT ·avx512(SB), NOSPLIT, $0-24
|
||
|
|
MOVQ p+0(FP), SI
|
||
|
|
VMOVDQU64 (SI), Z0
|
||
|
|
VPORQ Z0, Z0, Z1
|
||
|
|
VPOPCNTQ Z1, Z2
|
||
|
|
VPERMB Z1, Z0, Z2
|
||
|
|
VPXORD Z2, Z1, Z0
|
||
|
|
VMOVDQA64 Z0, (SI)
|
||
|
|
VZEROUPPER
|
||
|
|
XORQ AX, AX
|
||
|
|
MOVQ AX, ret+16(FP)
|
||
|
|
RET
|