feat(amd64): encode the GOROOT instruction families
Assisted-by: GLM 5.3 Flash
This commit is contained in:
Vendored
+69
@@ -0,0 +1,69 @@
|
||||
// Atomics and carry-extending multi-word arithmetic: exchange,
|
||||
// compare-exchange, exchange-add, ADCX/ADOX and the CRC-32 accumulator
|
||||
// family. Every result is folded back so no instruction is dead.
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
// func xchg(p *uint64, v uint64) uint64
|
||||
TEXT ·xchg(SB), NOSPLIT, $0-24
|
||||
MOVQ p+0(FP), AX
|
||||
MOVQ v+8(FP), BX
|
||||
XCHGQ BX, (AX)
|
||||
XCHGQ BX, CX
|
||||
XCHGL BX, CX
|
||||
XCHGW BX, CX
|
||||
XCHGB BL, CL
|
||||
MOVQ AX, ret+16(FP)
|
||||
RET
|
||||
|
||||
// func cmpxchg(p *uint64, old, new uint64) uint8
|
||||
TEXT ·cmpxchg(SB), NOSPLIT, $0-25
|
||||
MOVQ p+0(FP), AX
|
||||
MOVQ old+8(FP), BX
|
||||
MOVQ new+16(FP), CX
|
||||
CMPXCHGQ CX, (AX)
|
||||
CMPXCHGL CX, BX
|
||||
CMPXCHGW CX, BX
|
||||
CMPXCHGB CL, BL
|
||||
SETEQ AL
|
||||
MOVB AL, ret+24(FP)
|
||||
RET
|
||||
|
||||
// func xadd(p *uint64, v uint64) uint64
|
||||
TEXT ·xadd(SB), NOSPLIT, $0-24
|
||||
MOVQ p+0(FP), AX
|
||||
MOVQ v+8(FP), BX
|
||||
XADDQ BX, (AX)
|
||||
XADDL BX, CX
|
||||
XADDW BX, CX
|
||||
XADDB BL, CL
|
||||
MOVQ AX, ret+16(FP)
|
||||
RET
|
||||
|
||||
// func adcx_adox(lo, hi, x, y uint64) uint64
|
||||
TEXT ·adcx_adox(SB), NOSPLIT, $0-40
|
||||
MOVQ lo+0(FP), AX
|
||||
MOVQ hi+8(FP), DX
|
||||
MOVQ x+16(FP), BX
|
||||
MOVQ y+24(FP), CX
|
||||
ADCXQ BX, AX
|
||||
ADOXQ CX, DX
|
||||
ADCXL BX, AX
|
||||
ADOXL CX, DX
|
||||
XORQ BX, BX
|
||||
ADCXQ BX, AX
|
||||
MOVQ AX, ret+32(FP)
|
||||
RET
|
||||
|
||||
// func crc32(crc uint32, p *byte, n int) uint32
|
||||
TEXT ·crc32(SB), NOSPLIT, $0-28
|
||||
MOVL crc+0(FP), AX
|
||||
MOVQ p+8(FP), SI
|
||||
MOVQ n+16(FP), CX
|
||||
CRC32B (SI), AX
|
||||
CRC32Q (SI), CX
|
||||
CRC32L (SI), AX
|
||||
MOVW (SI), DX
|
||||
CRC32W DX, AX
|
||||
MOVL AX, ret+24(FP)
|
||||
RET
|
||||
Vendored
+102
@@ -0,0 +1,102 @@
|
||||
// The AVX/AVX-512 gap families: fused scalar multiply-add, carries through
|
||||
// GF(2^8) affine transforms, population counts, non-temporal stores, mask
|
||||
// moves and the KMOV widths. Every result is folded back so no instruction
|
||||
// is dead.
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
// func avxblend(a, b []float64) float64
|
||||
TEXT ·avxblend(SB), NOSPLIT, $0-56
|
||||
MOVQ a_base+0(FP), SI
|
||||
MOVQ b_base+24(FP), DI
|
||||
VMOVUPD (SI), Y0
|
||||
VMOVUPD (DI), Y1
|
||||
VXORPS Y2, Y2, Y2
|
||||
VSHUFPD $5, Y0, Y1, Y3
|
||||
VMOVUPD Y3, (SI)
|
||||
VPBLENDD $3, Y0, Y1, Y4
|
||||
VPERM2F128 $1, Y4, Y0, Y0
|
||||
VEXTRACTF128 $1, Y0, X1
|
||||
VZEROALL
|
||||
VMOVSD X1, ret+48(FP)
|
||||
RET
|
||||
|
||||
// func avxint(p *byte, n int) uint64
|
||||
TEXT ·avxint(SB), NOSPLIT, $0-24
|
||||
MOVQ p+0(FP), SI
|
||||
VMOVDQU (SI), Y0
|
||||
VPCMPEQB Y0, Y0, Y1
|
||||
VPSLLDQ $2, X0, X0
|
||||
VPSRLDQ $4, Y0, Y0
|
||||
VPALIGNR $3, X0, X1, X1
|
||||
VPCLMULQDQ $0, X0, X1, X2
|
||||
VGF2P8AFFINEQB $7, X2, X0, X3
|
||||
VPOPCNTB X3, X4
|
||||
VPOPCNTD Y0, Y5
|
||||
VPERMI2B X0, X1, X2
|
||||
VPTEST X0, X0
|
||||
VPMOVMSKB X1, AX
|
||||
VZEROUPPER
|
||||
MOVQ AX, ret+16(FP)
|
||||
RET
|
||||
|
||||
// func avxnt(p *float64)
|
||||
TEXT ·avxnt(SB), NOSPLIT, $0-8
|
||||
MOVQ p+0(FP), DI
|
||||
VMOVUPD (DI), Y0
|
||||
VADDPD Y0, Y0, Y0
|
||||
VMOVNTDQ Y0, (DI)
|
||||
VMOVNTDQ X0, 16(DI)
|
||||
VZEROALL
|
||||
RET
|
||||
|
||||
// func avxmas(a, b []float64) float64
|
||||
TEXT ·avxmas(SB), NOSPLIT, $0-56
|
||||
MOVQ a_base+0(FP), SI
|
||||
MOVQ b_base+24(FP), DI
|
||||
VMOVSD (SI), X0
|
||||
VMOVSD (DI), X1
|
||||
VFMADD213SD X1, X0, X0
|
||||
VFNMADD231SD X1, X0, X0
|
||||
VADDSD X1, X0, X0
|
||||
VMOVSD X0, ret+48(FP)
|
||||
RET
|
||||
|
||||
// func avxgpr(x, y uint64) uint64
|
||||
TEXT ·avxgpr(SB), NOSPLIT, $0-24
|
||||
MOVQ x+0(FP), AX
|
||||
MOVQ y+8(FP), BX
|
||||
ANDNL BX, AX, CX
|
||||
MULXQ BX, DX, SI
|
||||
RORXL $3, AX, CX
|
||||
RORXQ $7, BX, SI
|
||||
MOVQ CX, ret+16(FP)
|
||||
RET
|
||||
|
||||
// func avxmask(kin uint8, p *byte) uint8
|
||||
TEXT ·avxmask(SB), NOSPLIT, $0-17
|
||||
MOVQ p+8(FP), SI
|
||||
KMOVB kin+0(FP), K1
|
||||
KMOVB K1, K2
|
||||
KMOVW K2, K1
|
||||
KMOVD K1, K3
|
||||
KMOVQ K3, K4
|
||||
KMOVB K4, K1
|
||||
KMOVB K1, AX
|
||||
KMOVD K1, (SI)
|
||||
MOVB AL, ret+8(FP)
|
||||
RET
|
||||
|
||||
// func avx512(p *uint64, n int) uint64
|
||||
TEXT ·avx512(SB), NOSPLIT, $0-24
|
||||
MOVQ p+0(FP), SI
|
||||
VMOVDQU64 (SI), Z0
|
||||
VPORQ Z0, Z0, Z1
|
||||
VPOPCNTQ Z1, Z2
|
||||
VPERMB Z1, Z0, Z2
|
||||
VPXORD Z2, Z1, Z0
|
||||
VMOVDQA64 Z0, (SI)
|
||||
VZEROUPPER
|
||||
XORQ AX, AX
|
||||
MOVQ AX, ret+16(FP)
|
||||
RET
|
||||
Vendored
+57
@@ -0,0 +1,57 @@
|
||||
// The AES-NI, SHA and carry-less multiply round instructions as GOROOT's
|
||||
// crypto kernels spell them. Every result is folded back so no instruction
|
||||
// is dead.
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
// func aesround(blk, rk *byte)
|
||||
TEXT ·aesround(SB), NOSPLIT, $0-16
|
||||
MOVQ blk+0(FP), SI
|
||||
MOVQ rk+8(FP), DI
|
||||
MOVOU (SI), X0
|
||||
MOVOU (DI), X1
|
||||
AESENC X1, X0
|
||||
AESENCLAST X1, X0
|
||||
AESDEC X1, X0
|
||||
AESDECLAST X1, X0
|
||||
AESIMC X1, X2
|
||||
AESKEYGENASSIST $1, X1, X3
|
||||
MOVOU X0, (SI)
|
||||
MOVOU X2, (DI)
|
||||
RET
|
||||
|
||||
// func sha1block(p *byte, n int, h *[5]uint32)
|
||||
TEXT ·sha1block(SB), NOSPLIT, $0-24
|
||||
MOVQ p+0(FP), SI
|
||||
MOVQ h+16(FP), DI
|
||||
MOVOU (SI), X0
|
||||
MOVOU 16(SI), X1
|
||||
SHA1RNDS4 $0, X1, X0
|
||||
SHA1NEXTE X1, X0
|
||||
SHA1MSG1 X1, X2
|
||||
SHA1MSG2 X1, X2
|
||||
MOVOU X0, (DI)
|
||||
RET
|
||||
|
||||
// func sha256block(p *byte, n int, h *[8]uint32)
|
||||
TEXT ·sha256block(SB), NOSPLIT, $0-24
|
||||
MOVQ p+0(FP), SI
|
||||
MOVQ h+16(FP), DI
|
||||
MOVOU (SI), X0
|
||||
MOVOU 16(SI), X1
|
||||
SHA256RNDS2 X0, X1, X0
|
||||
SHA256MSG1 X1, X2
|
||||
SHA256MSG2 X1, X2
|
||||
MOVOU X0, (DI)
|
||||
RET
|
||||
|
||||
// func pclmul(a, b *byte)
|
||||
TEXT ·pclmul(SB), NOSPLIT, $0-16
|
||||
MOVQ a+0(FP), SI
|
||||
MOVQ b+8(FP), DI
|
||||
MOVOU (SI), X0
|
||||
MOVOU (DI), X1
|
||||
PCLMULQDQ $0, X1, X0
|
||||
PCLMULQDQ $17, (DI), X0
|
||||
MOVOU X0, (SI)
|
||||
RET
|
||||
Vendored
+76
@@ -0,0 +1,76 @@
|
||||
// Carry arithmetic, rotates, unsigned/signed division and bit tests: the
|
||||
// scalar families GOROOT's big-number and crypto kernels use. Every result
|
||||
// is folded back so no instruction is dead.
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
// func carry(a, b uint64) uint64
|
||||
TEXT ·carry(SB), NOSPLIT, $0-24
|
||||
MOVQ a+0(FP), AX
|
||||
MOVQ b+8(FP), BX
|
||||
ADDQ BX, AX
|
||||
ADCQ $0, AX
|
||||
MOVQ BX, CX
|
||||
SBBQ $1, CX
|
||||
ADCL BX, AX
|
||||
ADCB AL, BL
|
||||
ADCW $7, CX
|
||||
MOVQ AX, ret+16(FP)
|
||||
RET
|
||||
|
||||
// func borrow(a, b uint64) uint64
|
||||
TEXT ·borrow(SB), NOSPLIT, $0-24
|
||||
MOVQ a+0(FP), AX
|
||||
MOVQ b+8(FP), BX
|
||||
SUBQ BX, AX
|
||||
SBBQ $0, AX
|
||||
SBBQ BX, CX
|
||||
MOVQ AX, ret+16(FP)
|
||||
RET
|
||||
|
||||
// func rot(x uint64, n uint32) uint64
|
||||
TEXT ·rot(SB), NOSPLIT, $0-24
|
||||
MOVQ x+0(FP), AX
|
||||
MOVL n+8(FP), CX
|
||||
ROLQ CL, AX
|
||||
RORQ $7, AX
|
||||
ROLL $1, AX
|
||||
RORL CL, AX
|
||||
RCLQ $1, AX
|
||||
RCRQ CL, AX
|
||||
ROLW $3, AX
|
||||
SALQ $2, AX
|
||||
SALB $1, AX
|
||||
MOVQ AX, ret+8(FP)
|
||||
RET
|
||||
|
||||
// func muldiv(a, b uint64) uint64
|
||||
TEXT ·muldiv(SB), NOSPLIT, $0-24
|
||||
MOVQ a+0(FP), AX
|
||||
MOVQ b+8(FP), BX
|
||||
MULQ BX
|
||||
MULQ (BX)
|
||||
MOVL (BX), CX
|
||||
MULL CX
|
||||
DIVQ BX
|
||||
IDIVQ BX
|
||||
MOVL a+0(FP), AX
|
||||
DIVL CX
|
||||
IDIVL CX
|
||||
MOVQ AX, ret+16(FP)
|
||||
RET
|
||||
|
||||
// func bitfield(w *uint64) uint64
|
||||
TEXT ·bitfield(SB), NOSPLIT, $0-16
|
||||
MOVQ (DI), AX
|
||||
MOVQ (DI), CX
|
||||
BTQ AX, CX
|
||||
BTQ $3, (DI)
|
||||
BTL AX, CX
|
||||
BTW $1, CX
|
||||
BTSQ $5, AX
|
||||
BTRQ AX, CX
|
||||
BTCQ $7, (DI)
|
||||
SETCS AL
|
||||
MOVQ AX, ret+8(FP)
|
||||
RET
|
||||
Vendored
+77
@@ -0,0 +1,77 @@
|
||||
// The legacy SSE gap families: scalar compares and square roots, the Plan 9
|
||||
// packed spellings, shuffles, lane extracts and inserts, packed integer
|
||||
// shifts and the octa moves. Every result is folded back so no instruction
|
||||
// is dead.
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
// func cmporder(a, b *float64) int
|
||||
TEXT ·cmporder(SB), NOSPLIT, $0-24
|
||||
MOVQ a+0(FP), SI
|
||||
MOVQ b+8(FP), DI
|
||||
MOVSD (SI), X0
|
||||
MOVSD (DI), X1
|
||||
ANDNPD X0, X2
|
||||
ANDNPS X0, X3
|
||||
COMISD X0, X1
|
||||
SQRTSD X0, X2
|
||||
CMPSD X0, X1, $5
|
||||
MOVL SI, CX
|
||||
SETPL CL
|
||||
MOVL CX, ret+16(FP)
|
||||
RET
|
||||
|
||||
// func packed(w *uint64) uint64
|
||||
TEXT ·packed(SB), NOSPLIT, $0-16
|
||||
MOVQ w+0(FP), SI
|
||||
MOVO (SI), X0
|
||||
MOVOA (SI), X1
|
||||
PADDL X0, X1
|
||||
PSUBL X0, X1
|
||||
PCMPEQL X0, X1
|
||||
PUNPCKLBW X0, X1
|
||||
PSHUFL $27, X0, X2
|
||||
MOVOU X2, (SI)
|
||||
MOVQ (SI), AX
|
||||
MOVQ AX, ret+8(FP)
|
||||
RET
|
||||
|
||||
// func lanes(p *byte, buf *byte)
|
||||
TEXT ·lanes(SB), NOSPLIT, $0-16
|
||||
MOVQ p+0(FP), SI
|
||||
MOVQ buf+8(FP), DI
|
||||
MOVO (SI), X0
|
||||
MOVQ SI, AX
|
||||
PINSRB $1, AX, X0
|
||||
PINSRW $2, AX, X0
|
||||
PINSRD $3, AX, X0
|
||||
PINSRQ $1, AX, X0
|
||||
PEXTRB $1, X0, AX
|
||||
PEXTRW $2, X0, AX
|
||||
PEXTRD $3, X0, AX
|
||||
PEXTRQ $1, X0, CX
|
||||
PCMPESTRI $4, X0, X0
|
||||
MOVB AL, (DI)
|
||||
MOVOU X0, (SI)
|
||||
RET
|
||||
|
||||
// func shifts(p *uint64)
|
||||
TEXT ·shifts(SB), NOSPLIT, $0-8
|
||||
MOVQ p+0(FP), SI
|
||||
MOVO (SI), X0
|
||||
MOVO X0, X1
|
||||
PSLLW $3, X0
|
||||
PSRLW $1, X1
|
||||
PSRAW $2, X0
|
||||
PSLLL $4, X0
|
||||
PSRLL $5, X1
|
||||
PSRAL $1, X0
|
||||
PSLLQ $7, X0
|
||||
PSRLQ $9, X1
|
||||
PSLLL X1, X0
|
||||
PSRLQ X0, X1
|
||||
PSLLDQ $2, X0
|
||||
PSRLDQ $4, X1
|
||||
MOVOU X0, (SI)
|
||||
MOVOU X1, 16(SI)
|
||||
RET
|
||||
Vendored
+76
@@ -0,0 +1,76 @@
|
||||
// System, string-primitive and x87 families: flag register moves, the
|
||||
// serialising instructions, MOVS/STOS, the MXCSR pair, scalar float-to-int
|
||||
// conversions and FMOVD. Every result is folded back so no instruction is
|
||||
// dead.
|
||||
|
||||
#include "textflag.h"
|
||||
|
||||
// func system(x uint64) uint64
|
||||
TEXT ·system(SB), NOSPLIT, $0-16
|
||||
MOVQ x+0(FP), AX
|
||||
PUSHFQ
|
||||
POPFQ
|
||||
CPUID
|
||||
RDTSC
|
||||
RDTSCP
|
||||
SYSCALL
|
||||
XGETBV
|
||||
PAUSE
|
||||
LFENCE
|
||||
MFENCE
|
||||
SFENCE
|
||||
UNDEF
|
||||
XORQ AX, BX
|
||||
MOVQ BX, ret+8(FP)
|
||||
RET
|
||||
|
||||
// func stringprim(p *byte, n int) uint64
|
||||
TEXT ·stringprim(SB), NOSPLIT, $0-24
|
||||
MOVQ p+0(FP), DI
|
||||
MOVQ n+8(FP), CX
|
||||
LEAQ buf<>(SB), AX
|
||||
MOVQ AX, SI
|
||||
CLD
|
||||
MOVSB
|
||||
MOVSW
|
||||
MOVSL
|
||||
MOVSQ
|
||||
STOSB
|
||||
STOSQ
|
||||
STOSL
|
||||
STOSW
|
||||
MOVQ DI, ret+16(FP)
|
||||
RET
|
||||
|
||||
DATA buf<>+0x00(SB)/8, $0
|
||||
|
||||
GLOBL buf<>(SB), NOPTR, $8
|
||||
|
||||
// func intgate(x uint64) uint64
|
||||
TEXT ·intgate(SB), NOSPLIT, $0-16
|
||||
MOVQ x+0(FP), AX
|
||||
INT $3
|
||||
MOVQ AX, ret+8(FP)
|
||||
RET
|
||||
|
||||
// func fpmxcsr(x float64, csr *uint32) int64
|
||||
TEXT ·fpmxcsr(SB), NOSPLIT, $0-24
|
||||
MOVQ x+0(FP), X0
|
||||
MOVQ csr+8(FP), AX
|
||||
STMXCSR (AX)
|
||||
LDMXCSR (AX)
|
||||
CVTSD2SL X0, CX
|
||||
CVTTSD2SQ X0, DX
|
||||
MOVL (AX), SI
|
||||
MOVQ SI, ret+8(FP)
|
||||
RET
|
||||
|
||||
// func fmove(p *float64) float64
|
||||
TEXT ·fmove(SB), NOSPLIT, $0-16
|
||||
MOVQ p+0(FP), AX
|
||||
FMOVD (AX), F0
|
||||
FMOVD F0, F1
|
||||
FMOVD F0, (AX)
|
||||
MOVQ (AX), AX
|
||||
MOVQ AX, ret+8(FP)
|
||||
RET
|
||||
Reference in New Issue
Block a user