feat(amd64): assemble the double-shift and static-SB operand shapes

Assisted-by: GLM 5.3 Flash

asm/encode.go

@@ -173,7 +173,7 @@ func (e *enc) encode(mnem string, ops []Operand) error {
 	case "INC", "DEC", "NEG", "NOT", "MUL", "DIV", "IDIV":
 		return e.encodeUnary(unaryOp[base], ops, size)
 	case "SHL", "SHR", "SAR", "SAL", "ROL", "ROR", "RCL", "RCR":
-		return e.encodeShift(shiftOp[base], ops, size)
+		return e.encodeShift(base, ops, size)
 	case "BT", "BTS", "BTR", "BTC":
 		return e.encodeBitTest(base, ops, size)
 	case "XCHG":

asm/encode_test.go

@@ -200,10 +200,60 @@ func TestUnary(t *testing.T) {
 func TestShift(t *testing.T) {
 	checkSyntax(t, "shl rdx, 0x2", "SHLQ", Imm(2), DX)
 	checkSyntax(t, "shl rdx, cl", "SHLQ", CL, DX)
+	checkSyntax(t, "shl rdx, cl", "SHLQ", CX, DX)
 	checkSyntax(t, "shl rdx, 0x1", "SHLQ", Imm(1), DX)
 	checkSyntax(t, "sar rcx, 0x1f", "SARQ", Imm(31), CX)
 }
 
+// TestDoubleShift pins the three-operand SHL/SHR form, which encodes as
+// SHLD/SHRD: go tool asm accepts it for SHL/SHR at W/L/Q widths and rejects
+// it for SAR, SAL, the rotates and the B width.  The byte pins mirror the
+// oracle's objdump output (48 0f a4 fe 0d for the first case, and so on).
+func TestDoubleShift(t *testing.T) {
+	cases := []struct {
+		name string
+		mnem string
+		ops  []Operand
+		want string // hex encoding
+	}{
+		{"SHLQ imm", "SHLQ", []Operand{Imm(0x0d), DI, SI}, "480fa4fe0d"},
+		{"SHLQ CX high regs", "SHLQ", []Operand{CX, Reg{idx: 8, size: 8}, Reg{idx: 9, size: 8}}, "4d0fa5c1"},
+		{"SHRQ imm", "SHRQ", []Operand{Imm(1), AX, CX}, "480facc101"},
+		{"SHLW imm", "SHLW", []Operand{Imm(1), AX, CX}, "660fa4c101"},
+		{"SHRD CL", "SHRQ", []Operand{CL, AX, CX}, "480fadc1"},
+		{"SHLD imm high regs", "SHLQ", []Operand{Imm(2), Reg{idx: 10, size: 8}, Reg{idx: 11, size: 8}}, "4d0fa4d302"},
+		{"SHRD imm max", "SHRQ", []Operand{Imm(63), Reg{idx: 9, size: 8}, Reg{idx: 15, size: 8}}, "4d0faccf3f"},
+	}
+	for _, c := range cases {
+		code, err := Encode(c.mnem, c.ops...)
+		if err != nil {
+			t.Errorf("%s: Encode: %v", c.name, err)
+			continue
+		}
+		if got := hexCompact(code); got != c.want {
+			t.Errorf("%s: bytes %s, want %s", c.name, got, c.want)
+		}
+	}
+	// Rejected forms: the oracle rejects every one of these.
+	rejected := []struct {
+		name string
+		mnem string
+		ops  []Operand
+	}{
+		{"SARQ three operands", "SARQ", []Operand{Imm(1), AX, CX}},
+		{"SALQ three operands", "SALQ", []Operand{Imm(1), AX, CX}},
+		{"ROLQ three operands", "ROLQ", []Operand{Imm(1), AX, CX}},
+		{"SHLB three operands", "SHLB", []Operand{Imm(1), AL, CL}},
+		{"SHRQ memory source", "SHRQ", []Operand{Imm(1), Ptr(AX, 0, 8), CX}},
+		{"SHRQ ECX count", "SHRQ", []Operand{Reg{idx: 1, size: 4}, AX, CX}},
+	}
+	for _, c := range rejected {
+		if _, err := Encode(c.mnem, c.ops...); err == nil {
+			t.Errorf("%s: Encode succeeded, want rejection", c.name)
+		}
+	}
+}
+
 func TestImul(t *testing.T) {
 	checkSyntax(t, "imul rdx, rcx", "IMULQ", CX, DX)
 	checkSyntax(t, "imul edx, edx, 0x3", "IMULL", Imm(3), DX, DX)
@@ -277,6 +327,12 @@ func TestSSEMoveGroundTruth(t *testing.T) {
 		{"MOVSD (SI),X1", "MOVSD", []Operand{Ptr(SI, 0, 8), vreg(t, "X1")}, "f20f100e", "MOVSD_XMM"},
 		{"MOVSD X1,X2", "MOVSD", []Operand{vreg(t, "X1"), vreg(t, "X2")}, "f20f10d1", "MOVSD_XMM"},
 		{"MOVSS X3,(DI)", "MOVSS", []Operand{vreg(t, "X3"), Ptr(DI, 0, 4)}, "f30f111f", "MOVSS"},
+		// Static-symbol (SB) references: the GOROOT crypto kernels load and
+		// store octa constants by name (MOVOU bswapMask<>+0(SB), X0).
+		{"MOVOU sym,X0", "MOVOU", []Operand{sbMem{size: 16, name: "bswapMask"}, vreg(t, "X0")}, "f30f6f0500000000", "MOVDQU"},
+		{"MOVOU X0,sym+8", "MOVOU", []Operand{vreg(t, "X0"), sbMem{size: 16, name: "bswapMask", addend: 8}}, "f30f7f0500000000", "MOVDQU"},
+		{"MOVO sym,X1", "MOVO", []Operand{sbMem{size: 16, name: "gcmPoly"}, vreg(t, "X1")}, "660f6f0d00000000", "MOVDQA"},
+		{"MOVO X2,sym", "MOVO", []Operand{vreg(t, "X2"), sbMem{size: 16, name: "gcmPoly"}}, "660f7f1500000000", "MOVDQA"},
 	}
 	for _, c := range cases {
 		code, err := Encode(c.mnem, c.ops...)

asm/instrs.go

@@ -498,13 +498,34 @@ func (e *enc) encodeUnary(op struct {
 
 // --- SHL/SHR/SAR ------------------------------------------------------------
 
-func (e *enc) encodeShift(digit int, ops []Operand, size int) error {
+// doubleShiftOp maps the two mnemonics whose three-operand form go tool asm
+// accepts to the SHLD/SHRD opcode pair (imm8 form, CL form).  SAR, SAL and
+// the rotates have no such form: the oracle rejects SARQ/ROLQ with three
+// operands, and so do we.
+var doubleShiftOp = map[string][2]byte{
+	"SHL": {0xA4, 0xA5}, // SHLD
+	"SHR": {0xAC, 0xAD}, // SHRD
+}
+
+// isShiftCountCL reports whether a count operand is the CL register or its
+// CX spelling: go tool asm accepts both (CX names the same low byte) and
+// rejects ECX/RCX.
+func isShiftCountCL(o Operand) bool {
+	reg, ok := o.(Reg)
+	return ok && reg.idx == 1 && (reg.size == 1 || reg.size == 2)
+}
+
+func (e *enc) encodeShift(base string, ops []Operand, size int) error {
+	digit := shiftOp[base]
+	if len(ops) == 3 {
+		return e.encodeDoubleShift(base, ops, size)
+	}
 	if len(ops) != 2 {
 		return fmt.Errorf("shift expects 2 operands, got %d", len(ops))
 	}
 	count, dst := ops[0], ops[1]
-	// Count is $1, %CL, or an imm8.
+	// Count is $1, CL (or its CX spelling), or an imm8.
-	if reg, ok := count.(Reg); ok && reg.idx == 1 && reg.size <= 1 {
+	if isShiftCountCL(count) {
 		// CL: 0xD2 (8-bit) / 0xD3.
 		op := byte(0xD3)
 		if size == 1 {
@@ -551,6 +572,44 @@ func (e *enc) encodeShift(digit int, ops []Operand, size int) error {
 	return e.emit(i)
 }
 
+// encodeDoubleShift emits the three-operand SHL/SHR form, which the Go
+// assembler spells as a shift but encodes as SHLD/SHRD (0F A4/A5, 0F AC/AD):
+// the first operand is the count ($imm or CL), the second feeds the vacated
+// bits (the reg field) and the third is the shifted value (the r/m field),
+// matching go tool asm byte for byte.  The W/L/Q widths exist; the oracle
+// rejects the three-operand B form and every SAR/rotate one.
+func (e *enc) encodeDoubleShift(base string, ops []Operand, size int) error {
+	opc, ok := doubleShiftOp[base]
+	if !ok || size == 1 {
+		return fmt.Errorf("%s: shift expects 2 operands, got %d", base, len(ops))
+	}
+	count, src, dst := ops[0], ops[1], ops[2]
+	srcReg, ok := src.(Reg)
+	if !ok {
+		return fmt.Errorf("%s: middle operand must be a register, like go tool asm", base)
+	}
+	i := newInstr(size, []byte{0x0F, opc[0]})
+	if isShiftCountCL(count) {
+		// CL (or CX) form: 0F A5/AD.
+		i.opcode[1] = opc[1]
+	} else {
+		imm, ok := count.(Imm)
+		if !ok {
+			return fmt.Errorf("shift count must be $1, CL or an immediate")
+		}
+		// The count is an unsigned imm8: the same range convention as the
+		// two-operand shift above.
+		if imm < 0 || imm > 255 {
+			return fmt.Errorf("shift count $%d is out of the 0..255 range", int64(imm))
+		}
+		i.imm = []byte{byte(imm)}
+	}
+	if err := setRMReg(i, srcReg.idx, srcReg.idx >= 8, false, dst, size); err != nil {
+		return err
+	}
+	return e.emit(i)
+}
+
 // --- IMUL -------------------------------------------------------------------
 
 func (e *enc) encodeImul(ops []Operand, size int) error {
@@ -986,12 +1045,12 @@ func (e *enc) encodeSSEMove(m sseMove, ops []Operand) error {
 		op = m.load
 		reg, rm = dstReg, src
 	case srcVec:
-		if _, ok := dst.(Mem); !ok {
+		if !isX86Mem(dst) {
 			return fmt.Errorf("SSE move: invalid destination operand")
 		}
 		reg, rm = srcReg, dst
 	case dstVec:
-		if _, ok := src.(Mem); !ok {
+		if !isX86Mem(src) {
 			return fmt.Errorf("SSE move: invalid source operand")
 		}
 		op = m.load

asm/operand.go

@@ -48,3 +48,16 @@ type sbMem struct {
 }
 
 func (sbMem) isOperand() {}
+
+// isX86Mem reports whether the operand is an amd64 memory reference: a base
+// or indexed Mem, or an SB-relative sbMem.  Encoders that gate on "memory in
+// this position" must accept both; the r/m emitters distinguish the two
+// themselves.
+func isX86Mem(o Operand) bool {
+	switch o.(type) {
+	case Mem, sbMem:
+		return true
+	default:
+		return false
+	}
+}

testdata/verify/doubleshift_amd64.s

@@ -0,0 +1,33 @@
+// The three-operand SHL/SHR forms, which go tool asm encodes as SHLD/SHRD:
+// immediate and CL (or its CX spelling) counts at the Q and W widths, next
+// to the two-operand CX-count spelling GOROOT's bignum kernels use.  Every
+// result is folded back so no instruction is dead.
+
+#include "textflag.h"
+
+// func dblshift(x, y uint64) uint64
+TEXT ·dblshift(SB), NOSPLIT, $0-24
+	MOVQ x+0(FP), SI
+	MOVQ y+8(FP), DI
+	MOVQ $12, CX
+	SHLQ $13, SI, DI
+	SHRQ $7, DI, SI
+	SHLQ CX, SI, DI
+	SHRQ CX, DI, SI
+	SHLQ CX, SI
+	SHLQ $9, DI
+	SHLW $1, SI, DI
+	SHRW $3, DI, SI
+	XORQ DI, SI
+	MOVQ SI, ret+16(FP)
+	RET
+
+// func dblshift32(a, b uint32) uint32
+TEXT ·dblshift32(SB), NOSPLIT, $0-12
+	MOVL a+0(FP), SI
+	MOVL b+4(FP), DI
+	SHLL $5, SI, DI
+	SHRL $2, DI, SI
+	XORL SI, DI
+	MOVL DI, ret+8(FP)
+	RET

testdata/verify/ssestatic_amd64.s

@@ -0,0 +1,27 @@
+// Legacy SSE octa moves against static (SB) symbols: the load and store
+// shapes GOROOT's AES-CTR, AES-GCM and P-256 kernels spell (MOVOU
+// bswapMask<>+0(SB), X0 and the reverse), including offsets into the symbol
+// and the aligned MOVO pair.  Every result is folded back so no instruction
+// is dead.
+
+#include "textflag.h"
+
+// func ssestatic() uint64
+TEXT ·ssestatic(SB), NOSPLIT, $0-8
+	MOVOU bswapMask<>+0(SB), X0
+	MOVOU bswapMask<>+8(SB), X1
+	MOVO  rodataMask<>+0(SB), X2
+	PXOR  X1, X0
+	PXOR  X2, X0
+	MOVOU X0, sink<>+0(SB)
+	MOVOU sink<>+0(SB), X3
+	PXOR  X3, X0
+	MOVQ  X0, AX
+	MOVQ  AX, ret+0(FP)
+	RET
+
+GLOBL bswapMask<>(SB), RODATA|NOPTR, $16
+
+GLOBL rodataMask<>(SB), RODATA|NOPTR, $16
+
+GLOBL sink<>(SB), NOPTR, $16

verify/groundtruth_test.go

@@ -122,6 +122,8 @@ func TestGroundTruthAMD64(t *testing.T) {
 		"../testdata/verify/crypto_amd64.s",
 		"../testdata/verify/sse_amd64.s",
 		"../testdata/verify/avx_amd64.s",
+		"../testdata/verify/doubleshift_amd64.s",
+		"../testdata/verify/ssestatic_amd64.s",
 	} {
 		t.Run(path, func(t *testing.T) {
 			f, errs := parser.Parse(path, mustRead(t, path))