Files
gasm-sdk/asm/encode.go
T

289 lines
6.5 KiB
Go
Raw Normal View History

// Copyright (c) 2026 Petr Balvín <opensource@petrbalvin.org> (https://petrbalvin.org)
// SPDX-License-Identifier: BSD-3-Clause
package asm
import (
"fmt"
"strings"
)
// Encode encodes one Plan 9 instruction (mnemonic plus operands, in source
// order) into x86-64 machine code.
func Encode(mnemonic string, ops ...Operand) ([]byte, error) {
e := &enc{}
if err := e.encode(mnemonic, ops); err != nil {
return nil, err
}
return e.out, nil
}
type enc struct {
out []byte
}
func (e *enc) encode(mnem string, ops []Operand) error {
upper := strings.ToUpper(mnem)
// Fixed-name instructions (no size suffix).
switch {
case upper == "RET":
return e.encodeRet()
case upper == "NOP":
return e.emit(&instr{opcode: []byte{0x90}, modrm: -1, sib: -1})
case upper == "CALL":
return e.encodeJmpRel(ops, []byte{0xE8})
case upper == "JMP":
return e.encodeJmpRel(ops, []byte{0xE9})
}
if cc, ok := condCode(upper); ok {
return e.encodeJcc(cc, ops)
}
// VEX (AVX/AVX2) instructions: the trailing B/W/L/Q/D is part of the
// mnemonic, not a size suffix, so dispatch before splitSize.
if isVex(upper) {
return e.encodeVex(upper, ops)
}
base, size := splitSize(upper)
if size == 0 {
size = 8 // default operand size in 64-bit mode (e.g. PUSHQ)
}
switch base {
case "MOV":
return e.encodeMov(ops, size)
case "ADD", "SUB", "AND", "OR", "XOR", "CMP":
return e.encodeALU(aluOp[base], ops, size)
case "TEST":
return e.encodeTest(ops, size)
case "LEA":
return e.encodeLea(ops, size)
case "INC", "DEC", "NEG", "NOT":
return e.encodeUnary(unaryOp[base], ops, size)
case "SHL", "SHR", "SAR":
return e.encodeShift(shiftOp[base], ops, size)
case "IMUL":
return e.encodeImul(ops, size)
case "PUSH":
return e.encodePushPop(ops, true)
case "POP":
return e.encodePushPop(ops, false)
}
return fmt.Errorf("unsupported instruction %q", mnem)
}
// splitSize separates a trailing B/W/L/Q size suffix from the mnemonic.
func splitSize(upper string) (base string, size int) {
if upper == "" {
return upper, 0
}
switch upper[len(upper)-1] {
case 'B':
return upper[:len(upper)-1], 1
case 'W':
return upper[:len(upper)-1], 2
case 'L':
return upper[:len(upper)-1], 4
case 'Q':
return upper[:len(upper)-1], 8
}
return upper, 0
}
// --- instruction components -------------------------------------------------
type instr struct {
opSize16 bool
rexW bool
rexR bool
rexX bool
rexB bool
rexForced bool // REX needed even with all bits zero (8-bit low registers)
opcode []byte
modrm int // -1 if absent
sib int // -1 if absent
disp []byte
imm []byte
}
func (e *enc) emit(i *instr) error {
if i.opSize16 {
e.out = append(e.out, 0x66)
}
rex := byte(0)
if i.rexW {
rex |= 0x08
}
if i.rexR {
rex |= 0x04
}
if i.rexX {
rex |= 0x02
}
if i.rexB {
rex |= 0x01
}
if rex != 0 || i.rexForced {
e.out = append(e.out, 0x40|rex)
}
e.out = append(e.out, i.opcode...)
if i.modrm >= 0 {
e.out = append(e.out, byte(i.modrm))
}
if i.sib >= 0 {
e.out = append(e.out, byte(i.sib))
}
e.out = append(e.out, i.disp...)
e.out = append(e.out, i.imm...)
return nil
}
// newInstr starts an instruction with a size-derived REX.W and 0x66 prefix.
func newInstr(opSize int, opcode []byte) *instr {
return &instr{
opSize16: opSize == 2,
rexW: opSize == 8,
opcode: opcode,
modrm: -1,
sib: -1,
}
}
// --- ModR/M, SIB, displacement ----------------------------------------------
// setRM fills in the ModR/M (and SIB, displacement, REX bits) for an
// instruction whose reg field holds a real register `reg` and whose r/m field
// holds `rm`.
func setRM(i *instr, reg Reg, rm Operand, opSize int) error {
return setRMReg(i, reg.idx&7, reg.idx >= 8, reg.needsREX(opSize), rm, opSize)
}
// setRMDigit fills in the ModR/M for an instruction whose reg field is an
// opcode /digit extension (0–7), which carries none of the register REX rules.
func setRMDigit(i *instr, digit int, rm Operand, opSize int) error {
return setRMReg(i, digit, false, false, rm, opSize)
}
func setRMReg(i *instr, regField int, rexR, regForced bool, rm Operand, opSize int) error {
i.rexR = rexR
if regForced {
i.rexForced = true
}
switch r := rm.(type) {
case Reg:
i.rexB = r.idx >= 8
if r.needsREX(opSize) {
i.rexForced = true
}
i.modrm = 0xC0 | regField<<3 | (r.idx & 7)
return nil
case Mem:
return setMem(i, regField, r)
default:
return fmt.Errorf("invalid r/m operand %T", rm)
}
}
func setMem(i *instr, regField int, m Mem) error {
modrm, sib, disp, xBit, bBit, err := memComponents(regField, m)
if err != nil {
return err
}
i.modrm = modrm
i.sib = sib
i.disp = disp
i.rexX = xBit == 1
i.rexB = bBit == 1
return nil
}
// memComponents computes the ModR/M byte (with the given reg field), the SIB
// byte (-1 if none), the displacement bytes, and the high index/base bits, for
// a memory operand. It is shared by the REX (scalar) and VEX (vector) paths.
func memComponents(regField int, m Mem) (modrm, sib int, disp []byte, xBit, bBit int, err error) {
sib = -1
// RIP-relative: neither base nor index.
if !m.HasBase && !m.HasIndex {
return regField<<3 | 0x05, -1, le32(m.Disp), 0, 0, nil // mod=00, rm=101
}
needSIB := m.HasIndex || (m.HasBase && m.Base.idx&7 == 4)
var mod int
switch {
case !m.HasBase:
mod = 0
disp = le32(m.Disp)
case m.Base.idx&7 == 5 && m.Disp == 0:
mod = 1
disp = []byte{0}
case m.Disp == 0:
mod = 0
case fits8(m.Disp):
mod = 1
disp = []byte{byte(int8(m.Disp))}
default:
mod = 2
disp = le32(m.Disp)
}
if needSIB {
idxField := 4 // 100 = no index
if m.HasIndex {
idxField = m.Index.idx & 7
if m.Index.idx >= 8 {
xBit = 1
}
}
baseField := 5 // 101 = no base (with mod=00 → disp32)
if m.HasBase {
baseField = m.Base.idx & 7
if m.Base.idx >= 8 {
bBit = 1
}
}
return mod<<6 | regField<<3 | 0x04, scaleBits(m.Scale)<<6 | idxField<<3 | baseField, disp, xBit, bBit, nil
}
if m.Base.idx >= 8 {
bBit = 1
}
return mod<<6 | regField<<3 | (m.Base.idx & 7), -1, disp, 0, bBit, nil
}
func scaleBits(scale int) int {
switch scale {
case 2:
return 1
case 4:
return 2
case 8:
return 3
default:
return 0 // scale 1 (or unset)
}
}
func fits8(v int64) bool { return v >= -128 && v <= 127 }
func le32(v int64) []byte {
u := uint32(v)
return []byte{byte(u), byte(u >> 8), byte(u >> 16), byte(u >> 24)}
}
func le16(v int64) []byte {
u := uint16(v)
return []byte{byte(u), byte(u >> 8)}
}
func le64(v int64) []byte {
u := uint64(v)
b := make([]byte, 8)
for i := 0; i < 8; i++ {
b[i] = byte(u >> (8 * i))
}
return b
}