feat(verify): add runtime ABI checks with sentinel registers and red-zone canary

Assisted-by: Qwen 3.8 Max Preview

verify/abi_amd64.go

@@ -0,0 +1,130 @@
+// Copyright (c) 2026 Petr Balvín <opensource@petrbalvin.org> (https://petrbalvin.org)
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build amd64
+
+package verify
+
+import (
+	"encoding/binary"
+	"fmt"
+	"syscall"
+	"unsafe"
+)
+
+// abiResult records register-clobber violations detected by the ABI-checking
+// trampoline.  Bit 0: BP clobbered.  Bit 1: R14 clobbered.
+var abiResult uint64
+
+// savedBP holds the caller's frame pointer across the ABI-checked JIT call.
+// Referenced by enterJITChecked to satisfy go vet's save-before-clobber rule.
+var savedBP uintptr
+
+// leaveCheckedPtr is initialised by the linker from the GLOBL/DATA in
+// abi_amd64.s: it holds the raw address of leaveJITCheckedRaw (which has
+// no ABIInternal wrapper, so the JIT function RETs directly into it).
+var leaveCheckedPtr uintptr
+
+// enterJITChecked sets sentinels in BP and R14, switches to the prepared
+// stack and jumps to fn.
+//
+//go:nosplit
+func enterJITChecked(fn uintptr, stack uintptr)
+
+// leaveJITCheckedRaw is the raw return trampoline for ABI checks.  Its
+// address is obtained from the GLOBL in abi_amd64.s (leaveCheckedPtr),
+// which points to the .abi0 code — NOT the ABIInternal wrapper that this
+// declaration would generate.  The declaration exists solely to satisfy
+// go vet's "missing Go declaration" check.
+//
+//go:nosplit
+func leaveJITCheckedRaw()
+
+// ABIReport describes the result of an ABI-checking call.
+type ABIReport struct {
+	BPClobbered  bool // BP was modified by the function
+	R14Clobbered bool // R14 (goroutine pointer) was modified
+	RedZoneHit   bool // the 128-byte red zone below SP was written
+}
+
+// OK returns true when no violations were detected.
+func (r ABIReport) OK() bool {
+	return !r.BPClobbered && !r.R14Clobbered && !r.RedZoneHit
+}
+
+// String returns a human-readable summary.
+func (r ABIReport) String() string {
+	if r.OK() {
+		return "ABI clean"
+	}
+	s := "ABI violation:"
+	if r.BPClobbered {
+		s += " BP clobbered"
+	}
+	if r.R14Clobbered {
+		s += " R14 clobbered"
+	}
+	if r.RedZoneHit {
+		s += " red-zone written"
+	}
+	return s
+}
+
+// redZoneSize is the System V AMD64 red zone: 128 bytes below SP that a
+// leaf function may use without adjusting SP.  Go does not use the red zone,
+// so any write there is a bug.
+const redZoneSize = 128
+
+// redZoneFill is the byte pattern used to detect red-zone writes.
+const redZoneFill = 0xA5
+
+// CallChecked invokes the function with ABI sentinels and a red-zone
+// canary, returning both the argument block (with results) and an ABIReport.
+func CallChecked(fnAddr uintptr, args []byte) ([]byte, ABIReport, error) {
+	report := ABIReport{}
+
+	// Reset the global result.
+	abiResult = 0
+
+	// Prepare the stack: [red-zone canary][padding][leaveJITCheckedRaw][args...]
+	// The red zone sits below the initial SP, so the function would have to
+	// write below SP to corrupt it.
+	totalSize := redZoneSize + stackPad + 8 + len(args) + 64
+	stackMem, err := syscall.Mmap(-1, 0, totalSize,
+		syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_PRIVATE|syscall.MAP_ANON)
+	if err != nil {
+		return nil, report, fmt.Errorf("verify: stack mmap: %w", err)
+	}
+	defer syscall.Munmap(stackMem)
+
+	// Fill the red zone with the canary pattern.
+	for i := 0; i < redZoneSize; i++ {
+		stackMem[i] = redZoneFill
+	}
+
+	// Return address and args after the red zone and padding.
+	retOff := redZoneSize + stackPad
+	binary.LittleEndian.PutUint64(stackMem[retOff:retOff+8], uint64(leaveCheckedPtr))
+	copy(stackMem[retOff+8:], args)
+
+	stackBase := uintptr(unsafe.Pointer(&stackMem[retOff]))
+	enterJITChecked(fnAddr, stackBase)
+
+	// Read the register-clobber result.
+	res := abiResult
+	report.BPClobbered = res&1 != 0
+	report.R14Clobbered = res&2 != 0
+
+	// Check the red zone.
+	for i := 0; i < redZoneSize; i++ {
+		if stackMem[i] != redZoneFill {
+			report.RedZoneHit = true
+			break
+		}
+	}
+
+	// Copy out the argument area.
+	out := make([]byte, len(args))
+	copy(out, stackMem[retOff+8:retOff+8+len(args)])
+	return out, report, nil
+}