fix(debug): make ptrace sessions reliable on Go tracees
This commit is contained in:
@@ -28,6 +28,14 @@ Unreleased changes on the `development` branch.
|
|||||||
the hits per instruction and reports the executed instructions with
|
the hits per instruction and reports the executed instructions with
|
||||||
their hit counts, with the label coverage derived from the same run.
|
their hit counts, with the label coverage derived from the same run.
|
||||||
Expect the run to slow to ptrace speed.
|
Expect the run to slow to ptrace speed.
|
||||||
|
- **Debugger reliability fixes.** The ptrace session now drains runtime
|
||||||
|
signal-delivery-stops (a Go tracee reports SIGURG preemption to the
|
||||||
|
tracer) instead of mistaking them for the launch barrier, pins the
|
||||||
|
tracer thread the fork ran on (ptrace requests from another thread fail
|
||||||
|
with ESRCH), prefers the debuggee's reported code base over an RWX scan,
|
||||||
|
and single-steps over a hit breakpoint so resuming cannot re-trap on the
|
||||||
|
same instruction. A ptrace integration test
|
||||||
|
(`debug/ptrace_integration_test.go`) drives a real session end to end.
|
||||||
- **FP register display on riscv64 and loong64.** The debugger `regs`
|
- **FP register display on riscv64 and loong64.** The debugger `regs`
|
||||||
command shows the 32 FP registers plus fcsr (and fcc on loong64) via
|
command shows the 32 FP registers plus fcsr (and fcc on loong64) via
|
||||||
`PTRACE_GETREGSET`, where it previously printed nothing.
|
`PTRACE_GETREGSET`, where it previously printed nothing.
|
||||||
|
|||||||
+21
-12
@@ -223,29 +223,38 @@ REPL commands:
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
hits := map[uint64]int{}
|
|
||||||
traps := 0
|
|
||||||
fmt.Printf("gasm debug: coverage run over %d instructions\n", len(instrs))
|
fmt.Printf("gasm debug: coverage run over %d instructions\n", len(instrs))
|
||||||
for {
|
for {
|
||||||
for _, bp := range bm.All() {
|
for _, bp := range bm.All() {
|
||||||
bm.Reinsert(bp.Addr)
|
bm.Reinsert(bp.Addr)
|
||||||
}
|
}
|
||||||
// Remove the breakpoint the run is parked on, count the hit,
|
|
||||||
// then continue.
|
|
||||||
regs, rerr := sess.GetRegs()
|
|
||||||
if rerr == nil {
|
|
||||||
if bp := bm.At(regs.GetPC()); bp != nil {
|
|
||||||
bm.Clear(bp.Addr)
|
|
||||||
traps++
|
|
||||||
hits[regs.GetPC()-base]++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := sess.Continue(); err != nil {
|
if err := sess.Continue(); err != nil {
|
||||||
break // debuggee finished or died
|
break // debuggee finished or died
|
||||||
}
|
}
|
||||||
if sess.Exited() {
|
if sess.Exited() {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
regs, rerr := sess.GetRegs()
|
||||||
|
if rerr != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
// HandleTrap restores the original byte, rewinds PC and counts
|
||||||
|
// the hit on the breakpoint itself. Single-step over the
|
||||||
|
// restored instruction so the reinsertion at the top of the
|
||||||
|
// loop cannot re-trap on the same breakpoint.
|
||||||
|
if bp := bm.HandleTrap(®s); bp != nil {
|
||||||
|
if err := sess.Step(); err != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
hits := map[uint64]int{}
|
||||||
|
traps := 0
|
||||||
|
for _, bp := range bm.All() {
|
||||||
|
if n := bp.Hits(); n > 0 {
|
||||||
|
hits[bp.Addr-base] = n
|
||||||
|
traps += n
|
||||||
|
}
|
||||||
}
|
}
|
||||||
var hit []string
|
var hit []string
|
||||||
var missed []string
|
var missed []string
|
||||||
|
|||||||
@@ -188,6 +188,9 @@ func (bm *Breakpoints) All() []*Breakpoint {
|
|||||||
// whether the trap was caused by one of our breakpoints (PC-adjust matches
|
// whether the trap was caused by one of our breakpoints (PC-adjust matches
|
||||||
// a breakpoint address), restores the original byte, rewinds PC, and
|
// a breakpoint address), restores the original byte, rewinds PC, and
|
||||||
// returns the breakpoint that was hit (or nil if it was a single-step).
|
// returns the breakpoint that was hit (or nil if it was a single-step).
|
||||||
|
// Hits returns how many times the breakpoint has been hit.
|
||||||
|
func (bp *Breakpoint) Hits() int { return bp.hits }
|
||||||
|
|
||||||
func (bm *Breakpoints) HandleTrap(regs *Regs) *Breakpoint {
|
func (bm *Breakpoints) HandleTrap(regs *Regs) *Breakpoint {
|
||||||
// After a breakpoint trap, PC points past the breakpoint instruction.
|
// After a breakpoint trap, PC points past the breakpoint instruction.
|
||||||
trapAddr := regs.GetPC() - uint64(breakpointPCAdjust)
|
trapAddr := regs.GetPC() - uint64(breakpointPCAdjust)
|
||||||
|
|||||||
@@ -0,0 +1,126 @@
|
|||||||
|
// Copyright (c) 2026 Petr Balvín <opensource@petrbalvin.org> (https://petrbalvin.org)
|
||||||
|
// SPDX-License-Identifier: BSD-3-Clause
|
||||||
|
|
||||||
|
//go:build linux && amd64
|
||||||
|
|
||||||
|
package debug
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"sourcedock.dev/petrbalvin/gasm-devkit/verify"
|
||||||
|
)
|
||||||
|
|
||||||
|
// buildGasm produces the gasm binary the debugger spawns as its debuggee.
|
||||||
|
func buildGasm(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
if p := os.Getenv("GASM_TEST_BIN"); p != "" {
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
bin := filepath.Join(t.TempDir(), "gasm")
|
||||||
|
cmd := exec.Command("go", "build", "-o", bin, "sourcedock.dev/petrbalvin/gasm-devkit/cmd/gasm")
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("build gasm: %v: %s", err, out)
|
||||||
|
}
|
||||||
|
return bin
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLaunchAndBreakpoint drives a real ptrace session end to end: launch the
|
||||||
|
// debuggee, break on the first instruction of the function and expect a
|
||||||
|
// breakpoint trap instead of a clean exit.
|
||||||
|
func TestLaunchAndBreakpoint(t *testing.T) {
|
||||||
|
if runtime.GOARCH != "amd64" {
|
||||||
|
t.Skip("runs only on amd64 hosts")
|
||||||
|
}
|
||||||
|
// The tracer is the OS thread that forked the debuggee (PTRACE_TRACEME
|
||||||
|
// binds the relation to that thread); every ptrace request must come
|
||||||
|
// from the same thread, so pin the test goroutine to one thread.
|
||||||
|
runtime.LockOSThread()
|
||||||
|
defer runtime.UnlockOSThread()
|
||||||
|
bin := buildGasm(t)
|
||||||
|
|
||||||
|
const kernelPath = "../testdata/verify/basic_amd64.s"
|
||||||
|
k, err := verify.Load(kernelPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(k.Close)
|
||||||
|
fl, err := k.Func("wideCopy")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Func: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sess, err := Launch(bin, kernelPath, "wideCopy", make([]byte, fl.Args))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Launch: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(sess.Kill)
|
||||||
|
|
||||||
|
bm := NewBreakpoints(sess)
|
||||||
|
entry := sess.CodeBase() + uint64(fl.Offset)
|
||||||
|
if _, err := bm.Set(entry, "entry"); err != nil {
|
||||||
|
t.Fatalf("Set: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The INT3 must be visible in the debuggee's memory.
|
||||||
|
word, err := sess.Peek(entry)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Peek: %v", err)
|
||||||
|
}
|
||||||
|
if b := word & 0xFF; b != 0xCC {
|
||||||
|
t.Fatalf("int3 not patched: first byte %#02x at %#x", b, entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The debuggee raises a second SIGSTOP after the launch barrier (the
|
||||||
|
// child's RunTarget marks its entry), so like the REPL and the cover
|
||||||
|
// mode the test keeps resuming until the breakpoint trap arrives.
|
||||||
|
for range 10 {
|
||||||
|
if err := sess.Continue(); err != nil {
|
||||||
|
st, _ := os.ReadFile(fmt.Sprintf("/proc/%d/stat", sess.Pid()))
|
||||||
|
status, _ := os.ReadFile(fmt.Sprintf("/proc/%d/status", sess.Pid()))
|
||||||
|
t.Fatalf("Continue: %v\nstate: %s\n%s", err, fieldName(st), statusDump(status))
|
||||||
|
}
|
||||||
|
if sess.Exited() {
|
||||||
|
t.Fatal("debuggee exited instead of trapping on the breakpoint")
|
||||||
|
}
|
||||||
|
regs, err := sess.GetRegs()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetRegs: %v", err)
|
||||||
|
}
|
||||||
|
if bp := bm.HandleTrap(®s); bp != nil {
|
||||||
|
if bp.Addr != entry {
|
||||||
|
t.Fatalf("trap at %#x, want %#x", bp.Addr, entry)
|
||||||
|
}
|
||||||
|
return // trap on the entry breakpoint: the whole flow works
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("no breakpoint trap after 10 resumes")
|
||||||
|
}
|
||||||
|
|
||||||
|
func fieldName(stat []byte) string {
|
||||||
|
f := strings.Split(string(stat), " ")
|
||||||
|
if len(f) > 2 {
|
||||||
|
return "state=" + f[2]
|
||||||
|
}
|
||||||
|
return "no stat"
|
||||||
|
}
|
||||||
|
|
||||||
|
func statusDump(b []byte) string {
|
||||||
|
var out []string
|
||||||
|
for _, l := range strings.Split(string(b), "\n") {
|
||||||
|
if strings.HasPrefix(l, "State") || strings.HasPrefix(l, "Pid") ||
|
||||||
|
strings.HasPrefix(l, "PPid") || strings.HasPrefix(l, "TracerPid") ||
|
||||||
|
strings.HasPrefix(l, "Threads") || strings.HasPrefix(l, "SigPnd") ||
|
||||||
|
strings.HasPrefix(l, "SigBlk") || strings.HasPrefix(l, "SigIgn") {
|
||||||
|
out = append(out, l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(out, "\n")
|
||||||
|
}
|
||||||
+66
-26
@@ -10,6 +10,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
@@ -32,7 +33,14 @@ func Launch(gasmBin, asmPath, funcName string, args []byte) (*Session, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// LaunchWithBuffers is like Launch but also allocates buffers in the debuggee.
|
// LaunchWithBuffers is like Launch but also allocates buffers in the debuggee.
|
||||||
|
//
|
||||||
|
// It pins the calling goroutine to its OS thread and leaves it pinned: the
|
||||||
|
// debuggee's PTRACE_TRACEME binds the tracer relation to the forking thread,
|
||||||
|
// and every ptrace request on the session must come from that same thread.
|
||||||
|
// All Session methods must therefore be called from the goroutine that
|
||||||
|
// launched the session (the REPL and coverage loops do exactly that).
|
||||||
func LaunchWithBuffers(gasmBin, asmPath, funcName string, args []byte, bufSpec string) (*Session, []uint64, error) {
|
func LaunchWithBuffers(gasmBin, asmPath, funcName string, args []byte, bufSpec string) (*Session, []uint64, error) {
|
||||||
|
runtime.LockOSThread() // ptrace requests must stay on the forking thread
|
||||||
self, err := os.Executable()
|
self, err := os.Executable()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, fmt.Errorf("debug: cannot find gasm binary: %w", err)
|
return nil, nil, fmt.Errorf("debug: cannot find gasm binary: %w", err)
|
||||||
@@ -78,37 +86,26 @@ func LaunchWithBuffers(gasmBin, asmPath, funcName string, args []byte, bufSpec s
|
|||||||
}
|
}
|
||||||
time.Sleep(5 * time.Millisecond)
|
time.Sleep(5 * time.Millisecond)
|
||||||
}
|
}
|
||||||
var ws syscall.WaitStatus
|
|
||||||
if _, err := syscall.Wait4(s.pid, &ws, syscall.WUNTRACED, nil); err != nil {
|
// The debuggee parks itself with SIGSTOP once the JIT code is mapped.
|
||||||
|
// A Go tracee also reports SIGURG preemption as signal-delivery-stops,
|
||||||
|
// so the wait loops until a stop the debugger cares about instead of
|
||||||
|
// assuming the first event is the SIGSTOP.
|
||||||
|
if _, err := s.waitStopped(); err != nil {
|
||||||
cmd.Process.Kill()
|
cmd.Process.Kill()
|
||||||
os.RemoveAll(tmpDir)
|
os.RemoveAll(tmpDir)
|
||||||
return nil, nil, fmt.Errorf("debug: wait for stop: %w", err)
|
return nil, nil, fmt.Errorf("debug: wait for debuggee: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
entryFile := filepath.Join(tmpDir, "entry")
|
|
||||||
for range 500 {
|
|
||||||
if _, err := os.Stat(entryFile); err == nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
time.Sleep(5 * time.Millisecond)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := s.Continue(); err != nil {
|
|
||||||
return nil, nil, fmt.Errorf("debug: continue to entry: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := syscall.Wait4(s.pid, &ws, syscall.WUNTRACED, nil); err != nil {
|
|
||||||
return nil, nil, fmt.Errorf("debug: wait for entry: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
s.stopped = true
|
s.stopped = true
|
||||||
|
|
||||||
s.codeBase = findRWXMapping(s.pid)
|
// The debuggee reports its JIT mapping in the codebase file; that is the
|
||||||
if s.codeBase == 0 {
|
// exact region the kernel was written to. Scanning /proc/pid/maps for
|
||||||
baseFile := filepath.Join(tmpDir, "codebase")
|
// any RWX region is only the fallback.
|
||||||
if data, err := os.ReadFile(baseFile); err == nil {
|
if data, err := os.ReadFile(filepath.Join(tmpDir, "codebase")); err == nil {
|
||||||
fmt.Sscanf(string(data), "%d", &s.codeBase)
|
fmt.Sscanf(string(data), "%d", &s.codeBase)
|
||||||
}
|
}
|
||||||
|
if s.codeBase == 0 {
|
||||||
|
s.codeBase = findRWXMapping(s.pid)
|
||||||
}
|
}
|
||||||
|
|
||||||
var bufAddrs []uint64
|
var bufAddrs []uint64
|
||||||
@@ -142,6 +139,47 @@ func (s *Session) wait() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// waitStopped consumes ptrace-stop events until one the debugger cares
|
||||||
|
// about arrives: SIGTRAP (a breakpoint or a completed single-step) or the
|
||||||
|
// debuggee's own SIGSTOP. A Go tracee's runtime raises SIGURG for
|
||||||
|
// asynchronous preemption, and every signal on a traced thread surfaces as
|
||||||
|
// a signal-delivery-stop, so those are suppressed and the tracee resumed
|
||||||
|
// without them. Runtime noise is why a single wait can return in the
|
||||||
|
// middle of runtime code and a resume can then fail: the event stream must
|
||||||
|
// be drained by the tracer.
|
||||||
|
func (s *Session) waitStopped() (syscall.Signal, error) {
|
||||||
|
for {
|
||||||
|
var ws syscall.WaitStatus
|
||||||
|
if _, err := syscall.Wait4(s.pid, &ws, syscall.WUNTRACED, nil); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if ws.Exited() {
|
||||||
|
s.exited = true
|
||||||
|
return 0, fmt.Errorf("debuggee exited with status %d", ws.ExitStatus())
|
||||||
|
}
|
||||||
|
if ws.Signaled() {
|
||||||
|
s.exited = true
|
||||||
|
return 0, fmt.Errorf("debuggee killed by signal %v", ws.Signal())
|
||||||
|
}
|
||||||
|
switch sig := ws.StopSignal(); sig {
|
||||||
|
case syscall.SIGTRAP, syscall.SIGSTOP:
|
||||||
|
s.stopped = true
|
||||||
|
return sig, nil
|
||||||
|
default:
|
||||||
|
// Runtime noise (SIGURG preemption and friends): resume the
|
||||||
|
// tracee without delivering the signal.
|
||||||
|
if _, _, errno := syscall.Syscall6(
|
||||||
|
syscall.SYS_PTRACE,
|
||||||
|
uintptr(syscall.PTRACE_CONT),
|
||||||
|
uintptr(s.pid),
|
||||||
|
0, 0, 0, 0,
|
||||||
|
); errno != 0 {
|
||||||
|
return 0, fmt.Errorf("debug: PTRACE_CONT: %w", errno)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Peek reads a word (8 bytes) from the debuggee's memory at addr.
|
// Peek reads a word (8 bytes) from the debuggee's memory at addr.
|
||||||
func (s *Session) Peek(addr uint64) (uint64, error) {
|
func (s *Session) Peek(addr uint64) (uint64, error) {
|
||||||
mem, err := os.OpenFile(fmt.Sprintf("/proc/%d/mem", s.pid), os.O_RDONLY, 0)
|
mem, err := os.OpenFile(fmt.Sprintf("/proc/%d/mem", s.pid), os.O_RDONLY, 0)
|
||||||
@@ -224,7 +262,8 @@ func (s *Session) Step() error {
|
|||||||
if errno != 0 {
|
if errno != 0 {
|
||||||
return fmt.Errorf("debug: PTRACE_SINGLESTEP: %w", errno)
|
return fmt.Errorf("debug: PTRACE_SINGLESTEP: %w", errno)
|
||||||
}
|
}
|
||||||
return s.wait()
|
_, err := s.waitStopped()
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Continue resumes execution until the next breakpoint or exit.
|
// Continue resumes execution until the next breakpoint or exit.
|
||||||
@@ -241,7 +280,8 @@ func (s *Session) Continue() error {
|
|||||||
if errno != 0 {
|
if errno != 0 {
|
||||||
return fmt.Errorf("debug: PTRACE_CONT: %w", errno)
|
return fmt.Errorf("debug: PTRACE_CONT: %w", errno)
|
||||||
}
|
}
|
||||||
return s.wait()
|
_, err := s.waitStopped()
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Exited returns true if the debuggee has terminated.
|
// Exited returns true if the debuggee has terminated.
|
||||||
|
|||||||
@@ -178,6 +178,13 @@ func REPL(s *Session, bm *Breakpoints, codeBase uint64, funcOffset, funcSize, ar
|
|||||||
}
|
}
|
||||||
regs, _ := s.GetRegs()
|
regs, _ := s.GetRegs()
|
||||||
if bp := bm.HandleTrap(®s); bp != nil {
|
if bp := bm.HandleTrap(®s); bp != nil {
|
||||||
|
// Execute the instruction under the restored breakpoint
|
||||||
|
// so the next continue cannot re-trap on the same
|
||||||
|
// breakpoint; the process parks right after it.
|
||||||
|
if err := s.Step(); err != nil {
|
||||||
|
fmt.Println(err)
|
||||||
|
break
|
||||||
|
}
|
||||||
name := bp.Label
|
name := bp.Label
|
||||||
if name == "" {
|
if name == "" {
|
||||||
name = fmt.Sprintf("%#x", bp.Addr)
|
name = fmt.Sprintf("%#x", bp.Addr)
|
||||||
|
|||||||
Reference in New Issue
Block a user