#!/usr/bin/perl
# Rig for sglang-deploy.pl: runs the real script as root inside a container against
# stub commands, and checks the state, the written files and the exit codes it leaves.
use strict;
use warnings;

# The rig file lives beside this repository's other checks, everything it writes lives
# in a work directory that is bind mounted into the container, and the script under test
# is the one in the repository root.
my $RIG    = $0 =~ m{^(.*)/[^/]+$} ? $1 : '.';
my $WORK   = $ENV{SGLANG_RIG_WORK} // '/tmp/sglang-rig';
my $BIN    = "$WORK/bin";
my $FIX    = "$WORK/fixture";
my $ST     = "$WORK/state";
my $LOG    = "$WORK/log/stubs.log";
my $SCRIPT = $ENV{SGLANG_RIG_SCRIPT} // "$RIG/../../sglang-deploy.pl";
my $UNIT   = '/etc/systemd/system/sglang.service';
my $ENVFILE = '/etc/sysconfig/sglang';
my $NGINX  = '/etc/nginx/conf.d/sglang.conf';
my $CERTDIR = '/etc/ssl/sglang';
my $STATEDIR = '/opt/sglang';

my ($passed, $failed) = (0, 0);
my @failures;

sub check {
    my ($cond, $label) = @_;
    if (!defined $label || $label eq '') {
        my @stack;
        for my $i (0 .. 3) {
            my @c = caller($i);
            push @stack, join(':', $c[1] // '?', $c[2] // '?');
        }
        $label = 'no label, stack ' . join(' <- ', @stack);
    }
    if ($cond) { $passed++; print "ok   $label\n"; return 1 }
    $failed++;
    push @failures, $label;
    print "FAIL $label\n";
    return 0;
}

# The verdict is forced into a boolean: a bare match in a sub argument list is a list
# context, where a failed match yields the empty list and would shift the label into
# the condition slot, turning a failure into a silent pass.
sub check_like {
    my ($text, $re, $label) = @_;
    $label = 'check_like without a label' unless defined $label;
    my $matched = (defined $text && $text =~ $re) ? 1 : 0;
    return check($matched, $label);
}

sub check_unlike {
    my ($text, $re, $label) = @_;
    $label = 'check_unlike without a label' unless defined $label;
    my $matched = (defined $text && $text !~ $re) ? 1 : 0;
    return check($matched, $label);
}

# Directories are made and removed with the language's own calls, so no module has to
# be installed on the machine running the rig.
sub make_dirs {
    for my $path (@_) {
        next if -d $path;
        my @parts = split m{/}, $path;
        my $built = '';
        for my $part (@parts) {
            next unless length $part;
            $built .= "/$part";
            mkdir($built, 0755) unless -d $built;
        }
    }
    return;
}

sub remove_tree {
    my ($path) = @_;
    return unless -d $path;
    if (opendir(my $dh, $path)) {
        for my $entry (readdir($dh)) {
            next if $entry eq '.' || $entry eq '..';
            my $child = "$path/$entry";
            remove_tree($child) if -d $child;
            unlink($child);
        }
        closedir($dh);
    }
    rmdir($path);
    return;
}

sub slurp_file {
    my ($path) = @_;
    return '' unless -f $path;
    open(my $fh, '<', $path) or return '';
    local $/ = undef;
    my $text = <$fh>;
    close($fh);
    return defined $text ? $text : '';
}

sub mode_of {
    my ($path) = @_;
    return sprintf('%04o', (stat($path))[2] & 07777);
}

sub reset_fixture {
    for my $path ($UNIT, $ENVFILE, $NGINX) {
        unlink($path);
    }
    remove_tree($CERTDIR);
    remove_tree($STATEDIR);
    remove_tree($FIX);
    remove_tree($ST);
    # Directories a host that ran server-setup.pl has: nginx's configuration drop-in
    # and systemd's unit directory.
    make_dirs($FIX, $ST, "$WORK/log", '/etc/nginx/conf.d', '/etc/systemd/system');
    my $fh;
    open($fh, q{>}, $LOG) and close($fh);
    # Packages a real host or a previous run has already installed.
    write_fixture('rpm-installed', "nginx\n");
    write_fixture('fw-services', "\n");
    # firewalld is running: server-setup.pl ensures it, and the firewall step needs it.
    my $fw;
    open($fw, '>', "$ST/active.firewalld") and close($fw);
    return;
}

sub write_fixture {
    my ($name, $content) = @_;
    open(my $fh, '>', "$FIX/$name") or die "cannot write $FIX/$name: $!\n";
    print {$fh} $content;
    close($fh);
    return;
}

sub fixture_exists {
    my ($name) = @_;
    return -f "$FIX/$name" ? 1 : 0;
}

# Run the script with the stub PATH and return (rc, merged output).
sub run_script {
    my (@args) = @_;
    my $out_file = "$WORK/log/run.out";
    my $pid = fork();
    die "cannot fork: $!\n" unless defined $pid;
    if ($pid == 0) {
        open(STDOUT, '>', $out_file);
        open(STDERR, '>&', \*STDOUT);
        $ENV{PATH} = "$BIN:/usr/bin:/bin";
        # The stubs read the work directory from the environment, since they are
        # reached through links and cannot tell where the rig file lives.
        $ENV{STUB_WORK} = $WORK;
        exec '/usr/bin/perl', $SCRIPT, @args;
        exit 126;
    }
    waitpid($pid, 0);
    my $rc = $? >> 8;
    return ($rc, slurp_file($out_file));
}

sub stub_log {
    return slurp_file($LOG);
}

sub count_in_log {
    my ($pattern) = @_;
    my @lines = grep { /$pattern/ } split /\n/, stub_log();
    return scalar @lines;
}

sub reset_log {
    my $fh;
    open($fh, q{>}, $LOG) and close($fh);
    return;
}

# ---------------------------------------------------------------------------
# Scenarios
# ---------------------------------------------------------------------------

# The stubs are one dispatcher file reached through links named after each command,
# so the script's own PATH lookup finds them and nothing of the real system is used.
sub prepare_stubs {
    make_dirs($BIN, $FIX, $ST, "$WORK/log");
    for my $name (qw(lspci rpm dnf podman systemctl curl openssl nginx
        firewall-cmd getenforce getsebool setsebool)) {
        my $link = "$BIN/$name";
        # A link left over from a work directory that moved reads as broken to
        # -e, and its stale target would leave the stubs unreachable: it is
        # replaced rather than kept or died on.
        if (-l $link && !-e $link) { unlink($link) }
        next if -e $link;
        symlink("$RIG/stub.pl", $link) or die "cannot link $link: $!\n";
    }
    return;
}

sub prepare_devices {
    # The driver creates these on a real host; the rig fakes them (needs --privileged).
    return if -e q{/dev/kfd};
    system(qw(mknod /dev/kfd c 237 0));
    mkdir(q{/dev/dri}, 0755) unless -d q{/dev/dri};
    return;
}

sub scenario_fresh {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1] (rev 01)\n");
    write_fixture('rocm.txt', "7.2.4\n");
    my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');

    check($rc == 0, 'fresh: exit 0');
    check(-f $UNIT, 'fresh: unit written');
    check(-f $ENVFILE, 'fresh: environment file written');
    check(-f $NGINX, 'fresh: nginx configuration written');
    check(-f "$CERTDIR/sglang.crt" && -f "$CERTDIR/sglang.key", 'fresh: certificate written');
    check(-d "$STATEDIR/modelscope", 'fresh: model cache directory created');
    check(mode_of($ENVFILE) eq '0600', 'fresh: environment file is 0600 (' . mode_of($ENVFILE) . ')');
    check(mode_of("$CERTDIR/sglang.key") eq '0600', 'fresh: key is 0600');
    check(mode_of("$CERTDIR/sglang.crt") eq '0644', 'fresh: certificate is 0644');

    my $env = slurp_file($ENVFILE);
    check_like($env, qr/^SGLANG_API_KEY=([A-Za-z0-9_-]{43})\n$/, 'fresh: generated key, url-safe, 43 chars');

    my $unit = slurp_file($UNIT);
    check_like($unit, qr|docker\.io/lmsysorg/sglang:v0\.5\.19-rocm724-mi30x|, 'fresh: image resolved from ROCm 7.2.4 and MI300');
    check_like($unit, qr/--tp-size 1/, 'fresh: tensor parallel default');
    check_like($unit, qr/--context-length 4096/, 'fresh: context length default');
    check_like($unit, qr/--mem-fraction-static 0\.9/, 'fresh: memory fraction default');
    check_unlike($unit, qr/SGLANG_USE_AITER/, 'fresh: no Radeon variables on an Instinct host');

    my $nginx = slurp_file($NGINX);
    check_like($nginx, qr|proxy_pass http://\[::1\]:8000;|, 'fresh: nginx proxies to the loopback engine');
    check_like($nginx, qr|ssl_certificate /etc/ssl/sglang/sglang\.crt;|, 'fresh: nginx uses the sglang certificate');

    check(count_in_log(qr/^podman pull /) == 1, 'fresh: exactly one image pull');
    check_like(stub_log(), qr/^podman pull docker\.io\/lmsysorg\/sglang:v0\.5\.19-rocm724-mi30x$/m,
        'fresh: the pulled image is the resolved tag');
    check(count_in_log(qr/^systemctl enable sglang$/) == 1, 'fresh: service enabled');
    check(count_in_log(qr/^systemctl start sglang$/) == 1, 'fresh: service started');
    check_like(stub_log(), qr/^firewall-cmd --permanent --add-service=https$/m, 'fresh: HTTPS opened');
    check_like($out, qr/Engine image: docker\.io\/lmsysorg\/sglang:v0\.5\.19-rocm724-mi30x/,
        'fresh: summary names the image');
    check_like($out, qr/systemd: sglang running on \[::1\]:8000/, 'fresh: summary names the endpoint');
    check_like($out, qr/API key \(shown once, store it securely\)/, 'fresh: the generated key is shown once');
    check_unlike($out, qr/✗/, 'fresh: no failed step');
    return;
}

sub scenario_rerun {
    scenario_fresh();
    reset_log();
    my $key_before = slurp_file($ENVFILE);
    my $unit_before = slurp_file($UNIT);
    my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');

    check($rc == 0, 'rerun: exit 0');
    check(count_in_log(qr/^podman pull /) == 0, 'rerun: no second pull');
    check_like(stub_log(), qr/^podman image exists /m, 'rerun: the image is checked instead');
    check(count_in_log(qr/^systemctl enable sglang$/) == 0, 'rerun: no second enable');
    check(count_in_log(qr/^systemctl start sglang$/) == 0, 'rerun: no second start');
    check(count_in_log(qr/try-restart/) == 0, 'rerun: no restart, the unit did not change');
    check(count_in_log(qr/^dnf install /) == 0, 'rerun: no second package transaction');
    check(slurp_file($ENVFILE) eq $key_before, 'rerun: the API key is reused, not regenerated');
    check(slurp_file($UNIT) eq $unit_before, 'rerun: the unit file is byte-identical');
    check_like($out, qr/API key: existing key reused/, 'rerun: the key is reported as reused');
    check_like($out, qr/Engine image: already present/, 'rerun: the image is reported present');
    check_like($out, qr/systemd: sglang running/, 'rerun: the service is reported running');
    check_unlike($out, qr/would /, 'rerun: nothing is phrased as would');
    check_unlike($out, qr/✗/, 'rerun: no failed step');
    return;
}

sub scenario_dry_run {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1] (rev 01)\n");
    write_fixture('rocm.txt', "7.2.4\n");
    my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--dry-run');

    check($rc == 0, 'dry run: exit 0');
    check(!-f $UNIT, 'dry run: no unit written');
    check(!-f $ENVFILE, 'dry run: no environment file written');
    check(!-f $NGINX, 'dry run: no nginx configuration written');
    check(!-d $CERTDIR, 'dry run: no certificate directory');
    check(count_in_log(qr/^podman pull /) == 0, 'dry run: no pull');
    check(count_in_log(qr/^systemctl (enable|start) /) == 0, 'dry run: no service change');
    check(count_in_log(qr/^dnf install /) == 0, 'dry run: no package transaction');
    check_like($out, qr/DRY RUN: no changes will be made/, 'dry run: banner');
    check_like($out, qr/would be pulled|would be installed/, 'dry run: phrased as would');
    check_like($out, qr|Engine image: docker\.io/lmsysorg/sglang:v0\.5\.19-rocm724-mi30x|,
        'dry run: the resolved image is reported');
    return;
}

sub scenario_uninstall {
    scenario_fresh();
    reset_log();
    my ($rc, $out) = run_script('--uninstall');

    check($rc == 0, 'uninstall: exit 0');
    check(!-f $UNIT, 'uninstall: unit removed');
    check(!-f $ENVFILE, 'uninstall: environment file removed');
    check(!-f $NGINX, 'uninstall: nginx configuration removed');
    check(!-d $CERTDIR, 'uninstall: certificate directory removed');
    check(-d $STATEDIR, 'uninstall: model cache kept');
    check(-e "$ST/image.docker.io_lmsysorg_sglang_v0.5.19-rocm724-mi30x",
        'uninstall: the image is kept in podman');
    check_like(stub_log(), qr/^systemctl stop sglang$/m, 'uninstall: service stopped');
    check_like(stub_log(), qr/^systemctl disable sglang$/m, 'uninstall: service disabled');
    check_like($out, qr/SGLang service stopped/, 'uninstall: summary reports the stop');
    check_like($out, qr/Kept on the system/, 'uninstall: the kept state is listed');
    check_like($out, qr/ModelScope cache/, 'uninstall: the cache is named as kept');
    return;
}

sub scenario_uninstall_twice {
    scenario_uninstall();
    reset_log();
    my ($rc, $out) = run_script('--uninstall');
    check($rc == 0, 'uninstall twice: exit 0');
    check_like($out, qr/already absent/, 'uninstall twice: idempotent');
    check(count_in_log(qr/^systemctl stop /) == 0, 'uninstall twice: nothing to stop');
    return;
}

sub scenario_radeon {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Navi 31 [Radeon RX 7900 XTX] [1002:744c] (rev c8)\n");
    write_fixture('rocm.txt', "7.2.4\n");
    my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');

    check($rc == 1, 'radeon: exit 1');
    check_like($out, qr/Radeon card detected/, 'radeon: names the cause');
    check_like($out, qr/rocm\.Dockerfile/, 'radeon: names the build recipe');
    check_like($out, qr/--image sglang-rocm:latest/, 'radeon: names the way forward');
    check(!-f $UNIT, 'radeon: nothing deployed');
    return;
}

sub scenario_radeon_dev {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Strix Halo [Radeon Graphics / Radeon 8050S Graphics / Radeon 8060S Graphics] [1002:1586] (rev d1)\n");
    write_fixture('rocm.txt', "7.2.4\n");
    my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');

    check($rc == 0, 'radeon dev: exit 0');
    check_like(stub_log(), qr{^curl .*rocm/sgl-dev/tags\?.*$}m,
        'radeon dev: the AMD tag index is asked for the newest build');
    my $unit = slurp_file($UNIT);
    check_like($unit, qr|docker\.io/rocm/sgl-dev:v0\.5\.19-rocm724-gfx1151-20260916|,
        'radeon dev: the newest dated build is deployed');
    check_like($unit, qr/Environment=SGLANG_USE_AITER=false/, 'radeon dev: AITER off');
    check_like($unit, qr/Environment=SGLANG_ROCM_FUSED_DECODE_MLA=false/,
        'radeon dev: fused decode MLA off');
    check_like($out, qr/AMD publishes this build daily/, 'radeon dev: the moving tag is stated');
    check_like($out, qr/the only flavour published for gfx1151/, 'radeon dev: the flavour is explained');

    reset_log();
    my ($rc2, $out2) = run_script('--model', 'ZhipuAI/GLM-5.3', '--image', 'localhost/pinned:1');
    check_like(slurp_file($UNIT), qr|localhost/pinned:1|, 'radeon dev: --image pins a build');
    check(count_in_log(qr{^curl .*rocm/sgl-dev/tags\?.*$}) == 0,
        'radeon dev: no tag resolution when --image is given');
    return;
}

sub scenario_radeon_with_image {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Strix Halo [Radeon 8060S] [1002:150e]\n");
    my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--image', 'localhost/sglang-rocm:gfx1151');

    check($rc == 0, 'radeon with image: exit 0');
    my $unit = slurp_file($UNIT);
    check_like($unit, qr/Environment=SGLANG_USE_AITER=false/, 'radeon with image: AITER off');
    check_like($unit, qr/Environment=SGLANG_ROCM_FUSED_DECODE_MLA=false/,
        'radeon with image: fused decode MLA off');
    check_like($unit, qr|localhost/sglang-rocm:gfx1151|, 'radeon with image: the requested image');
    my ($rc2, $out2) = run_script('--model', 'ZhipuAI/GLM-5.3', '--image', 'localhost/sglang-rocm:gfx1151');
    check($rc2 == 0, 'radeon with image: rerun exit 0');
    check(slurp_file($UNIT) eq $unit, 'radeon with image: the unit is unchanged on a rerun');
    return;
}

sub scenario_rocm_flavours {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    write_fixture('rocm.txt', "7.2.3\n");
    my ($rc, $out) = run_script('--dry-run', '--model', 'ZhipuAI/GLM-5.3');
    check_like($out, qr/rocm720-mi30x/, 'flavour: ROCm 7.2.3 takes rocm720');
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    write_fixture('rocm.txt', "10.0.0\n");
    ($rc, $out) = run_script('--dry-run', '--model', 'ZhipuAI/GLM-5.3');
    check_like($out, qr/rocm10-mi30x/, 'flavour: ROCm 10.0.0 takes rocm10');
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI350X [1002:75a0]\n");
    write_fixture('rocm.txt', "7.2.4\n");
    ($rc, $out) = run_script('--dry-run', '--model', 'ZhipuAI/GLM-5.3');
    check_like($out, qr/rocm724-mi35x/, 'flavour: MI350X takes mi35x');
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    write_fixture('rocm.txt', "6.4.0\n");
    ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
    check($rc == 1, 'flavour: an old host ROCm is refused');
    check_like($out, qr/No published image targets ROCm 6\.4\.0/, 'flavour: the refusal names the version');
    check_like($out, qr/--image/, 'flavour: the refusal names the way forward');
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    ($rc, $out) = run_script('--dry-run', '--model', 'ZhipuAI/GLM-5.3');
    check_like($out, qr/rocm10-mi30x/, 'flavour: no host ROCm takes the newest');
    check_like($out, qr/No ROCm userland found on the host/, 'flavour: the assumption is stated out loud');
    return;
}

sub scenario_modelscope {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    write_fixture('rocm.txt', "7.2.4\n");
    write_fixture('ms.code', "200\n");
    write_fixture('ms.json', qq({"Data":{"Name":"GLM-5.3"}}\n));
    my ($rc, $out) = run_script('--dry-run', '--model', 'ZhipuAI/GLM-5.3');
    check_like($out, qr/Verifying GLM 5\.3 on ModelScope\.\.\. found/, 'modelscope: found');

    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    write_fixture('rocm.txt', "7.2.4\n");
    write_fixture('ms.code', "404\n");
    ($rc, $out) = run_script('--model', 'nope/does-not-exist');
    check($rc == 1, 'missing model: exit 1');
    check_like($out, qr/does not exist on ModelScope/, 'missing model: named as missing');

    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    write_fixture('rocm.txt', "7.2.4\n");
    # A 200 without a Name carries no usable answer: the deployment proceeds with
    # the probe marked unverified rather than blocked.
    write_fixture('ms.code', "200\n");
    write_fixture('ms.json', qq({}\n));
    ($rc, $out) = run_script('--dry-run', '--model', 'ZhipuAI/GLM-5.3');
    check($rc == 0, 'modelscope: an unnamed answer does not block');
    check_like($out, qr/could not verify/, 'modelscope: the unnamed answer reads as unverified');
    return;
}

sub scenario_hardware {
    reset_fixture();
    my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
    check($rc == 1, 'no GPU: exit 1');
    check_like($out, qr/No AMD GPU detected/, 'no GPU: named as missing');

    # The missing driver is checked outside the rig: podman bind-mounts /dev/kfd into a
    # privileged container, so it cannot be removed from inside one.
    return;
}

# Run without a GPU device node (a container without --privileged, or a host whose
# driver is not loaded): the script must refuse and say what is missing.
sub scenario_no_driver {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    if (-e '/dev/kfd') {
        print "skip no driver: this run has the device node (needs a run without --privileged)\n";
        return;
    }
    my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
    check($rc == 1, 'no driver: exit 1');
    check_like($out, qr/amdgpu kernel driver is not ready/, 'no driver: named as missing');
    check_like($out, qr{/dev/kfd}, 'no driver: names the device node');
    check(!-f $UNIT, 'no driver: nothing deployed');
    return;
}

sub scenario_bad_os {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    write_fixture('os-release', "ID=ubuntu\nVERSION_ID=\"24.04\"\n");
    my $real = slurp_file('/etc/os-release');
    open(my $fh, '>', '/etc/os-release') or die "cannot write /etc/os-release: $!\n";
    print {$fh} slurp_file("$FIX/os-release");
    close($fh);
    my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--dry-run');
    open(my $back, '>', '/etc/os-release') or die "cannot restore /etc/os-release: $!\n";
    print {$back} $real;
    close($back);

    check($rc == 1, 'bad OS: exit 1');
    check_like($out, qr/Unsupported operating system: 'ubuntu'/, 'bad OS: names the system');
    check_like($out, qr/Supported systems: Fedora, CentOS Stream, openEuler/, 'bad OS: lists the supported ones');
    return;
}

sub scenario_validation {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--port', '443', '--dry-run');
    check($rc == 1, 'validation: port 443 refused');
    check_like($out, qr/must be 1-65535 and not 443/, 'validation: port message');
    ($rc, $out) = run_script('--model', 'not-a-model-id', '--dry-run');
    check($rc == 1, 'validation: bad model ID refused');
    check_like($out, qr/Invalid model ID/, 'validation: model message');
    ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--gpu-memory-utilization', '1.5', '--dry-run');
    check($rc == 1, 'validation: memory fraction above 1 refused');
    check_like($out, qr/must be in \(0, 1\]/, 'validation: memory fraction message');
    ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--api-key', 'has space', '--dry-run');
    check($rc == 1, 'validation: whitespace API key refused');
    ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--rocm-flavour', 'rocm999', '--dry-run');
    check($rc == 1, 'validation: unknown flavour refused');
    check_like($out, qr/Invalid --rocm-flavour/, 'validation: flavour message');
    ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--image', 'bad image%name', '--dry-run');
    check($rc == 1, 'validation: image with whitespace or % refused');
    check_like($out, qr/Invalid --image/, 'validation: image message');
    return;
}

sub scenario_api_key_argument {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    write_fixture('rocm.txt', "7.2.4\n");
    my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--api-key', 'chosen-key-1234');
    check($rc == 0, 'api key argument: exit 0');
    check_like(slurp_file($ENVFILE), qr/^SGLANG_API_KEY=chosen-key-1234$/, 'api key argument: the key is stored');
    check_unlike($out, qr/chosen-key-1234/, 'api key argument: a given key is never echoed');
    my $unit = slurp_file($UNIT);
    check_unlike($unit, qr/chosen-key-1234/, 'api key argument: the key is not in the unit');
    return;
}

sub scenario_non_root {
    reset_fixture();
    my $out_file = "$WORK/log/run.out";
    my $pid = fork();
    die "cannot fork: $!\n" unless defined $pid;
    if ($pid == 0) {
        open(STDOUT, '>', $out_file);
        open(STDERR, '>&', \*STDOUT);
        $ENV{PATH} = "$BIN:/usr/bin:/bin";
        $ENV{STUB_WORK} = $WORK;
        $) = 1000;
        $> = 1000;
        exec '/usr/bin/perl', $SCRIPT, '--version';
        exit 126;
    }
    waitpid($pid, 0);
    my $rc = $? >> 8;
    my $out = slurp_file($out_file);
    check($rc == 0, 'non-root: --version works without root');
    check_like($out, qr/^sglang-deploy\.pl 2\.0\.0$/, 'non-root: the version is printed');

    my $pid3 = fork();
    die "cannot fork: $!\n" unless defined $pid3;
    if ($pid3 == 0) {
        open(STDOUT, '>', $out_file);
        open(STDERR, '>&', \*STDOUT);
        $ENV{PATH} = "$BIN:/usr/bin:/bin";
        $ENV{STUB_WORK} = $WORK;
        $) = 1000;
        $> = 1000;
        exec '/usr/bin/perl', $SCRIPT, '--model', 'ZhipuAI/GLM-5.3';
        exit 126;
    }
    waitpid($pid3, 0);
    my $rc3 = $? >> 8;
    my $out3 = slurp_file($out_file);
    check($rc3 == 1, 'non-root: exit 1');
    check_like($out3, qr/must be run as root/, 'non-root: the message says so');
    return;
}

sub scenario_offline {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    write_fixture('rocm.txt', "7.2.4\n");
    write_fixture('image-missing', "1\n");
    my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
    check($rc == 1, 'unpublished tag: exit 1');
    check_like($out, qr/resolved image tag does not exist/, 'unpublished tag: named as missing');
    check_like($out, qr/--image/, 'unpublished tag: names the way forward');
    return;
}

sub scenario_dependency_section {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    write_fixture('rocm.txt', "7.2.4\n");
    write_fixture('rpm-installed', "pciutils\ncurl\nopenssl\npodman\nnginx\n");
    reset_log();
    my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--dry-run');
    check($rc == 0, 'deps: exit 0');
    check(count_in_log(qr/^dnf install /) == 0, 'deps: no transaction when all packages are present');
    check_like($out, qr/All 4 packages already installed/, 'deps: reported as present');

    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    write_fixture('rocm.txt', "7.2.4\n");
    write_fixture('rpm-installed', "\n");
    reset_log();
    ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
    check_like(stub_log(), qr/^dnf install -y pciutils curl openssl podman$/m,
        'deps: the four packages are installed in one transaction');
    check_like($out, qr/nginx \(reverse proxy\) is not installed: run server-setup.pl first/,
        'deps: the missing reverse proxy is warned about');
    return;
}

sub scenario_custom_layout {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    write_fixture('rocm.txt', "7.2.4\n");
    my ($rc, $out) = run_script(
        '--model', 'ZhipuAI/GLM-5.3',
        '--service-name', 'llm',
        '--state-dir', '/srv/llm',
        '--cert-dir', '/etc/ssl/llm',
        '--port', '9000',
        '--tensor-parallel', '8',
        '--max-model-len', '131072',
        '--gpu-memory-utilization', '0.85',
    );
    check($rc == 0, 'custom layout: exit 0');
    check(-f '/etc/systemd/system/llm.service', 'custom layout: the named unit is written');
    check(-f '/etc/nginx/conf.d/llm.conf', 'custom layout: the named nginx file is written');
    # The certificate file names follow the program, as they did before, not the
    # service name; the directory follows --cert-dir.
    check(-f '/etc/ssl/llm/sglang.crt', 'custom layout: certificates follow the directory');
    check(-d '/srv/llm/modelscope', 'custom layout: the state directory follows');
    my $unit = slurp_file('/etc/systemd/system/llm.service');
    check_like($unit, qr/--port 9000/, 'custom layout: port in the unit');
    check_like($unit, qr/--tp-size 8/, 'custom layout: tensor parallel in the unit');
    check_like($unit, qr/--context-length 131072/, 'custom layout: context length in the unit');
    check_like($unit, qr/--mem-fraction-static 0\.85/, 'custom layout: memory fraction in the unit');
    check_like($unit, qr|--volume /srv/llm/modelscope:/root/\.cache/modelscope:Z|,
        'custom layout: the cache volume follows the state directory');
    unlink('/etc/systemd/system/llm.service');
    unlink('/etc/nginx/conf.d/llm.conf');
    remove_tree('/etc/ssl/llm');
    remove_tree('/srv/llm');
    return;
}

sub scenario_tls_existing {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    write_fixture('rocm.txt', "7.2.4\n");
    run_script('--model', 'ZhipuAI/GLM-5.3');
    my $crt = slurp_file("$CERTDIR/sglang.crt");
    reset_log();
    my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
    check(slurp_file("$CERTDIR/sglang.crt") eq $crt, 'tls: an existing certificate is kept');
    check(count_in_log(qr/^openssl /) == 0, 'tls: no second certificate');
    check_like($out, qr/TLS certificate: configured/, 'tls: reported as configured');
    return;
}

sub scenario_stale_container {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    write_fixture('rocm.txt', "7.2.4\n");
    my $fh;
    open($fh, q{>}, "$ST/container.sglang") and close($fh);
    my ($rc, $out) = run_script('--uninstall');
    check($rc == 0, 'stale container: uninstall exit 0');
    check_like(stub_log(), qr/^podman rm -f sglang$/m, 'stale container: removed');
    check(!-e "$ST/container.sglang", 'stale container: gone from podman');
    return;
}

sub scenario_selinux {
    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    write_fixture('rocm.txt', "7.2.4\n");
    $ENV{STUB_SELINUX} = 'Permissive';
    my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
    check(count_in_log(qr/^setsebool /) == 0, 'selinux: nothing set while permissive');
    check_like($out, qr/SELinux: permissive \(skipped\)/, 'selinux: reported as skipped');

    reset_fixture();
    write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
    write_fixture('rocm.txt', "7.2.4\n");
    $ENV{STUB_SELINUX} = 'Enforcing';
    reset_log();
    ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
    check_like(stub_log(), qr/^setsebool -P httpd_can_network_connect=1$/m, 'selinux: the boolean is set');
    check_like($out, qr/SELinux: httpd_can_network_connect on/, 'selinux: reported as on');
    reset_log();
    my ($rc2, $out2) = run_script('--model', 'ZhipuAI/GLM-5.3');
    check(count_in_log(qr/^setsebool /) == 0, 'selinux: nothing set when already on');
    delete $ENV{STUB_SELINUX};
    return;
}

my %scenarios = (
    fresh             => \&scenario_fresh,
    rerun             => \&scenario_rerun,
    dry_run           => \&scenario_dry_run,
    uninstall         => \&scenario_uninstall,
    uninstall_twice   => \&scenario_uninstall_twice,
    radeon            => \&scenario_radeon,
    radeon_dev        => \&scenario_radeon_dev,
    radeon_with_image => \&scenario_radeon_with_image,
    rocm_flavours     => \&scenario_rocm_flavours,
    modelscope => \&scenario_modelscope,
    hardware          => \&scenario_hardware,
    bad_os            => \&scenario_bad_os,
    no_driver         => \&scenario_no_driver,
    validation        => \&scenario_validation,
    api_key_argument  => \&scenario_api_key_argument,
    non_root          => \&scenario_non_root,
    offline           => \&scenario_offline,
    dependencies      => \&scenario_dependency_section,
    custom_layout     => \&scenario_custom_layout,
    tls_existing      => \&scenario_tls_existing,
    stale_container   => \&scenario_stale_container,
    selinux           => \&scenario_selinux,
);

prepare_stubs();
prepare_devices();

my @wanted = @ARGV ? @ARGV : sort keys %scenarios;
for my $name (@wanted) {
    if (!exists $scenarios{$name}) {
        print "FAIL unknown scenario $name\n";
        $failed++;
        next;
    }
    print "\n== $name ==\n";
    $scenarios{$name}->();
}

print "\n" . ($failed ? "$failed FAILED of " . ($passed + $failed) : "all " . $passed . " checks passed") . "\n";
exit($failed ? 1 : 0);