897 lines
42 KiB
Perl
897 lines
42 KiB
Perl
#!/usr/bin/perl
|
|
# Rig for sglang-deploy.pl: runs the real script as root inside a container against
|
|
# stub commands, and checks the state, the written files and the exit codes it leaves.
|
|
use strict;
|
|
use warnings;
|
|
|
|
# The rig file lives beside this repository's other checks, everything it writes lives
|
|
# in a work directory that is bind mounted into the container, and the script under test
|
|
# is the one in the repository root.
|
|
my $RIG = $0 =~ m{^(.*)/[^/]+$} ? $1 : '.';
|
|
my $WORK = $ENV{SGLANG_RIG_WORK} // '/tmp/sglang-rig';
|
|
my $BIN = "$WORK/bin";
|
|
my $FIX = "$WORK/fixture";
|
|
my $ST = "$WORK/state";
|
|
my $LOG = "$WORK/log/stubs.log";
|
|
my $SCRIPT = $ENV{SGLANG_RIG_SCRIPT} // "$RIG/../../sglang-deploy.pl";
|
|
my $UNIT = '/etc/systemd/system/sglang.service';
|
|
my $ENVFILE = '/etc/sysconfig/sglang';
|
|
my $CADDY = '/etc/caddy/Caddyfile.d/sglang.caddyfile';
|
|
my $CADDY_MAIN = '/etc/caddy/Caddyfile';
|
|
my $CADDY_UNIT = '/etc/systemd/system/caddy.service';
|
|
my $CADDY_BIN = '/usr/local/bin/caddy';
|
|
my $NGINX_LEGACY = '/etc/nginx/conf.d/sglang.conf';
|
|
my $CERTDIR = '/etc/ssl/sglang';
|
|
my $STATEDIR = '/opt/sglang';
|
|
|
|
my ($passed, $failed) = (0, 0);
|
|
my @failures;
|
|
|
|
sub check {
|
|
my ($cond, $label) = @_;
|
|
if (!defined $label || $label eq '') {
|
|
my @stack;
|
|
for my $i (0 .. 3) {
|
|
my @c = caller($i);
|
|
push @stack, join(':', $c[1] // '?', $c[2] // '?');
|
|
}
|
|
$label = 'no label, stack ' . join(' <- ', @stack);
|
|
}
|
|
if ($cond) { $passed++; print "ok $label\n"; return 1 }
|
|
$failed++;
|
|
push @failures, $label;
|
|
print "FAIL $label\n";
|
|
return 0;
|
|
}
|
|
|
|
# The verdict is forced into a boolean: a bare match in a sub argument list is a list
|
|
# context, where a failed match yields the empty list and would shift the label into
|
|
# the condition slot, turning a failure into a silent pass.
|
|
sub check_like {
|
|
my ($text, $re, $label) = @_;
|
|
$label = 'check_like without a label' unless defined $label;
|
|
my $matched = (defined $text && $text =~ $re) ? 1 : 0;
|
|
return check($matched, $label);
|
|
}
|
|
|
|
sub check_unlike {
|
|
my ($text, $re, $label) = @_;
|
|
$label = 'check_unlike without a label' unless defined $label;
|
|
my $matched = (defined $text && $text !~ $re) ? 1 : 0;
|
|
return check($matched, $label);
|
|
}
|
|
|
|
# Directories are made and removed with the language's own calls, so no module has to
|
|
# be installed on the machine running the rig.
|
|
sub make_dirs {
|
|
for my $path (@_) {
|
|
next if -d $path;
|
|
my @parts = split m{/}, $path;
|
|
my $built = '';
|
|
for my $part (@parts) {
|
|
next unless length $part;
|
|
$built .= "/$part";
|
|
mkdir($built, 0755) unless -d $built;
|
|
}
|
|
}
|
|
return;
|
|
}
|
|
|
|
sub remove_tree {
|
|
my ($path) = @_;
|
|
return unless -d $path;
|
|
if (opendir(my $dh, $path)) {
|
|
for my $entry (readdir($dh)) {
|
|
next if $entry eq '.' || $entry eq '..';
|
|
my $child = "$path/$entry";
|
|
remove_tree($child) if -d $child;
|
|
unlink($child);
|
|
}
|
|
closedir($dh);
|
|
}
|
|
rmdir($path);
|
|
return;
|
|
}
|
|
|
|
sub slurp_file {
|
|
my ($path) = @_;
|
|
return '' unless -f $path;
|
|
open(my $fh, '<', $path) or return '';
|
|
local $/ = undef;
|
|
my $text = <$fh>;
|
|
close($fh);
|
|
return defined $text ? $text : '';
|
|
}
|
|
|
|
sub mode_of {
|
|
my ($path) = @_;
|
|
return sprintf('%04o', (stat($path))[2] & 07777);
|
|
}
|
|
|
|
sub reset_fixture {
|
|
for my $path ($UNIT, $ENVFILE, $CADDY, $CADDY_MAIN, $CADDY_UNIT, $CADDY_BIN,
|
|
$NGINX_LEGACY) {
|
|
unlink($path);
|
|
}
|
|
remove_tree('/etc/caddy');
|
|
remove_tree($CERTDIR);
|
|
remove_tree($STATEDIR);
|
|
remove_tree($FIX);
|
|
remove_tree($ST);
|
|
# Directories a host that ran server-setup.pl has: the legacy nginx drop-in
|
|
# directory an earlier release wrote into, and systemd's unit directory.
|
|
make_dirs($FIX, $ST, "$WORK/log", '/etc/nginx/conf.d', '/etc/systemd/system');
|
|
my $fh;
|
|
open($fh, q{>}, $LOG) and close($fh);
|
|
# Packages a real host or a previous run has already installed. nginx stands
|
|
# for the drop-in the previous release left behind.
|
|
write_fixture('rpm-installed', "nginx\n");
|
|
write_fixture('fw-services', "\n");
|
|
# firewalld is running: server-setup.pl ensures it, and the firewall step needs it.
|
|
# The handle is declared first: a my inside the open's argument list does not
|
|
# reach the right-hand operand of the and on this interpreter.
|
|
my $fw;
|
|
open($fw, '>', "$ST/active.firewalld") and close($fw);
|
|
return;
|
|
}
|
|
|
|
sub write_fixture {
|
|
my ($name, $content) = @_;
|
|
open(my $fh, '>', "$FIX/$name") or die "cannot write $FIX/$name: $!\n";
|
|
print {$fh} $content;
|
|
close($fh);
|
|
return;
|
|
}
|
|
|
|
sub fixture_exists {
|
|
my ($name) = @_;
|
|
return -f "$FIX/$name" ? 1 : 0;
|
|
}
|
|
|
|
# Run the script with the stub PATH and return (rc, merged output).
|
|
sub run_script {
|
|
my (@args) = @_;
|
|
my $out_file = "$WORK/log/run.out";
|
|
my $pid = fork();
|
|
die "cannot fork: $!\n" unless defined $pid;
|
|
if ($pid == 0) {
|
|
open(STDOUT, '>', $out_file);
|
|
open(STDERR, '>&', \*STDOUT);
|
|
$ENV{PATH} = "$BIN:/usr/bin:/bin";
|
|
# The stubs read the work directory from the environment, since they are
|
|
# reached through links and cannot tell where the rig file lives.
|
|
$ENV{STUB_WORK} = $WORK;
|
|
exec '/usr/bin/perl', $SCRIPT, @args;
|
|
exit 126;
|
|
}
|
|
waitpid($pid, 0);
|
|
my $rc = $? >> 8;
|
|
return ($rc, slurp_file($out_file));
|
|
}
|
|
|
|
sub stub_log {
|
|
return slurp_file($LOG);
|
|
}
|
|
|
|
sub count_in_log {
|
|
my ($pattern) = @_;
|
|
my @lines = grep { /$pattern/ } split /\n/, stub_log();
|
|
return scalar @lines;
|
|
}
|
|
|
|
sub reset_log {
|
|
my $fh;
|
|
open($fh, q{>}, $LOG) and close($fh);
|
|
return;
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scenarios
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# The stubs are one dispatcher file reached through links named after each command,
|
|
# so the script's own PATH lookup finds them and nothing of the real system is used.
|
|
sub prepare_stubs {
|
|
make_dirs($BIN, $FIX, $ST, "$WORK/log");
|
|
for my $name (qw(lspci rpm dnf podman systemctl curl openssl caddy tar useradd
|
|
semodule firewall-cmd getenforce getsebool setsebool)) {
|
|
my $link = "$BIN/$name";
|
|
# A link left over from a work directory that moved reads as broken to
|
|
# -e, and its stale target would leave the stubs unreachable: it is
|
|
# replaced rather than kept or died on.
|
|
if (-l $link && !-e $link) { unlink($link) }
|
|
next if -e $link;
|
|
symlink("$RIG/stub.pl", $link) or die "cannot link $link: $!\n";
|
|
}
|
|
return;
|
|
}
|
|
|
|
# The caddy package creates its group; the rig creates it the same way, so the
|
|
# key permission the package makes possible is exercised for real. The group
|
|
# file is edited directly: the minimal openEuler image carries no groupadd to
|
|
# call, and a group entry is all the getgrnam in the script needs.
|
|
sub prepare_group {
|
|
return if defined getgrnam('caddy');
|
|
my $existing = slurp_file('/etc/group');
|
|
my $gid = 995;
|
|
$gid++ while $existing =~ /^[^:]+:[^:]*:\Q$gid\E:/m;
|
|
open(my $fh, '>>', '/etc/group') or die "cannot append to /etc/group: $!\n";
|
|
print {$fh} "caddy:x:$gid:\n";
|
|
close($fh);
|
|
return;
|
|
}
|
|
|
|
sub prepare_devices {
|
|
# The driver creates these on a real host; the rig fakes them (needs --privileged).
|
|
return if -e q{/dev/kfd};
|
|
system(qw(mknod /dev/kfd c 237 0));
|
|
mkdir(q{/dev/dri}, 0755) unless -d q{/dev/dri};
|
|
return;
|
|
}
|
|
|
|
sub scenario_fresh {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1] (rev 01)\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
|
|
check($rc == 0, 'fresh: exit 0');
|
|
check(-f $UNIT, 'fresh: unit written');
|
|
check(-f $ENVFILE, 'fresh: environment file written');
|
|
check(-f $CADDY, 'fresh: caddy drop-in written');
|
|
check(-f $CADDY_MAIN, 'fresh: main Caddyfile written');
|
|
check(-f "$CERTDIR/sglang.crt" && -f "$CERTDIR/sglang.key", 'fresh: certificate written');
|
|
check(-d "$STATEDIR/modelscope", 'fresh: model cache directory created');
|
|
check(mode_of($ENVFILE) eq '0600', 'fresh: environment file is 0600 (' . mode_of($ENVFILE) . ')');
|
|
check(mode_of("$CERTDIR/sglang.key") eq '0640', 'fresh: key is 0640 for the caddy group (' . mode_of("$CERTDIR/sglang.key") . ')');
|
|
check(mode_of("$CERTDIR/sglang.crt") eq '0644', 'fresh: certificate is 0644');
|
|
check((stat("$CERTDIR/sglang.key"))[5] == getgrnam('caddy'),
|
|
'fresh: the key belongs to the caddy group');
|
|
|
|
my $env = slurp_file($ENVFILE);
|
|
check_like($env, qr/^SGLANG_API_KEY=([A-Za-z0-9_-]{43})\n$/, 'fresh: generated key, url-safe, 43 chars');
|
|
|
|
my $unit = slurp_file($UNIT);
|
|
check_like($unit, qr|docker\.io/lmsysorg/sglang:v0\.5\.19-rocm724-mi30x|, 'fresh: image resolved from ROCm 7.2.4 and MI300');
|
|
check_like($unit, qr/--tp-size 1/, 'fresh: tensor parallel default');
|
|
check_like($unit, qr/--context-length 4096/, 'fresh: context length default');
|
|
check_like($unit, qr/--mem-fraction-static 0\.9/, 'fresh: memory fraction default');
|
|
check_unlike($unit, qr/SGLANG_USE_AITER/, 'fresh: no Radeon variables on an Instinct host');
|
|
|
|
my $caddy = slurp_file($CADDY);
|
|
check_like($caddy, qr/reverse_proxy \[::1\]:8000 \{/, 'fresh: caddy proxies to the loopback engine');
|
|
check_like($caddy, qr|tls /etc/ssl/sglang/sglang\.crt /etc/ssl/sglang/sglang\.key|,
|
|
'fresh: caddy uses the sglang certificate pair');
|
|
check_like($caddy, qr/flush_interval -1/, 'fresh: streaming is unbuffered');
|
|
my $main = slurp_file($CADDY_MAIN);
|
|
check_like($main, qr/^import Caddyfile\.d\/\*\.caddyfile$/m, 'fresh: the main Caddyfile imports the drop-ins');
|
|
|
|
check(count_in_log(qr/^podman pull /) == 1, 'fresh: exactly one image pull');
|
|
check_like(stub_log(), qr/^podman pull docker\.io\/lmsysorg\/sglang:v0\.5\.19-rocm724-mi30x$/m,
|
|
'fresh: the pulled image is the resolved tag');
|
|
check_like(stub_log(), qr/^caddy version$/m, 'fresh: caddy is reported from the binary');
|
|
check_like(stub_log(), qr/^caddy validate --config \/etc\/caddy\/Caddyfile$/m,
|
|
'fresh: the configuration is validated before the reload');
|
|
check_like(stub_log(), qr/^systemctl enable --now caddy$/m, 'fresh: caddy enabled and started');
|
|
check_like(stub_log(), qr/^systemctl reload caddy$/m, 'fresh: caddy reloaded');
|
|
check(count_in_log(qr/^systemctl enable sglang$/) == 1, 'fresh: service enabled');
|
|
check(count_in_log(qr/^systemctl start sglang$/) == 1, 'fresh: service started');
|
|
check_like(stub_log(), qr/^firewall-cmd --permanent --add-service=https$/m, 'fresh: HTTPS opened');
|
|
check_like($out, qr/Engine image: docker\.io\/lmsysorg\/sglang:v0\.5\.19-rocm724-mi30x/,
|
|
'fresh: summary names the image');
|
|
check_like($out, qr/systemd: sglang running on \[::1\]:8000/, 'fresh: summary names the endpoint');
|
|
check_like($out, qr/Caddy: configured and reloaded/, 'fresh: summary reports caddy');
|
|
check_like($out, qr/API key \(shown once, store it securely\)/, 'fresh: the generated key is shown once');
|
|
check_unlike($out, qr/✗/, 'fresh: no failed step');
|
|
return;
|
|
}
|
|
|
|
sub scenario_rerun {
|
|
scenario_fresh();
|
|
reset_log();
|
|
my $key_before = slurp_file($ENVFILE);
|
|
my $unit_before = slurp_file($UNIT);
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
|
|
check($rc == 0, 'rerun: exit 0');
|
|
check(count_in_log(qr/^podman pull /) == 0, 'rerun: no second pull');
|
|
check_like(stub_log(), qr/^podman image exists /m, 'rerun: the image is checked instead');
|
|
check(count_in_log(qr/^systemctl enable sglang$/) == 0, 'rerun: no second enable');
|
|
check(count_in_log(qr/^systemctl start sglang$/) == 0, 'rerun: no second start');
|
|
check(count_in_log(qr/^systemctl enable --now caddy$/) == 0, 'rerun: no second caddy enable');
|
|
check(count_in_log(qr/try-restart/) == 0, 'rerun: no restart, the unit did not change');
|
|
check(count_in_log(qr/^dnf install /) == 0, 'rerun: no second package transaction');
|
|
check(slurp_file($ENVFILE) eq $key_before, 'rerun: the API key is reused, not regenerated');
|
|
check(slurp_file($UNIT) eq $unit_before, 'rerun: the unit file is byte-identical');
|
|
check_like($out, qr/API key: existing key reused/, 'rerun: the key is reported as reused');
|
|
check_like($out, qr/Engine image: already present/, 'rerun: the image is reported present');
|
|
check_like($out, qr/systemd: sglang running/, 'rerun: the service is reported running');
|
|
check_unlike($out, qr/would /, 'rerun: nothing is phrased as would');
|
|
check_unlike($out, qr/✗/, 'rerun: no failed step');
|
|
return;
|
|
}
|
|
|
|
sub scenario_dry_run {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1] (rev 01)\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--dry-run');
|
|
|
|
check($rc == 0, 'dry run: exit 0');
|
|
check(!-f $UNIT, 'dry run: no unit written');
|
|
check(!-f $ENVFILE, 'dry run: no environment file written');
|
|
check(!-f $CADDY, 'dry run: no caddy drop-in written');
|
|
check(!-e '/etc/caddy', 'dry run: no caddy directory created');
|
|
check(!-d $CERTDIR, 'dry run: no certificate directory');
|
|
check(count_in_log(qr/^podman pull /) == 0, 'dry run: no pull');
|
|
check(count_in_log(qr/^systemctl (enable|start) /) == 0, 'dry run: no service change');
|
|
check(count_in_log(qr/^dnf install /) == 0, 'dry run: no package transaction');
|
|
check_like($out, qr/DRY RUN: no changes will be made/, 'dry run: banner');
|
|
check_like($out, qr/would be pulled|would be installed/, 'dry run: phrased as would');
|
|
check_like($out, qr|Engine image: docker\.io/lmsysorg/sglang:v0\.5\.19-rocm724-mi30x|,
|
|
'dry run: the resolved image is reported');
|
|
return;
|
|
}
|
|
|
|
sub scenario_uninstall {
|
|
scenario_fresh();
|
|
reset_log();
|
|
my ($rc, $out) = run_script('--uninstall');
|
|
|
|
check($rc == 0, 'uninstall: exit 0');
|
|
check(!-f $UNIT, 'uninstall: unit removed');
|
|
check(!-f $ENVFILE, 'uninstall: environment file removed');
|
|
check(!-f $CADDY, 'uninstall: caddy drop-in removed');
|
|
check(-f $CADDY_MAIN, 'uninstall: the main Caddyfile is kept');
|
|
check(!-d $CERTDIR, 'uninstall: certificate directory removed');
|
|
check(-d $STATEDIR, 'uninstall: model cache kept');
|
|
check(-e "$ST/image.docker.io_lmsysorg_sglang_v0.5.19-rocm724-mi30x",
|
|
'uninstall: the image is kept in podman');
|
|
check_like(stub_log(), qr/^systemctl stop sglang$/m, 'uninstall: service stopped');
|
|
check_like(stub_log(), qr/^systemctl disable sglang$/m, 'uninstall: service disabled');
|
|
check_like(stub_log(), qr/^systemctl reload caddy$/m, 'uninstall: caddy reloaded after the drop-in');
|
|
check_like($out, qr/SGLang service stopped/, 'uninstall: summary reports the stop');
|
|
check_like($out, qr/caddy drop-in removed/, 'uninstall: summary reports the drop-in');
|
|
check_like($out, qr/Kept on the system/, 'uninstall: the kept state is listed');
|
|
check_like($out, qr/ModelScope cache/, 'uninstall: the cache is named as kept');
|
|
return;
|
|
}
|
|
|
|
sub scenario_uninstall_twice {
|
|
scenario_uninstall();
|
|
reset_log();
|
|
my ($rc, $out) = run_script('--uninstall');
|
|
check($rc == 0, 'uninstall twice: exit 0');
|
|
check_like($out, qr/already absent/, 'uninstall twice: idempotent');
|
|
check(count_in_log(qr/^systemctl stop /) == 0, 'uninstall twice: nothing to stop');
|
|
check(count_in_log(qr/^systemctl reload caddy$/) == 0,
|
|
'uninstall twice: no reload without a drop-in');
|
|
return;
|
|
}
|
|
|
|
# A host deployed by the nginx release carries its drop-in. Both a deploy and
|
|
# an uninstall remove it, so exactly one proxy owns :443 afterwards.
|
|
sub scenario_legacy_nginx {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
open(my $fh, '>', $NGINX_LEGACY) or die "cannot write $NGINX_LEGACY: $!\n";
|
|
print {$fh} "server {\n listen 443 ssl;\n}\n";
|
|
close($fh);
|
|
# The stub state: nginx is running on this host, so the removal reloads it.
|
|
my $st;
|
|
open($st, '>', "$ST/active.nginx") and close($st);
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
check($rc == 0, 'legacy nginx: exit 0');
|
|
check(!-f $NGINX_LEGACY, 'legacy nginx: the old drop-in is gone on deploy');
|
|
check_like(stub_log(), qr/^systemctl reload nginx$/m,
|
|
'legacy nginx: the running nginx is reloaded');
|
|
check_like($out, qr/Legacy nginx configuration: removed/, 'legacy nginx: the summary names it');
|
|
|
|
# An uninstall on a host the new release never deployed cleans it too.
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
open($fh, '>', $NGINX_LEGACY) or die "cannot write $NGINX_LEGACY: $!\n";
|
|
print {$fh} "server {\n listen 443 ssl;\n}\n";
|
|
close($fh);
|
|
reset_log();
|
|
($rc, $out) = run_script('--uninstall');
|
|
check($rc == 0, 'legacy nginx: uninstall exit 0');
|
|
check(!-f $NGINX_LEGACY, 'legacy nginx: the old drop-in is gone on uninstall');
|
|
check_like($out, qr/legacy nginx configuration removed/, 'legacy nginx: the uninstall summary names it');
|
|
return;
|
|
}
|
|
|
|
# Where no repository carries the caddy package, the official release binary
|
|
# takes its place: download, extract, install, the service user, and the unit
|
|
# file the package would have carried.
|
|
sub scenario_caddy_binary {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
write_fixture('caddy-no-package', "1\n");
|
|
unlink('/usr/bin/caddy'); # order independence: no package binary, no stub
|
|
unlink("$BIN/caddy") or die "cannot remove the caddy stub: $!\n";
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
|
|
check($rc == 0, 'caddy binary: exit 0');
|
|
check_like(stub_log(), qr/^dnf install -y caddy$/m, 'caddy binary: the package install was attempted');
|
|
check_like(stub_log(), qr{^curl -fsSL -o \S+ https://github\.com/caddyserver/caddy/releases/download/v2\.10\.2/caddy_2\.10\.2_linux_amd64\.tar\.gz$}m,
|
|
'caddy binary: the release asset is downloaded');
|
|
check_like(stub_log(), qr/^tar -xzf \S+ -C \S+$/m, 'caddy binary: the archive is extracted');
|
|
check(-x $CADDY_BIN, 'caddy binary: the binary is installed executable');
|
|
check_like(stub_log(), qr/^useradd --system --home-dir \/var\/lib\/caddy --create-home --shell \/sbin\/nologin caddy$/m,
|
|
'caddy binary: the service user is created');
|
|
check(-f $CADDY_UNIT, 'caddy binary: the unit file is written');
|
|
my $unit = slurp_file($CADDY_UNIT);
|
|
check_like($unit, qr|ExecStart=/usr/local/bin/caddy run --environ --config /etc/caddy/Caddyfile|,
|
|
'caddy binary: the unit runs the release binary');
|
|
check_like(stub_log(), qr/^systemctl daemon-reload$/m, 'caddy binary: systemd reloaded');
|
|
check_like(stub_log(), qr{^caddy validate --config /etc/caddy/Caddyfile$}m,
|
|
'caddy binary: the installed binary validates');
|
|
check_like($out, qr/Caddy: configured and reloaded/, 'caddy binary: the deployment completes');
|
|
|
|
unlink($CADDY_BIN);
|
|
unlink($CADDY_UNIT);
|
|
unlink("$FIX/caddy-no-package");
|
|
symlink("$RIG/stub.pl", "$BIN/caddy") or die "cannot restore the caddy stub: $!\n";
|
|
return;
|
|
}
|
|
|
|
sub scenario_radeon {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Navi 31 [Radeon RX 7900 XTX] [1002:744c] (rev c8)\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
|
|
check($rc == 1, 'radeon: exit 1');
|
|
check_like($out, qr/Radeon card detected/, 'radeon: names the cause');
|
|
check_like($out, qr/rocm\.Dockerfile/, 'radeon: names the build recipe');
|
|
check_like($out, qr/--image sglang-rocm:latest/, 'radeon: names the way forward');
|
|
check(!-f $UNIT, 'radeon: nothing deployed');
|
|
return;
|
|
}
|
|
|
|
sub scenario_radeon_dev {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Strix Halo [Radeon Graphics / Radeon 8050S Graphics / Radeon 8060S Graphics] [1002:1586] (rev d1)\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
|
|
check($rc == 0, 'radeon dev: exit 0');
|
|
check_like(stub_log(), qr{^curl .*rocm/sgl-dev/tags\?.*$}m,
|
|
'radeon dev: the AMD tag index is asked for the newest build');
|
|
my $unit = slurp_file($UNIT);
|
|
check_like($unit, qr|docker\.io/rocm/sgl-dev:v0\.5\.19-rocm724-gfx1151-20260916|,
|
|
'radeon dev: the newest dated build is deployed');
|
|
check_like($unit, qr/Environment=SGLANG_USE_AITER=false/, 'radeon dev: AITER off');
|
|
check_like($unit, qr/Environment=SGLANG_ROCM_FUSED_DECODE_MLA=false/,
|
|
'radeon dev: fused decode MLA off');
|
|
check_like($out, qr/AMD publishes this build daily/, 'radeon dev: the moving tag is stated');
|
|
check_like($out, qr/the only flavour published for gfx1151/, 'radeon dev: the flavour is explained');
|
|
|
|
reset_log();
|
|
my ($rc2, $out2) = run_script('--model', 'ZhipuAI/GLM-5.3', '--image', 'localhost/pinned:1');
|
|
check_like(slurp_file($UNIT), qr|localhost/pinned:1|, 'radeon dev: --image pins a build');
|
|
check(count_in_log(qr{^curl .*rocm/sgl-dev/tags\?.*$}) == 0,
|
|
'radeon dev: no tag resolution when --image is given');
|
|
return;
|
|
}
|
|
|
|
sub scenario_radeon_with_image {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Strix Halo [Radeon 8060S] [1002:150e]\n");
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--image', 'localhost/sglang-rocm:gfx1151');
|
|
|
|
check($rc == 0, 'radeon with image: exit 0');
|
|
my $unit = slurp_file($UNIT);
|
|
check_like($unit, qr/Environment=SGLANG_USE_AITER=false/, 'radeon with image: AITER off');
|
|
check_like($unit, qr/Environment=SGLANG_ROCM_FUSED_DECODE_MLA=false/,
|
|
'radeon with image: fused decode MLA off');
|
|
check_like($unit, qr|localhost/sglang-rocm:gfx1151|, 'radeon with image: the requested image');
|
|
my ($rc2, $out2) = run_script('--model', 'ZhipuAI/GLM-5.3', '--image', 'localhost/sglang-rocm:gfx1151');
|
|
check($rc2 == 0, 'radeon with image: rerun exit 0');
|
|
check(slurp_file($UNIT) eq $unit, 'radeon with image: the unit is unchanged on a rerun');
|
|
return;
|
|
}
|
|
|
|
sub scenario_rocm_flavours {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.3\n");
|
|
my ($rc, $out) = run_script('--dry-run', '--model', 'ZhipuAI/GLM-5.3');
|
|
check_like($out, qr/rocm720-mi30x/, 'flavour: ROCm 7.2.3 takes rocm720');
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "10.0.0\n");
|
|
($rc, $out) = run_script('--dry-run', '--model', 'ZhipuAI/GLM-5.3');
|
|
check_like($out, qr/rocm10-mi30x/, 'flavour: ROCm 10.0.0 takes rocm10');
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI350X [1002:75a0]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
($rc, $out) = run_script('--dry-run', '--model', 'ZhipuAI/GLM-5.3');
|
|
check_like($out, qr/rocm724-mi35x/, 'flavour: MI350X takes mi35x');
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "6.4.0\n");
|
|
($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
check($rc == 1, 'flavour: an old host ROCm is refused');
|
|
check_like($out, qr/No published image targets ROCm 6\.4\.0/, 'flavour: the refusal names the version');
|
|
check_like($out, qr/--image/, 'flavour: the refusal names the way forward');
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
($rc, $out) = run_script('--dry-run', '--model', 'ZhipuAI/GLM-5.3');
|
|
check_like($out, qr/rocm10-mi30x/, 'flavour: no host ROCm takes the newest');
|
|
check_like($out, qr/No ROCm userland found on the host/, 'flavour: the assumption is stated out loud');
|
|
return;
|
|
}
|
|
|
|
sub scenario_modelscope {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
write_fixture('ms.code', "200\n");
|
|
write_fixture('ms.json', qq({"Data":{"Name":"GLM-5.3"}}\n));
|
|
my ($rc, $out) = run_script('--dry-run', '--model', 'ZhipuAI/GLM-5.3');
|
|
check_like($out, qr/Verifying GLM 5\.3 on ModelScope\.\.\. found/, 'modelscope: found');
|
|
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
write_fixture('ms.code', "404\n");
|
|
($rc, $out) = run_script('--model', 'nope/does-not-exist');
|
|
check($rc == 1, 'missing model: exit 1');
|
|
check_like($out, qr/does not exist on ModelScope/, 'missing model: named as missing');
|
|
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
# A 200 without a Name carries no usable answer: the deployment proceeds with
|
|
# the probe marked unverified rather than blocked.
|
|
write_fixture('ms.code', "200\n");
|
|
write_fixture('ms.json', qq({}\n));
|
|
($rc, $out) = run_script('--dry-run', '--model', 'ZhipuAI/GLM-5.3');
|
|
check($rc == 0, 'modelscope: an unnamed answer does not block');
|
|
check_like($out, qr/could not verify/, 'modelscope: the unnamed answer reads as unverified');
|
|
return;
|
|
}
|
|
|
|
sub scenario_hardware {
|
|
reset_fixture();
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
check($rc == 1, 'no GPU: exit 1');
|
|
check_like($out, qr/No AMD GPU detected/, 'no GPU: named as missing');
|
|
|
|
# The missing driver is checked outside the rig: podman bind-mounts /dev/kfd into a
|
|
# privileged container, so it cannot be removed from inside one.
|
|
return;
|
|
}
|
|
|
|
# Run without a GPU device node (a container without --privileged, or a host whose
|
|
# driver is not loaded): the script must refuse and say what is missing.
|
|
sub scenario_no_driver {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
if (-e '/dev/kfd') {
|
|
print "skip no driver: this run has the device node (needs a run without --privileged)\n";
|
|
return;
|
|
}
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
check($rc == 1, 'no driver: exit 1');
|
|
check_like($out, qr/amdgpu kernel driver is not ready/, 'no driver: named as missing');
|
|
check_like($out, qr{/dev/kfd}, 'no driver: names the device node');
|
|
check(!-f $UNIT, 'no driver: nothing deployed');
|
|
return;
|
|
}
|
|
|
|
sub scenario_bad_os {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('os-release', "ID=ubuntu\nVERSION_ID=\"24.04\"\n");
|
|
my $real = slurp_file('/etc/os-release');
|
|
open(my $fh, '>', '/etc/os-release') or die "cannot write /etc/os-release: $!\n";
|
|
print {$fh} slurp_file("$FIX/os-release");
|
|
close($fh);
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--dry-run');
|
|
open(my $back, '>', '/etc/os-release') or die "cannot restore /etc/os-release: $!\n";
|
|
print {$back} $real;
|
|
close($back);
|
|
|
|
check($rc == 1, 'bad OS: exit 1');
|
|
check_like($out, qr/Unsupported operating system: 'ubuntu'/, 'bad OS: names the system');
|
|
check_like($out, qr/Supported systems: Fedora, CentOS Stream, openEuler/, 'bad OS: lists the supported ones');
|
|
return;
|
|
}
|
|
|
|
sub scenario_validation {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--port', '443', '--dry-run');
|
|
check($rc == 1, 'validation: port 443 refused');
|
|
check_like($out, qr/must be an integer 1-65535 and not 443/, 'validation: port message');
|
|
($rc, $out) = run_script('--model', 'not-a-model-id', '--dry-run');
|
|
check($rc == 1, 'validation: bad model ID refused');
|
|
check_like($out, qr/Invalid model ID/, 'validation: model message');
|
|
($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--gpu-memory-utilization', '1.5', '--dry-run');
|
|
check($rc == 1, 'validation: memory fraction above 1 refused');
|
|
check_like($out, qr/must be in \(0, 1\]/, 'validation: memory fraction message');
|
|
($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--api-key', 'has space', '--dry-run');
|
|
check($rc == 1, 'validation: whitespace API key refused');
|
|
($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--rocm-flavour', 'rocm999', '--dry-run');
|
|
check($rc == 1, 'validation: unknown flavour refused');
|
|
check_like($out, qr/Invalid --rocm-flavour/, 'validation: flavour message');
|
|
($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--image', 'bad image%name', '--dry-run');
|
|
check($rc == 1, 'validation: image with whitespace or % refused');
|
|
check_like($out, qr/Invalid --image/, 'validation: image message');
|
|
return;
|
|
}
|
|
|
|
sub scenario_api_key_argument {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--api-key', 'chosen-key-1234');
|
|
check($rc == 0, 'api key argument: exit 0');
|
|
check_like(slurp_file($ENVFILE), qr/^SGLANG_API_KEY=chosen-key-1234$/, 'api key argument: the key is stored');
|
|
check_unlike($out, qr/chosen-key-1234/, 'api key argument: a given key is never echoed');
|
|
my $unit = slurp_file($UNIT);
|
|
check_unlike($unit, qr/chosen-key-1234/, 'api key argument: the key is not in the unit');
|
|
return;
|
|
}
|
|
|
|
sub scenario_non_root {
|
|
reset_fixture();
|
|
my $out_file = "$WORK/log/run.out";
|
|
my $pid = fork();
|
|
die "cannot fork: $!\n" unless defined $pid;
|
|
if ($pid == 0) {
|
|
open(STDOUT, '>', $out_file);
|
|
open(STDERR, '>&', \*STDOUT);
|
|
$ENV{PATH} = "$BIN:/usr/bin:/bin";
|
|
$ENV{STUB_WORK} = $WORK;
|
|
$) = 1000;
|
|
$> = 1000;
|
|
exec '/usr/bin/perl', $SCRIPT, '--version';
|
|
exit 126;
|
|
}
|
|
waitpid($pid, 0);
|
|
my $rc = $? >> 8;
|
|
my $out = slurp_file($out_file);
|
|
check($rc == 0, 'non-root: --version works without root');
|
|
check_like($out, qr/^sglang-deploy\.pl 2\.1\.0$/, 'non-root: the version is printed');
|
|
|
|
my $pid3 = fork();
|
|
die "cannot fork: $!\n" unless defined $pid3;
|
|
if ($pid3 == 0) {
|
|
open(STDOUT, '>', $out_file);
|
|
open(STDERR, '>&', \*STDOUT);
|
|
$ENV{PATH} = "$BIN:/usr/bin:/bin";
|
|
$ENV{STUB_WORK} = $WORK;
|
|
$) = 1000;
|
|
$> = 1000;
|
|
exec '/usr/bin/perl', $SCRIPT, '--model', 'ZhipuAI/GLM-5.3';
|
|
exit 126;
|
|
}
|
|
waitpid($pid3, 0);
|
|
my $rc3 = $? >> 8;
|
|
my $out3 = slurp_file($out_file);
|
|
check($rc3 == 1, 'non-root: exit 1');
|
|
check_like($out3, qr/must be run as root/, 'non-root: the message says so');
|
|
return;
|
|
}
|
|
|
|
sub scenario_offline {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
write_fixture('image-missing', "1\n");
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
check($rc == 1, 'unpublished tag: exit 1');
|
|
check_like($out, qr/resolved image tag does not exist/, 'unpublished tag: named as missing');
|
|
check_like($out, qr/--image/, 'unpublished tag: names the way forward');
|
|
return;
|
|
}
|
|
|
|
sub scenario_dependency_section {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
write_fixture('rpm-installed', "pciutils\ncurl\nopenssl\npodman\ntar\n");
|
|
reset_log();
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3', '--dry-run');
|
|
check($rc == 0, 'deps: exit 0');
|
|
check(count_in_log(qr/^dnf install /) == 0, 'deps: no transaction when all packages are present');
|
|
check_like($out, qr/All 5 packages already installed/, 'deps: reported as present');
|
|
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
write_fixture('rpm-installed', "\n");
|
|
reset_log();
|
|
($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
check_like(stub_log(), qr/^dnf install -y pciutils curl openssl podman tar$/m,
|
|
'deps: the five packages are installed in one transaction');
|
|
check_like(stub_log(), qr/^caddy version$/m, 'deps: caddy is checked after the transaction');
|
|
check_unlike($out, qr/run server-setup\.pl first/,
|
|
'deps: no warning points at server-setup.pl, caddy is installed here');
|
|
return;
|
|
}
|
|
|
|
# CentOS Stream carries caddy in EPEL, and the repository file installs first,
|
|
# which is what makes the package transaction below it resolvable.
|
|
sub scenario_epel {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
my $real = slurp_file('/etc/os-release');
|
|
open(my $fh, '>', '/etc/os-release') or die "cannot write /etc/os-release: $!\n";
|
|
print {$fh} "ID=centos\nVERSION_ID=\"10\"\n";
|
|
close($fh);
|
|
unlink('/usr/bin/caddy'); # order independence: no binary, no stub
|
|
unlink("$BIN/caddy");
|
|
reset_log();
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
open(my $back, '>', '/etc/os-release') or die "cannot restore /etc/os-release: $!\n";
|
|
print {$back} $real;
|
|
close($back);
|
|
|
|
check($rc == 0, 'epel: exit 0');
|
|
check_like(stub_log(), qr/^dnf install -y epel-release$/m, 'epel: the repository file installs first');
|
|
check_like(stub_log(), qr/^dnf install -y caddy$/m, 'epel: the package installs after it');
|
|
check_like($out, qr/Installing caddy\.\.\. package/, 'epel: the package path is taken');
|
|
check_like($out, qr/Caddy: configured and reloaded/, 'epel: the deployment completes');
|
|
|
|
unlink('/usr/bin/caddy');
|
|
unlink("$FIX/caddy-no-package");
|
|
symlink("$RIG/stub.pl", "$BIN/caddy") or die "cannot restore the caddy stub: $!\n";
|
|
return;
|
|
}
|
|
|
|
sub scenario_custom_layout {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
my ($rc, $out) = run_script(
|
|
'--model', 'ZhipuAI/GLM-5.3',
|
|
'--service-name', 'llm',
|
|
'--state-dir', '/srv/llm',
|
|
'--cert-dir', '/etc/ssl/llm',
|
|
'--port', '9000',
|
|
'--tensor-parallel', '8',
|
|
'--max-model-len', '131072',
|
|
'--gpu-memory-utilization', '0.85',
|
|
);
|
|
check($rc == 0, 'custom layout: exit 0');
|
|
check(-f '/etc/systemd/system/llm.service', 'custom layout: the named unit is written');
|
|
check(-f '/etc/caddy/Caddyfile.d/llm.caddyfile', 'custom layout: the named caddy drop-in is written');
|
|
# The certificate file names follow the program, as they did before, not the
|
|
# service name; the directory follows --cert-dir.
|
|
check(-f '/etc/ssl/llm/sglang.crt', 'custom layout: certificates follow the directory');
|
|
check(-d '/srv/llm/modelscope', 'custom layout: the state directory follows');
|
|
my $unit = slurp_file('/etc/systemd/system/llm.service');
|
|
check_like($unit, qr/--port 9000/, 'custom layout: port in the unit');
|
|
check_like($unit, qr/--tp-size 8/, 'custom layout: tensor parallel in the unit');
|
|
check_like($unit, qr/--context-length 131072/, 'custom layout: context length in the unit');
|
|
check_like($unit, qr/--mem-fraction-static 0\.85/, 'custom layout: memory fraction in the unit');
|
|
check_like($unit, qr|--volume /srv/llm/modelscope:/root/\.cache/modelscope:Z|,
|
|
'custom layout: the cache volume follows the state directory');
|
|
unlink('/etc/systemd/system/llm.service');
|
|
unlink('/etc/caddy/Caddyfile.d/llm.caddyfile');
|
|
remove_tree('/etc/ssl/llm');
|
|
remove_tree('/srv/llm');
|
|
return;
|
|
}
|
|
|
|
sub scenario_tls_existing {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
my $crt = slurp_file("$CERTDIR/sglang.crt");
|
|
reset_log();
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
check(slurp_file("$CERTDIR/sglang.crt") eq $crt, 'tls: an existing certificate is kept');
|
|
check(count_in_log(qr/^openssl /) == 0, 'tls: no second certificate');
|
|
check_like($out, qr/TLS certificate: configured/, 'tls: reported as configured');
|
|
return;
|
|
}
|
|
|
|
sub scenario_stale_container {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
my $fh;
|
|
open($fh, q{>}, "$ST/container.sglang") and close($fh);
|
|
my ($rc, $out) = run_script('--uninstall');
|
|
check($rc == 0, 'stale container: uninstall exit 0');
|
|
check_like(stub_log(), qr/^podman rm -f sglang$/m, 'stale container: removed');
|
|
check(!-e "$ST/container.sglang", 'stale container: gone from podman');
|
|
return;
|
|
}
|
|
|
|
sub scenario_selinux {
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
$ENV{STUB_SELINUX} = 'Permissive';
|
|
my ($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
check(count_in_log(qr/^setsebool /) == 0, 'selinux: nothing set while permissive');
|
|
check_like($out, qr/SELinux: permissive \(skipped\)/, 'selinux: reported as skipped');
|
|
|
|
# Enforcing with no confined caddy policy: the distributions run caddy
|
|
# unconfined, so no boolean is touched.
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
$ENV{STUB_SELINUX} = 'Enforcing';
|
|
reset_log();
|
|
($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
check(count_in_log(qr/^setsebool /) == 0, 'selinux: no boolean without a confined policy');
|
|
check_like($out, qr/SELinux: caddy runs unconfined \(nothing to do\)/,
|
|
'selinux: the unconfined case is stated');
|
|
|
|
# Enforcing with a confined caddy policy loaded: the boolean is set.
|
|
reset_fixture();
|
|
write_fixture('gpu.txt', "03:00.0 VGA compatible controller [0300]: Advanced Micro Devices, Inc. [AMD/ATI] Instinct MI300X OAM [1002:74a1]\n");
|
|
write_fixture('rocm.txt', "7.2.4\n");
|
|
write_fixture('semodule-caddy', "1\n");
|
|
reset_log();
|
|
($rc, $out) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
check_like(stub_log(), qr/^semodule -l$/m, 'selinux: the loaded modules are asked for');
|
|
check_like(stub_log(), qr/^setsebool -P httpd_can_network_connect=1$/m, 'selinux: the boolean is set');
|
|
check_like($out, qr/SELinux: httpd_can_network_connect on/, 'selinux: reported as on');
|
|
reset_log();
|
|
my ($rc2, $out2) = run_script('--model', 'ZhipuAI/GLM-5.3');
|
|
check(count_in_log(qr/^setsebool /) == 0, 'selinux: nothing set when already on');
|
|
unlink("$FIX/semodule-caddy");
|
|
delete $ENV{STUB_SELINUX};
|
|
return;
|
|
}
|
|
|
|
my %scenarios = (
|
|
fresh => \&scenario_fresh,
|
|
rerun => \&scenario_rerun,
|
|
dry_run => \&scenario_dry_run,
|
|
uninstall => \&scenario_uninstall,
|
|
uninstall_twice => \&scenario_uninstall_twice,
|
|
legacy_nginx => \&scenario_legacy_nginx,
|
|
caddy_binary => \&scenario_caddy_binary,
|
|
epel => \&scenario_epel,
|
|
radeon => \&scenario_radeon,
|
|
radeon_dev => \&scenario_radeon_dev,
|
|
radeon_with_image => \&scenario_radeon_with_image,
|
|
rocm_flavours => \&scenario_rocm_flavours,
|
|
modelscope => \&scenario_modelscope,
|
|
hardware => \&scenario_hardware,
|
|
bad_os => \&scenario_bad_os,
|
|
no_driver => \&scenario_no_driver,
|
|
validation => \&scenario_validation,
|
|
api_key_argument => \&scenario_api_key_argument,
|
|
non_root => \&scenario_non_root,
|
|
offline => \&scenario_offline,
|
|
dependencies => \&scenario_dependency_section,
|
|
custom_layout => \&scenario_custom_layout,
|
|
tls_existing => \&scenario_tls_existing,
|
|
stale_container => \&scenario_stale_container,
|
|
selinux => \&scenario_selinux,
|
|
);
|
|
|
|
prepare_stubs();
|
|
prepare_group();
|
|
prepare_devices();
|
|
|
|
my @wanted = @ARGV ? @ARGV : sort keys %scenarios;
|
|
for my $name (@wanted) {
|
|
if (!exists $scenarios{$name}) {
|
|
print "FAIL unknown scenario $name\n";
|
|
$failed++;
|
|
next;
|
|
}
|
|
print "\n== $name ==\n";
|
|
$scenarios{$name}->();
|
|
}
|
|
|
|
print "\n" . ($failed ? "$failed FAILED of " . ($passed + $failed) : "all " . $passed . " checks passed") . "\n";
|
|
exit($failed ? 1 : 0);
|