diff --git a/packaging/README-ship.md b/packaging/README-ship.md index 3da24790e..0d62e74a5 100644 --- a/packaging/README-ship.md +++ b/packaging/README-ship.md @@ -117,4 +117,7 @@ program) or failed in 4 s (the 0.3.10 installer job). The file shape keeps the b `pc1-cpu-prove.ps1` does that in the job itself. `tools/ci/bash-body-check.sh` fails CI on an inline body that does not parse: it finds every body handed to bash (`bash -c "..."`, `bash -lc $var`, a `+` concatenation, a here-string written to a file and run), unescapes it the way PowerShell would, and runs `bash -n` on it. A body it sees but cannot read fails too. -`--self-test` shows it firing on the fixtures under `tools/ci/fixtures/`. +`--self-test` shows it firing on the fixtures under `tools/ci/fixtures/`. A job script is published from a worktree and +never passes CI before it runs, so `packaging/ota/publish-jobs.sh add --kind run` runs the same check (and +`tools/ci/prover-socket-check.sh`, the root-socket class; `bash -n` for a `.sh` script) on the script before anything is +signed, and refuses the publish with the check's output. `packaging/ota/test-publish-jobs.sh` proves the refusals. diff --git a/packaging/ota/publish-jobs.sh b/packaging/ota/publish-jobs.sh index 4e26f0ed9..8d444d5ff 100755 --- a/packaging/ota/publish-jobs.sh +++ b/packaging/ota/publish-jobs.sh @@ -256,6 +256,19 @@ if [ "$CMD" = add ]; then [ -n "$SCRIPT" ] && [ -f "$SCRIPT" ] || { echo "run: --script is required" >&2; exit 2; } [ -n "$SHELL_KIND" ] || { case "$SCRIPT" in *.sh) SHELL_KIND=bash ;; *) SHELL_KIND=powershell ;; esac; } [ -n "$PLATFORM" ] || { [ "$SHELL_KIND" = powershell ] && PLATFORM=windows || PLATFORM=any; } + # A job script is published from a worktree and never passes CI before it runs (5 October 2026: the root-socket + # fault came back from a branch without the check), so the class checks run here on the script itself, before + # anything is signed. A failure refuses the publish with the check's output; a missing check refuses it too. + # PowerShell: every inline bash body must parse (tools/ci/bash-body-check.sh, the lost-quote class; a body it + # cannot read fails, never skips). Bash: the script itself must parse. Both: a root prover run kills the GPU + # server and unlinks its socket (tools/ci/prover-socket-check.sh, the root-socket class). A check file that is + # missing from this tree refuses too (bash exits 127 with the reason), so no job goes out unchecked. + if [ "$SHELL_KIND" = powershell ]; then + out="$(bash "$ROOT/tools/ci/bash-body-check.sh" "$SCRIPT" 2>&1)" || { echo "run: bash-body-check.sh refuses $SCRIPT:" >&2; printf '%s\n' "$out" >&2; exit 1; } + else + out="$(bash -n "$SCRIPT" 2>&1)" || { echo "run: bash -n refuses $SCRIPT (the lost-quote class):" >&2; printf '%s\n' "$out" >&2; exit 1; } + fi + out="$(bash "$ROOT/tools/ci/prover-socket-check.sh" "$SCRIPT" 2>&1)" || { echo "run: prover-socket-check.sh refuses $SCRIPT:" >&2; printf '%s\n' "$out" >&2; exit 1; } PARAMS="$(python3 -c 'import json,sys; print(json.dumps({"script": open(sys.argv[1]).read(), "shell": sys.argv[2], "elevated": sys.argv[3]=="1", "stop_miners_first": sys.argv[4]=="1", **({"timeout_minutes": int(sys.argv[5])} if sys.argv[5] else {})}))' "$SCRIPT" "$SHELL_KIND" "$ELEVATED" "$STOP_MINERS" "$TIMEOUT_MIN")" [ -n "$TITLE" ] || TITLE="run $(basename "$SCRIPT")" ;; diff --git a/packaging/ota/test-publish-jobs.sh b/packaging/ota/test-publish-jobs.sh index d3fbc9686..c5f8b545e 100755 --- a/packaging/ota/test-publish-jobs.sh +++ b/packaging/ota/test-publish-jobs.sh @@ -8,7 +8,10 @@ # byte and its sig IS the plain signature; the signer reads the envelope back; a second `add` (a new publish) gives # a new envelope; an envelope made by hand from the FIRST file and the SECOND signature (the stale pair an edge can # serve across a deploy) is REFUSED by the signer with the words the app logs; a tampered inner byte is refused; -# `sign` rewrites all three consistently; and the real OTA key is the key the app embeds. +# `sign` rewrites all three consistently; a `run` script that fails a class check (a lost quote in an inline bash +# body, a body the check cannot read, a bash script that does not parse, a root prover run without the socket cleanup) +# is REFUSED before anything is signed and the envelope is left as it was; and the real OTA key is the key the app +# embeds. # # A check is trusted only once it has fired on a known-good and a known-bad case (standing rule, 4 October 2026), so # every negative case here must FAIL for the run to pass. Needs ~/.config/igneum/ota-signing-key (the publisher's own @@ -96,6 +99,21 @@ PY grep -q "^inner-identical True$" "$T/inner2.txt" && grep -q "^sig-identical True$" "$T/inner2.txt" && ok "after sign: the envelope still holds the plain file and its signature" || bad "after sign: the envelope and the pair differ" expect_ok "list reads the folder" "$PUBLISH" list --dest "$D" +echo "== the class checks refuse a bad script before anything is signed (5 October 2026: a job never passes CI first)" +cp "$D/igneum-jobs.signed.json" "$T/before.signed" +printf '& wsl.exe -d Ubuntu-24.04 -- bash -c '"'"'echo "started; ls /opt/igneum'"'"' 2>&1\n' > "$T/lost-quote.ps1" +expect_fail "a PowerShell script with a lost quote in its bash body" "$PUBLISH" add --kind run --target 1ccfe586 --script "$T/lost-quote.ps1" --id test-lost-quote --dest "$D" --base-url https://example.invalid/dl/t +grep -q "unexpected EOF while looking for matching" "$T/out" && ok "refused with the bash -n error" || bad "another reason: $(tail -1 "$T/out")" +printf '$cmd = (Get-Content body.txt) -join "; "\n& wsl.exe -- bash -c $cmd\n' > "$T/unreadable.ps1" +expect_fail "a PowerShell script whose bash body the check cannot read" "$PUBLISH" add --kind run --target 1ccfe586 --script "$T/unreadable.ps1" --id test-unreadable --dest "$D" --base-url https://example.invalid/dl/t +grep -q "unextractable body" "$T/out" && ok "refused as unextractable, not skipped" || bad "another reason: $(tail -1 "$T/out")" +printf 'echo "started; ls /opt/igneum\n' > "$T/lost-quote.sh" +expect_fail "a bash script with a lost quote" "$PUBLISH" add --kind run --target 1ccfe586 --script "$T/lost-quote.sh" --id test-lost-quote-sh --dest "$D" --base-url https://example.invalid/dl/t +printf '& wsl.exe -d Ubuntu-24.04 -u root -- bash /mnt/c/prove.sh\n# igneum-prove-host --mode shard --shard 0\n' > "$T/root-prover.ps1" +expect_fail "a root prover script without the socket cleanup" "$PUBLISH" add --kind run --target 1ccfe586 --script "$T/root-prover.ps1" --id test-root-socket --dest "$D" --base-url https://example.invalid/dl/t +grep -q "prover-socket" "$T/out" && ok "refused by the socket check" || bad "another reason: $(tail -1 "$T/out")" +cmp -s "$T/before.signed" "$D/igneum-jobs.signed.json" && ok "a refused publish leaves the envelope untouched" || bad "a refused publish changed the envelope" + echo "== the key" EMB="$("$SIGNER" embedded | head -1)" [ "$EMB" = "$(tr -d '[:space:]' < "$PUB")" ] && ok "the embedded key is the Mac's OTA public key" || bad "the embedded key is not $PUB" diff --git a/tools/ci/prover-socket-check.sh b/tools/ci/prover-socket-check.sh new file mode 100755 index 000000000..ddd10e561 --- /dev/null +++ b/tools/ci/prover-socket-check.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# The root-socket class (5 October 2026, 20:00Z): a PC 2 job ran igneum-prove-host as root inside WSL2, which started +# an sp1-gpu-server whose socket /tmp/sp1-cuda-0.sock stayed root-owned after the job; the live prover (the app's user) +# then failed every shard with "CudaClientError: Connect(PermissionDenied)" until the socket was gone. Rule: every +# playbook that runs the prover host as root on a shared card kills the server AND unlinks its socket (at the start +# and at the end), or runs as the app's user. This check fails CI when a script runs `igneum-prove-host` under a +# `-u root` WSL session without both lines. With file arguments it checks those files only (the jobs publisher runs it +# on the script being published, packaging/ota/publish-jobs.sh add --kind run; a PC job never passes CI before it runs). +set -euo pipefail +cd "$(dirname "$0")/../.." +fail=0 +# the publisher's test writes a known-bad root prover script on purpose, to prove the publish refuses it +ALLOW='^packaging/ota/test-publish-jobs\.sh$' +list_files() { if [ $# -gt 0 ]; then printf '%s\n' "$@"; else git ls-files 'tools/**' 'relay/playbooks/**' 'proving/**' 'packaging/**' | grep -E '\.(sh|ps1|mjs)$'; fi; } +while IFS= read -r f; do + [[ "$f" =~ $ALLOW ]] && continue + # a playbook that only runs `--mode id` or `--mode verify` starts no GPU server; the prove modes do + if grep -qE 'igneum-prove-host' "$f" && grep -qE -- '--mode (compressed|chain|aggregate|block|shard|all)\b' "$f" && grep -qE -- '-u root' "$f"; then + if ! grep -qE 'pkill -f sp1-gpu-server' "$f"; then echo "prover-socket: $f runs the prover host as root without killing sp1-gpu-server"; fail=1; fi + if ! grep -qE 'rm -f /tmp/sp1-cuda-' "$f"; then echo "prover-socket: $f runs the prover host as root without unlinking /tmp/sp1-cuda-*.sock"; fail=1; fi + fi +done < <(list_files "$@") +[ "$fail" = 0 ] && echo "prover-socket: every root prover playbook kills the GPU server and unlinks its socket" +exit $fail