diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1da843580..95d0c9661 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,8 @@ jobs: run: bash tools/ci/copied-sources-check.sh - name: bash bodies in PowerShell job scripts pass bash -n, the lost-quote class (self-test first, then the tree) run: bash tools/ci/bash-body-check.sh --self-test && bash tools/ci/bash-body-check.sh + - name: run jobs test their fetched kit before use, the wiped-jobs-folder class (self-test first, then the tree) + run: bash tools/ci/kit-path-check.sh --self-test && bash tools/ci/kit-path-check.sh - name: pinned guest programs match their manifest and are built only by pin-guests.sh run: bash tools/ci/pinned-guests-check.sh - name: no secret file names and no 64-hex secrets in the tree (self-test first, then the tree) diff --git a/packaging/README-ship.md b/packaging/README-ship.md index 0d62e74a5..ceb69dae2 100644 --- a/packaging/README-ship.md +++ b/packaging/README-ship.md @@ -121,3 +121,7 @@ file and run), unescapes it the way PowerShell would, and runs `bash -n` on it. never passes CI before it runs, so `packaging/ota/publish-jobs.sh add --kind run` runs the same check (and `tools/ci/prover-socket-check.sh`, the root-socket class; `bash -n` for a `.sh` script) on the script before anything is signed, and refuses the publish with the check's output. `packaging/ota/test-publish-jobs.sh` proves the refusals. +The wiped-jobs-folder class (5 October 2026, 21:49Z): an app install clears the jobs folder, so a run job that uses a kit +fetched by an earlier fetch job tests the kit is there (Test-Path on the kit root or any file under it) before its first +use, and the fetch is republished under a new id after any app update. `tools/ci/kit-path-check.sh` fails CI and the +publish on a kit path used before that check. diff --git a/packaging/ota/publish-jobs.sh b/packaging/ota/publish-jobs.sh index 8d444d5ff..1d3c9d7d0 100755 --- a/packaging/ota/publish-jobs.sh +++ b/packaging/ota/publish-jobs.sh @@ -261,14 +261,17 @@ if [ "$CMD" = add ]; then # anything is signed. A failure refuses the publish with the check's output; a missing check refuses it too. # PowerShell: every inline bash body must parse (tools/ci/bash-body-check.sh, the lost-quote class; a body it # cannot read fails, never skips). Bash: the script itself must parse. Both: a root prover run kills the GPU - # server and unlinks its socket (tools/ci/prover-socket-check.sh, the root-socket class). A check file that is - # missing from this tree refuses too (bash exits 127 with the reason), so no job goes out unchecked. + # server and unlinks its socket (tools/ci/prover-socket-check.sh, the root-socket class), and a fetched kit under + # the jobs folder is tested before its first use (tools/ci/kit-path-check.sh, the wiped-jobs-folder class). A + # check file that is missing from this tree refuses too (bash exits 127 with the reason), so no job goes out + # unchecked. if [ "$SHELL_KIND" = powershell ]; then out="$(bash "$ROOT/tools/ci/bash-body-check.sh" "$SCRIPT" 2>&1)" || { echo "run: bash-body-check.sh refuses $SCRIPT:" >&2; printf '%s\n' "$out" >&2; exit 1; } else out="$(bash -n "$SCRIPT" 2>&1)" || { echo "run: bash -n refuses $SCRIPT (the lost-quote class):" >&2; printf '%s\n' "$out" >&2; exit 1; } fi out="$(bash "$ROOT/tools/ci/prover-socket-check.sh" "$SCRIPT" 2>&1)" || { echo "run: prover-socket-check.sh refuses $SCRIPT:" >&2; printf '%s\n' "$out" >&2; exit 1; } + out="$(bash "$ROOT/tools/ci/kit-path-check.sh" "$SCRIPT" 2>&1)" || { echo "run: kit-path-check.sh refuses $SCRIPT:" >&2; printf '%s\n' "$out" >&2; exit 1; } PARAMS="$(python3 -c 'import json,sys; print(json.dumps({"script": open(sys.argv[1]).read(), "shell": sys.argv[2], "elevated": sys.argv[3]=="1", "stop_miners_first": sys.argv[4]=="1", **({"timeout_minutes": int(sys.argv[5])} if sys.argv[5] else {})}))' "$SCRIPT" "$SHELL_KIND" "$ELEVATED" "$STOP_MINERS" "$TIMEOUT_MIN")" [ -n "$TITLE" ] || TITLE="run $(basename "$SCRIPT")" ;; diff --git a/packaging/ota/test-publish-jobs.sh b/packaging/ota/test-publish-jobs.sh index c5f8b545e..3f149f8fd 100755 --- a/packaging/ota/test-publish-jobs.sh +++ b/packaging/ota/test-publish-jobs.sh @@ -9,7 +9,8 @@ # a new envelope; an envelope made by hand from the FIRST file and the SECOND signature (the stale pair an edge can # serve across a deploy) is REFUSED by the signer with the words the app logs; a tampered inner byte is refused; # `sign` rewrites all three consistently; a `run` script that fails a class check (a lost quote in an inline bash -# body, a body the check cannot read, a bash script that does not parse, a root prover run without the socket cleanup) +# body, a body the check cannot read, a bash script that does not parse, a root prover run without the socket cleanup, +# a fetched kit used before a presence check) # is REFUSED before anything is signed and the envelope is left as it was; and the real OTA key is the key the app # embeds. # @@ -112,6 +113,9 @@ expect_fail "a bash script with a lost quote" "$PUBLISH" add --kind run --target printf '& wsl.exe -d Ubuntu-24.04 -u root -- bash /mnt/c/prove.sh\n# igneum-prove-host --mode shard --shard 0\n' > "$T/root-prover.ps1" expect_fail "a root prover script without the socket cleanup" "$PUBLISH" add --kind run --target 1ccfe586 --script "$T/root-prover.ps1" --id test-root-socket --dest "$D" --base-url https://example.invalid/dl/t grep -q "prover-socket" "$T/out" && ok "refused by the socket check" || bad "another reason: $(tail -1 "$T/out")" +printf '$jobs = Split-Path $env:IGNEUM_JOB_DIR\n$exe = Join-Path (Join-Path $jobs "fetch-kit-1") "worker.exe"\n& $exe --list\n' > "$T/kit-unchecked.ps1" +expect_fail "a run script that uses a fetched kit before a presence check" "$PUBLISH" add --kind run --target 1ccfe586 --script "$T/kit-unchecked.ps1" --id test-kit-unchecked --dest "$D" --base-url https://example.invalid/dl/t +grep -q "kit path used before a presence check" "$T/out" && ok "refused by the kit-path check" || bad "another reason: $(tail -1 "$T/out")" cmp -s "$T/before.signed" "$D/igneum-jobs.signed.json" && ok "a refused publish leaves the envelope untouched" || bad "a refused publish changed the envelope" echo "== the key" diff --git a/tools/ci/fixtures/kit-path-ok.ps1 b/tools/ci/fixtures/kit-path-ok.ps1 new file mode 100644 index 000000000..14904f369 --- /dev/null +++ b/tools/ci/fixtures/kit-path-ok.ps1 @@ -0,0 +1,25 @@ +# Fixture for tools/ci/kit-path-check.sh --self-test: both ways a run job reaches a fetched kit under the app's jobs +# folder, each checked for presence before its first use. Must pass (2 kit paths). Never run; a stand-in for a job. +$ErrorActionPreference = 'Continue' +# the job's own folder always exists (the app made it for this run): not a kit path +$job = $env:IGNEUM_JOB_DIR +$jobFile = Join-Path $env:IGNEUM_JOB_DIR 'jobs.txt' +# 1. the race-5090.ps1 shape: the jobs folder is the parent of this job's folder, the kit is a sibling job's folder +$jobs = Split-Path $env:IGNEUM_JOB_DIR +$fetched = Join-Path $jobs 'fetch-race-worker-20261004' +$exe = Join-Path $fetched 'igneum-worker-cuda.exe' +if (-not (Test-Path $exe)) { Write-Output "RESULT race worker missing at $exe (the fetch job runs first)"; exit 2 } +& $exe --race --race-rounds 3 2>&1 | ForEach-Object { "RESULT $_" } +# 2. the amd-card-test.ps1 shape: jobs\\kit under the app folder, a wait loop then the check; a sibling file +# under the same kit (pack-a) is covered by the check on the worker +$KitJob = 'amd-kit-1' +$kit = Join-Path $env:IGNEUM_APP_DIR ("jobs\" + $KitJob + "\kit") +$worker = Join-Path $kit 'igneum-worker-opencl.exe' +$tele = Join-Path $kit 'igneum-gpu-telemetry.exe' +$pack = Join-Path $kit 'pack-a' +$deadline = (Get-Date).AddMinutes(2) +while (-not ((Test-Path $worker) -and (Test-Path $tele)) -and (Get-Date) -lt $deadline) { Start-Sleep -Seconds 5 } +if (-not (Test-Path $worker)) { "RESULT no worker at $worker"; exit 1 } +$list = & $worker --list 2>&1 +$serve = Start-Process -FilePath $worker -ArgumentList '--serve','--pack',"`"$pack`"" -RedirectStandardInput $jobFile -NoNewWindow -PassThru +exit 0 diff --git a/tools/ci/fixtures/kit-path-unchecked.ps1 b/tools/ci/fixtures/kit-path-unchecked.ps1 new file mode 100644 index 000000000..410235371 --- /dev/null +++ b/tools/ci/fixtures/kit-path-unchecked.ps1 @@ -0,0 +1,14 @@ +# Fixture for tools/ci/kit-path-check.sh --self-test: the wiped-jobs-folder class (5 October 2026, 21:49Z: the 0.3.10 +# install cleared the jobs folder and the AMD kit job failed in seconds). Line 9 runs the worker before its check at +# line 10; line 13 runs a literal jobs\ path that is never checked. Must fail twice. Never run; a stand-in for a job. +$ErrorActionPreference = 'Continue' +# 1. the worker is run first and tested after +$jobs = Split-Path $env:IGNEUM_JOB_DIR +$fetched = Join-Path $jobs 'fetch-race-worker-20261004' +$exe = Join-Path $fetched 'igneum-worker-cuda.exe' +& $exe --race 2>&1 | ForEach-Object { "RESULT $_" } +if (-not (Test-Path $exe)) { Write-Output "RESULT race worker missing at $exe"; exit 2 } +# 2. an inline literal under the jobs folder, never checked +$KitJob = 'amd-kit-1' +& "$env:IGNEUM_APP_DIR\jobs\$KitJob\kit\igneum-worker-opencl.exe" --list 2>&1 +exit 0 diff --git a/tools/ci/kit-path-check.sh b/tools/ci/kit-path-check.sh new file mode 100755 index 000000000..693791f8e --- /dev/null +++ b/tools/ci/kit-path-check.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# The wiped-jobs-folder class (5 October 2026, 21:49Z, bench-log 39f02ff): the app's install clears the jobs folder on +# a PC, so a run job whose kit was fetched by an earlier fetch job finds nothing after an update and fails in seconds. +# Rule: a run playbook that reaches a path under the app's jobs folder other than its own +# (`$jobs = Split-Path $env:IGNEUM_JOB_DIR` then `Join-Path $jobs ''`, the race-5090.ps1 shape; or a literal +# `jobs\\...`, the amd-card-test.ps1 shape) tests that the kit is there (Test-Path, [IO.File]::Exists, +# [IO.Directory]::Exists, Get-Item or Get-ChildItem with -ErrorAction) before its first use, and republishes the fetch +# after any app update. A check on the kit's root or on anything under it covers the whole kit. This check reads every +# *.ps1 under relay/playbooks/ and tools/ and fails on a kit path used before a presence check. The job's own folder +# ($env:IGNEUM_JOB_DIR, made by the app for the run) is not a kit path. +# +# Usage: tools/ci/kit-path-check.sh # the tree (tools/ci/fixtures/ left out) +# tools/ci/kit-path-check.sh ... # named files (the jobs publisher runs it on the script it publishes) +# tools/ci/kit-path-check.sh --self-test # must fire on the unchecked fixture and stay quiet on the correct one +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../.." && pwd)" + +check_files() { + python3 - "$@" <<'PY' +import re, sys + +SEED_LIT = re.compile(r'jobs\\') # a literal path under the jobs folder +SEED_FOLDER = re.compile(r'Split-Path\s+\$env:IGNEUM_JOB_DIR') # the jobs folder itself: the parent of this job's folder +CHECK = re.compile(r'Test-Path|\[IO\.(File|Directory)\]::Exists|Get-(Item|ChildItem)\b[^|]*-ErrorAction') +ASSIGN = re.compile(r'^\s*\$([A-Za-z_][A-Za-z0-9_]*)\s*=(?!=)\s*(.*)$') +# a right-hand side that only builds a path (the kit var it names is then a path, not a use of one) +DERIV = re.compile(r'^(Join-Path\b|Split-Path\b|\(|"|\'|\[IO\.Path\]::Combine|\$env:|\$[A-Za-z_][A-Za-z0-9_]*\s*(\+|\.(TrimEnd|Replace)|$))') +TEXT_ONLY = re.compile(r'^\s*(Write-Output|Write-Host|Write-Verbose|Say|Log)\b|^\s*"') +VAR = re.compile(r'\$([A-Za-z_][A-Za-z0-9_]*)\b(?!:)') +MSG = 'kit path used before a presence check: republish the fetch after any app update' + +rc = 0 +files = sys.argv[1:] +with_kits = 0 +for path in files: + try: + lines = open(path, encoding='utf-8', errors='replace').read().split('\n') + except OSError as e: + print('FAIL %s: %s' % (path, e)); rc = 1; continue + folder = set() # vars that are the jobs folder (always there) + root_of = {} # kit var -> its root var + root_line = {} # root var -> the line it is defined on + checked = set() # roots with a presence check so far + reported = set() + inline_checked = False + fails = [] + for ln, raw in enumerate(lines, 1): + s = raw.strip() + if not s or s.startswith('#'): continue + refs = set(VAR.findall(raw)) + kit_refs = refs & set(root_of) + m = ASSIGN.match(raw) + if m: + name, rhs = m.group(1), m.group(2).strip() + if SEED_FOLDER.search(rhs) and not SEED_LIT.search(rhs): + folder.add(name); continue + deriv = bool(DERIV.match(rhs)) + if deriv and (SEED_LIT.search(rhs) or (refs & folder)): + root_of[name] = name; root_line[name] = ln; continue + if deriv and kit_refs: + root_of[name] = root_of[sorted(kit_refs)[0]]; continue + if CHECK.search(raw): + for v in kit_refs: checked.add(root_of[v]) + if SEED_LIT.search(raw): inline_checked = True + continue + for v in sorted(kit_refs): + r = root_of[v] + if r in checked or r in reported: continue + reported.add(r) + fails.append('FAIL %s:%d %s ($%s, kit path $%s defined at line %d)' % (path, ln, MSG, v, r, root_line[r])) + if SEED_LIT.search(raw) and not inline_checked and not TEXT_ONLY.match(raw): + inline_checked = True + fails.append('FAIL %s:%d %s (a literal jobs\\ path in a command, never checked)' % (path, ln, MSG)) + if fails: + rc = 1 + for f in fails: print(f) + elif root_of or inline_checked: + with_kits += 1 + print('ok %s (%d kit path%s, checked before use)' % (path, len(root_line), '' if len(root_line) == 1 else 's')) +print('kit-path-check: %d files, %d with a fetched kit, %s' % (len(files), with_kits, 'all checked before use' if rc == 0 else 'FAILURES above')) +sys.exit(rc) +PY +} + +if [ "${1:-}" = "--self-test" ]; then + F="$HERE/fixtures" + ok_out="$(check_files "$F/kit-path-ok.ps1" 2>&1)"; ok_rc=$? + bad_out="$(check_files "$F/kit-path-unchecked.ps1" 2>&1)"; bad_rc=$? + echo "self-test correct fixture: rc $ok_rc"; printf '%s\n' "$ok_out" | sed 's/^/ /' + echo "self-test unchecked fixture: rc $bad_rc"; printf '%s\n' "$bad_out" | sed 's/^/ /' + [ $ok_rc -eq 0 ] || { echo "SELF-TEST FAILED: the correct fixture was flagged"; exit 1; } + printf '%s\n' "$ok_out" | grep -q '^ok .*2 kit paths' || { echo "SELF-TEST FAILED: the correct fixture holds 2 kit paths, a different number was found"; exit 1; } + [ $bad_rc -ne 0 ] || { echo "SELF-TEST FAILED: the unchecked fixture passed"; exit 1; } + for want in 'kit-path-unchecked.ps1:9 ' 'kit-path-unchecked.ps1:13 '; do + printf '%s\n' "$bad_out" | grep -q "^FAIL .*$want.*republish the fetch after any app update" || { echo "SELF-TEST FAILED: the unchecked use at $want was not reported"; exit 1; } + done + [ "$(printf '%s\n' "$bad_out" | grep -c '^FAIL ')" -eq 2 ] || { echo "SELF-TEST FAILED: 2 failures expected"; exit 1; } + echo "self-test passed: the unchecked kit paths fail, the checked ones pass" + exit 0 +fi + +if [ $# -gt 0 ]; then + check_files "$@"; exit $? +fi +cd "$REPO" +files=$(git ls-files 'relay/playbooks/**' 'tools/**' | grep -E '\.ps1$' | grep -v '^tools/ci/fixtures/' || true) +if [ -z "$files" ]; then echo "kit-path-check: no .ps1 files under relay/playbooks/ or tools/"; exit 0; fi +# shellcheck disable=SC2086 +check_files $files