From a5533177d3d60a38a2dde1c1b68521f60f06d9cd Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:38:41 +0000 Subject: [PATCH 1/3] CI: bash bodies in PowerShell jobs pass bash -n (the lost-quote class) Twice on 5 October 2026 a PowerShell job script carried a bash body inside a string, a quote was lost on the way through PowerShell, and bash refused the body: pc1-cpu-prove.ps1 (first version) reported exit 0 having done nothing, the 0.3.10 installer job failed in 4 s. tools/amd-prove/check-job-bash.sh covered only its own here-string. tools/ci/bash-body-check.sh reads every *.ps1 under relay/playbooks/ and tools/, finds each bash body however it is handed over (bash -c "...", bash -lc '...', bash -c $var, a + concatenation in parentheses, the Start-Process argument list, a here-string written to a file that is later run with bash), unescapes it the way PowerShell would (backtick escapes and "" in double-quoted strings, '' in single-quoted strings, here-strings verbatim; $var left as-is, a $(...) subexpression replaced by ${PS_SUBEXPR}), and runs bash -n on it. One line per body with the file line of the error. A body it sees but cannot read is "unextractable body" and fails too: a skip would be a hole in the class check. bash 3.2 compatible; python3 for the extractor. --self-test runs three fixtures under tools/ci/fixtures/: the correct shapes (8 bodies, must pass), the lost quotes (the awk apostrophe, a dropped closing quote in a literal and in a variable; must fail with the line), and three unreadable bodies (must fail). Wired into ci.yml next to the copied-sources check, self-test first. The current tree: 7 inline bodies in 3 playbooks, all parse. packaging/README-ship.md: the job-script rule (body to a file, bash ). Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 2 + packaging/README-ship.md | 12 + tools/ci/bash-body-check.sh | 402 ++++++++++++++++++ tools/ci/fixtures/bash-body-lost-quote.ps1 | 24 ++ tools/ci/fixtures/bash-body-ok.ps1 | 53 +++ tools/ci/fixtures/bash-body-unextractable.ps1 | 14 + 6 files changed, 507 insertions(+) create mode 100755 tools/ci/bash-body-check.sh create mode 100644 tools/ci/fixtures/bash-body-lost-quote.ps1 create mode 100644 tools/ci/fixtures/bash-body-ok.ps1 create mode 100644 tools/ci/fixtures/bash-body-unextractable.ps1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9879a98b0..1da843580 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,8 @@ jobs: run: bash tools/ci/no-conflict-markers.sh - name: copied sources are re-stamped before a build run: bash tools/ci/copied-sources-check.sh + - name: bash bodies in PowerShell job scripts pass bash -n, the lost-quote class (self-test first, then the tree) + run: bash tools/ci/bash-body-check.sh --self-test && bash tools/ci/bash-body-check.sh - name: pinned guest programs match their manifest and are built only by pin-guests.sh run: bash tools/ci/pinned-guests-check.sh - name: no secret file names and no 64-hex secrets in the tree (self-test first, then the tree) diff --git a/packaging/README-ship.md b/packaging/README-ship.md index f9498b751..3da24790e 100644 --- a/packaging/README-ship.md +++ b/packaging/README-ship.md @@ -106,3 +106,15 @@ from the repository root (the project's root directory is `site`), but the norma (`igneum-relay`) and the downloads folder (`igneum-dl`) are the other two projects in the `igneum` team; both deploy by CLI from their own folders (`relay/README.md`, `packaging/ota/README.md`). CLAUDE.md still says the site sits in the [other-business] team and deploys with `--scope [other-business]` from `site/`: that was true on 3 October and is not now. + +## Job scripts (5 October 2026, the lost-quote class) + +A bash body in a PowerShell job (`relay/playbooks/*.ps1`, `tools/**/*.ps1`) is written to a file and run with +`bash `, never inline. That is the 0.3.6 cut's rule. Twice on 5 October a body travelled inside a PowerShell +string, a quote was lost on the way through PowerShell, bash refused the whole body, and the job either reported exit 0 +having done nothing (`tools/amd-prove/pc1-cpu-prove.ps1`, first version: an apostrophe inside a single-quoted awk +program) or failed in 4 s (the 0.3.10 installer job). The file shape keeps the body readable by `bash -n` before it runs; +`pc1-cpu-prove.ps1` does that in the job itself. `tools/ci/bash-body-check.sh` fails CI on an inline body that does not +parse: it finds every body handed to bash (`bash -c "..."`, `bash -lc $var`, a `+` concatenation, a here-string written to a +file and run), unescapes it the way PowerShell would, and runs `bash -n` on it. A body it sees but cannot read fails too. +`--self-test` shows it firing on the fixtures under `tools/ci/fixtures/`. diff --git a/tools/ci/bash-body-check.sh b/tools/ci/bash-body-check.sh new file mode 100755 index 000000000..385b3366a --- /dev/null +++ b/tools/ci/bash-body-check.sh @@ -0,0 +1,402 @@ +#!/usr/bin/env bash +# The lost-quote class (5 October 2026, twice in one night): a PowerShell job script carries a bash body inside a +# string, a quote is lost on the way through PowerShell, bash refuses the whole body, and the job either reports +# exit 0 having done nothing (tools/amd-prove/pc1-cpu-prove.ps1, first version: an apostrophe inside a single-quoted +# awk program) or fails in 4 s (the 0.3.10 installer job). Rule (the 0.3.6 cut): a bash body in a PowerShell job is +# written to a file and run with `bash `, never inline. This check reads every *.ps1 under relay/playbooks/ and +# tools/, finds each bash body however it is handed over (`wsl ... bash -c "..."`, `bash -lc '...'`, `bash -c $var`, +# a `+` concatenation in parentheses, a here-string written to a file that is later run with bash), unescapes it the +# way PowerShell would (backtick escapes and "" in double-quoted strings, '' in single-quoted strings, here-strings +# verbatim; `$var` interpolation left as-is, a PowerShell `$(...)` subexpression replaced by `${PS_SUBEXPR}`; in a `+` +# concatenation a variable becomes `${name}`), and runs `bash -n` on it. One line per body: file, line, ok or the +# bash -n error. A body the extractor sees but cannot read (`bash -c` with an argument shape it does not parse, or a +# variable with no literal assignment above) is "unextractable body" and FAILS: a skip would be a hole in the check. +# Exit 1 on any failure. +# +# Usage: tools/ci/bash-body-check.sh # the tree (tools/ci/fixtures/ left out) +# tools/ci/bash-body-check.sh ... # named files +# tools/ci/bash-body-check.sh --self-test # must fire on the lost-quote and unextractable fixtures under +# # tools/ci/fixtures/ and stay quiet on the correct one +# Needs python3 (the extractor) and bash (the parser). Runs on this Mac's bash 3.2. +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../.." && pwd)" + +# ---- the extractor: python3 reads the ps1 files, writes one body per file under $1, prints an index to stdout ---- +# index line: idx file line kind offset status detail +# offset: file line = offset + body line (bash -n reports body lines); status: body | unextractable +extract() { + python3 - "$@" <<'PY' +import sys, os, re + +out_dir = sys.argv[1] +files = sys.argv[2:] +SUBEXPR = '${PS_SUBEXPR}' +BT = chr(96) +BT_MAP = {'n': '\n', 't': '\t', 'r': '\r', '0': '\0', 'a': '\a', 'b': '\b', 'f': '\f', 'v': '\v', 'e': '\x1b'} + +def skip_subexpr(s, i): + """s[i] == '(' of a PowerShell $( ... ); returns the index after the matching ')'.""" + depth = 0 + n = len(s) + while i < n: + c = s[i] + if c == '(': + depth += 1; i += 1 + elif c == ')': + depth -= 1; i += 1 + if depth == 0: return i + elif c == "'": + i += 1 + while i < n: + if s[i] == "'": + if i + 1 < n and s[i+1] == "'": i += 2; continue + i += 1; break + i += 1 + elif c == '"': + i = skip_dq(s, i + 1) + else: + i += 1 + return n + +def skip_dq(s, i): + """s[i] is the first char inside a double-quoted string; returns the index after the closing quote.""" + n = len(s) + while i < n: + c = s[i] + if c == BT: i += 2; continue + if c == '"': + if i + 1 < n and s[i+1] == '"': i += 2; continue + return i + 1 + if c == '$' and i + 1 < n and s[i+1] == '(': + i = skip_subexpr(s, i + 1); continue + i += 1 + return n + +def decode_dq(raw, here): + """A double-quoted string or here-string the way PowerShell reads it, with $var left as-is.""" + o = []; i = 0; n = len(raw) + while i < n: + c = raw[i] + if c == BT and i + 1 < n: + o.append(BT_MAP.get(raw[i+1], raw[i+1])); i += 2; continue + if c == '"' and not here and i + 1 < n and raw[i+1] == '"': + o.append('"'); i += 2; continue + if c == '$' and i + 1 < n and raw[i+1] == '(': + o.append(SUBEXPR); i = skip_subexpr(raw, i + 1); continue + o.append(c); i += 1 + return ''.join(o) + +def tokenize(text): + """PowerShell tokens: ('str', value, line, sub) | ('var', name, line, None) | ('word', text, line, None) | + ('op', char, line, None) | ('nl', '', line, None). Comments are dropped, a trailing backtick joins lines.""" + toks = []; i = 0; n = len(text); line = 1 + SPECIAL = set(' \t\r\n()[]{},;|=+\'"') + while i < n: + c = text[i] + if c == '\n': + toks.append(('nl', '', line, None)); line += 1; i += 1; continue + if c in ' \t\r': i += 1; continue + if c == BT and i + 1 < n and text[i+1] in '\r\n': + i += 1 + while i < n and text[i] == '\r': i += 1 + if i < n and text[i] == '\n': line += 1; i += 1 + continue + if text.startswith('<#', i): + j = text.find('#>', i + 2) + if j < 0: j = n + line += text.count('\n', i, j); i = j + 2; continue + if c == '#': + j = text.find('\n', i) + i = n if j < 0 else j; continue + if text.startswith('@"', i) or text.startswith("@'", i): + q = text[i+1] + j = i + 2 + while j < n and text[j] == '\r': j += 1 + if j < n and text[j] == '\n': + start = j + 1 + m = re.compile(r'(?:^|\n)' + re.escape(q + '@')).search(text, start - 1) + if m: + end = m.start() if text[m.start()] == '\n' else m.start() + raw = text[start:end] + sub = 'hdq' if q == '"' else 'hsq' + val = decode_dq(raw, True) if q == '"' else raw + toks.append(('str', val, line, sub)) + line += text.count('\n', i, m.end()); i = m.end(); continue + # not a here-string after all: fall through as an operator + toks.append(('op', '@', line, None)); i += 1; continue + if c == "'": + j = i + 1; o = [] + while j < n: + if text[j] == "'": + if j + 1 < n and text[j+1] == "'": o.append("'"); j += 2; continue + break + o.append(text[j]); j += 1 + toks.append(('str', ''.join(o), line, 'sq')) + line += text.count('\n', i, j); i = j + 1; continue + if c == '"': + j = skip_dq(text, i + 1) + raw = text[i+1:j-1] if j <= n and text[j-1] == '"' else text[i+1:j] + toks.append(('str', decode_dq(raw, False), line, 'dq')) + line += text.count('\n', i, j); i = j; continue + if c == '$': + if i + 1 < n and text[i+1] == '(': + j = skip_subexpr(text, i + 1) + toks.append(('subexpr', text[i:j], line, None)) + line += text.count('\n', i, j); i = j; continue + if i + 1 < n and text[i+1] == '{': + j = text.find('}', i) + if j < 0: j = n - 1 + toks.append(('var', text[i+2:j], line, None)); i = j + 1; continue + m = re.compile(r'\$([A-Za-z_][A-Za-z0-9_]*(?::[A-Za-z_][A-Za-z0-9_]*)?)').match(text, i) + if m: + toks.append(('var', m.group(1), line, None)); i = m.end(); continue + toks.append(('op', '$', line, None)); i += 1; continue + if c in SPECIAL: + toks.append(('op', c, line, None)); i += 1; continue + j = i + while j < n and text[j] not in SPECIAL: j += 1 + toks.append(('word', text[i:j], line, None)); i = j + toks.append(('nl', '', line, None)) + return toks + +def is_bash_word(t): + v = t[1] + return t[0] in ('word', 'str') and (v in ('bash', 'bash.exe') or v.endswith('/bin/bash') or v.endswith('\\bash.exe')) + +STOP_OPS = set('|;{})') + +def parse_concat(toks, i, assigns, stop_at_nl): + """Reads str|var (+ str|var)* from toks[i]; returns (value, kind, next_index, problem). kind: lit | here | concat.""" + parts = []; kinds = []; problem = None + want_operand = True + while i < len(toks): + t = toks[i] + if t[0] == 'nl': + if stop_at_nl and not want_operand: break + if want_operand and parts: i += 1; continue # a newline after '+' continues the expression + if not parts: break + i += 1; continue + if want_operand: + if t[0] == 'str': + parts.append(t[1]); kinds.append('here' if t[3] in ('hdq', 'hsq') else 'lit'); want_operand = False; i += 1; continue + if t[0] == 'var': + parts.append('${' + t[1] + '}'); kinds.append('var'); want_operand = False; i += 1; continue + problem = 'expression token %r at line %d' % (t[1] or t[0], t[2]); break + else: + if t[0] == 'op' and t[1] == '+': want_operand = True; i += 1; continue + if t[0] == 'op' and t[1] in STOP_OPS: break + if t[0] == 'word' and t[1].startswith('-'): problem = 'operator %s at line %d' % (t[1], t[2]); break + if t[0] in ('word', 'var', 'str', 'subexpr', 'op'): problem = 'expression token %r at line %d' % (t[1] or t[0], t[2]); break + break + if want_operand and parts and not problem: problem = 'expression ends after +' + if not parts and not problem: problem = 'no string' + if problem: return None, None, i, problem + if len(parts) == 1: kind = kinds[0] + else: kind = 'concat' + return ''.join(parts), kind, i, None + +def collect_assignments(toks): + """$name = at statement start. {name: [(line, value|None, kind, problem)]}""" + a = {} + for i, t in enumerate(toks): + if t[0] != 'var': continue + prev = toks[i-1] if i > 0 else None + if prev is not None and not (prev[0] == 'nl' or (prev[0] == 'op' and prev[1] in ';{')): continue + if i + 1 >= len(toks) or toks[i+1] != ('op', '=', toks[i+1][2], None): continue + val, kind, _, problem = parse_concat(toks, i + 2, a, True) + a.setdefault(t[1], []).append((t[2], val, kind, problem)) + return a + +def lookup(assigns, name, line): + best = None + for (l, val, kind, problem) in assigns.get(name, []): + if l < line: best = (l, val, kind, problem) + return best + +def bash_args(toks, i): + """Arguments after a bash word: list of (token-or-expr, index). An expr is ('expr', value, line, kind, problem).""" + args = [] + while i < len(toks): + t = toks[i] + if t[0] == 'nl': break + if t[0] == 'op' and t[1] in STOP_OPS: break + if t[0] == 'op' and t[1] == ',': i += 1; continue + if t[0] == 'word' and re.match(r'^[0-9]*>', t[1]): break + if t[0] == 'op' and t[1] == '(': + val, kind, j, problem = parse_concat(toks, i + 1, None, False) + # skip to the matching ')' + depth = 1; k = i + 1 + while k < len(toks) and depth > 0: + if toks[k][0] == 'op' and toks[k][1] == '(': depth += 1 + elif toks[k][0] == 'op' and toks[k][1] == ')': depth -= 1 + k += 1 + if problem is None and j < k - 1: problem = 'more than strings and + inside the parentheses' + names = set(x[1] for x in toks[i+1:k] if x[0] == 'var') + args.append((('expr', val, t[2], kind, problem, names), i)); i = k; continue + if t[0] == 'op' and t[1] == ')': break + args.append((t, i)); i += 1 + return args + +def value_of(arg, assigns): + """(value, def_line, kind, problem) for an argument token.""" + t = arg + if t[0] == 'str': return t[1], t[2], ('here' if t[3] in ('hdq', 'hsq') else 'lit'), None + if t[0] == 'expr': return t[1], t[2], t[3], t[4] + if t[0] == 'var': + a = lookup(assigns, t[1], t[2]) + if a is None: return None, t[2], 'var', '$%s has no literal assignment above line %d' % (t[1], t[2]) + l, val, kind, problem = a + if problem: return None, l, 'var', '$%s = (line %d) is not a string: %s' % (t[1], l, problem) + return val, l, kind, None + if t[0] == 'subexpr': return None, t[2], 'subexpr', 'a $(...) subexpression' + return None, t[2], t[0], 'token %r' % t[1] + +def option_value(arg): + if arg[0] in ('word', 'str'): return arg[1] + return None + +def find_bodies(path, toks): + assigns = collect_assignments(toks) + bodies = [] # (line, kind, value, offset, problem, def_line) + seen = set() + inv = [] # (index, args) of every bash invocation + for i, t in enumerate(toks): + if not is_bash_word(t): continue + args = bash_args(toks, i + 1) + inv.append((i, args)) + want_body = False; body_done = False; skip_next = False + for (a, ai) in args: + if body_done: break + if skip_next: skip_next = False; continue + if not want_body: + ov = option_value(a) + if ov is not None and re.match(r'^-[A-Za-z]+$', ov): + if 'c' in ov: want_body = True + if ov[-1] in ('o', 'O'): skip_next = True + continue + if ov is not None and ov.startswith('--'): continue + if ov is not None and re.match(r'^\+[A-Za-z]+$', ov): + if ov[-1] == 'O': skip_next = True + continue + break # a script file: not an inline body + val, dline, kind, problem = value_of(a, assigns) + body_done = True + how = 'bash -c ' + ('$' + a[1] if a[0] == 'var' else ('(...)' if a[0] == 'expr' else 'literal')) + if problem: bodies.append((t[2], how, None, 0, problem, dline)) + else: + offset = dline if kind == 'here' else dline - 1 + key = (dline, how) + if key in seen: continue + seen.add(key); bodies.append((t[2], how, val, offset, None, dline)) + if want_body and not body_done: + bodies.append((t[2], 'bash -c', None, 0, 'no body argument after -c', t[2])) + # here-strings (and other string variables) written to a file that is later run with bash + WRITERS = ('WriteAllText', 'WriteAllLines', 'Set-Content', 'Out-File', 'Add-Content') + lines_tokens = {} + for t in toks: lines_tokens.setdefault(t[2], []).append(t) + for name, lst in assigns.items(): + for (l, val, kind, problem) in lst: + if val is None: continue + for wl, lt in lines_tokens.items(): + if wl <= l: continue + if not any(x[0] == 'word' and any(w in x[1] for w in WRITERS) for x in lt): continue + if not any(x[0] == 'var' and x[1] == name for x in lt): continue + paths = set(x[1] for x in lt if x[0] == 'var' and x[1] != name) + sh_literal = any(x[0] == 'str' and x[1].endswith('.sh') for x in lt) + # a variable derived from the path ($w = WslPath $bashFile) names the same file + for al in sorted(lines_tokens): + if al <= wl: continue + alt = lines_tokens[al] + if len(alt) > 1 and alt[0][0] == 'var' and alt[1] == ('op', '=', al, None): + if any(x[0] == 'var' and x[1] in paths for x in alt[1:]): paths.add(alt[0][1]) + run = False + for (bi, args) in inv: + if toks[bi][2] <= wl: continue + for (a, ai) in args: + if a[0] == 'var' and a[1] in paths: run = True + if a[0] == 'expr' and (a[5] & paths): run = True + if a[0] == 'str' and a[1].endswith('.sh') and sh_literal: run = True + if run: break + if not run: continue + how = '$%s written at line %d and run with bash' % (name, wl) + key = (l, how) + if key in seen: continue + seen.add(key) + bodies.append((l, how, val, l if kind == 'here' else l - 1, None, l)) + break + bodies.sort(key=lambda b: (b[5], b[0])) + return bodies + +idx = 0 +for path in files: + try: + text = open(path, encoding='utf-8', errors='replace').read() + except OSError as e: + print('\t'.join(['0', path, '0', 'read', '0', 'unextractable', str(e)])); continue + toks = tokenize(text) + for (line, how, val, offset, problem, dline) in find_bodies(path, toks): + idx += 1 + if problem: + print('\t'.join([str(idx), path, str(line), how, '0', 'unextractable', problem])) + else: + with open(os.path.join(out_dir, '%d.body' % idx), 'w', encoding='utf-8') as f: f.write(val + '\n') + print('\t'.join([str(idx), path, str(dline), how, str(offset), 'body', ''])) +PY +} + +# ---- the check over a list of files; prints one line per body, returns 1 on any failure ---- +check_files() { + local T; T="$(mktemp -d)" + local rc=0 n=0 index + index="$T/index.tsv" + if ! extract "$T" "$@" > "$index"; then echo "FAIL extractor error"; rm -rf "$T"; return 1; fi + while IFS="$(printf '\t')" read -r idx file line how offset status detail; do + [ -n "$idx" ] || continue + n=$((n + 1)) + if [ "$status" = "unextractable" ]; then + echo "FAIL $file:$line unextractable body ($how): $detail"; rc=1; continue + fi + local err + if err="$(bash -n "$T/$idx.body" 2>&1)"; then + echo "ok $file:$line ($how)" + else + # bash prints ": line N: message"; map N to the file line + err="$(printf '%s\n' "$err" | sed -e "s#^$T/$idx.body: ##" | awk -v off="$offset" '{ if (match($0, /^line [0-9]+:/)) { n = substr($0, 6, RLENGTH - 6) + 0; sub(/^line [0-9]+:/, "file line " (n + off) " (body line " n "):") } print }' | tr '\n' ' ')" + echo "FAIL $file:$line ($how): $err"; rc=1 + fi + done < "$index" + rm -rf "$T" + echo "bash-body-check: $n bodies in $# files, $( [ $rc = 0 ] && echo 'all parse' || echo 'FAILURES above')" + return $rc +} + +if [ "${1:-}" = "--self-test" ]; then + F="$HERE/fixtures" + ok_out="$(check_files "$F/bash-body-ok.ps1" 2>&1)"; ok_rc=$? + bad_out="$(check_files "$F/bash-body-lost-quote.ps1" 2>&1)"; bad_rc=$? + unx_out="$(check_files "$F/bash-body-unextractable.ps1" 2>&1)"; unx_rc=$? + echo "self-test correct fixture: rc $ok_rc"; printf '%s\n' "$ok_out" | sed 's/^/ /' + echo "self-test lost-quote fixture: rc $bad_rc"; printf '%s\n' "$bad_out" | sed 's/^/ /' + echo "self-test unextractable fixture: rc $unx_rc"; printf '%s\n' "$unx_out" | sed 's/^/ /' + [ $ok_rc -eq 0 ] || { echo "SELF-TEST FAILED: the correct fixture was flagged"; exit 1; } + [ "$(printf '%s\n' "$ok_out" | grep -c '^ok ')" -eq 8 ] || { echo "SELF-TEST FAILED: the correct fixture has 8 bodies, a different number was found"; exit 1; } + [ $bad_rc -ne 0 ] || { echo "SELF-TEST FAILED: the lost-quote fixture passed"; exit 1; } + for want in 'bash-body-lost-quote.ps1:9 ' 'bash-body-lost-quote.ps1:18 ' 'bash-body-lost-quote.ps1:20 '; do + printf '%s\n' "$bad_out" | grep -q "^FAIL .*$want" || { echo "SELF-TEST FAILED: the lost quote at $want was not reported"; exit 1; } + done + [ "$(printf '%s\n' "$bad_out" | grep -c '^ok ')" -eq 1 ] || { echo "SELF-TEST FAILED: the one correct body in the lost-quote fixture was not reported ok"; exit 1; } + [ $unx_rc -ne 0 ] || { echo "SELF-TEST FAILED: the unextractable fixture passed"; exit 1; } + [ "$(printf '%s\n' "$unx_out" | grep -c 'unextractable body')" -eq 3 ] || { echo "SELF-TEST FAILED: 3 unextractable bodies expected"; exit 1; } + echo "self-test passed: bash -n fires on the lost quotes, the unreadable bodies fail, the correct bodies pass" + exit 0 +fi + +if [ $# -gt 0 ]; then + check_files "$@"; exit $? +fi +cd "$REPO" +files=$(git ls-files 'relay/playbooks/**' 'tools/**' | grep -E '\.ps1$' | grep -v '^tools/ci/fixtures/' || true) +if [ -z "$files" ]; then echo "bash-body-check: no .ps1 files under relay/playbooks/ or tools/"; exit 0; fi +# shellcheck disable=SC2086 +check_files $files diff --git a/tools/ci/fixtures/bash-body-lost-quote.ps1 b/tools/ci/fixtures/bash-body-lost-quote.ps1 new file mode 100644 index 000000000..c01e7373b --- /dev/null +++ b/tools/ci/fixtures/bash-body-lost-quote.ps1 @@ -0,0 +1,24 @@ +# Fixture for tools/ci/bash-body-check.sh --self-test: the lost-quote class of 5 October 2026. Three bodies do not +# parse (the here-string at line 9, the literal at line 18, the variable at line 20); one does (line 23). The check +# must report the three with the bash -n error and exit 1. Never run; a stand-in for a job script. +$ErrorActionPreference = 'Continue' +$distro = 'Ubuntu-24.04' +$job = Join-Path $env:TEMP 'igneum-fixture' +$jobW = '/mnt/c/igneum-fixture' +# 1. pc1-cpu-prove.ps1, first version: an apostrophe inside a single-quoted awk program ends the quote (body line 3) +$bash = @" +set -uo pipefail +for FX in a b; do + awk -v fx="`$FX" '/Maximum resident set size/ { print "RESULT " fx " the job's peak kb " `$NF }' "$jobW/results/`$FX-time.txt" +done +"@ +$bashFile = Join-Path $job 'body.sh' +[IO.File]::WriteAllText($bashFile, ($bash -replace "`r`n", "`n")) +& wsl.exe -d $distro -u root -- bash (WslPath $bashFile) 2>&1 | ForEach-Object { "$_" } +& wsl.exe -d $distro -- bash -c 'echo "installer started; ls /opt/igneum' 2>&1 | ForEach-Object { "$_" } +# 3. the 0.3.10 installer job: a closing quote dropped on the way through PowerShell +$cmd = "cd /opt/igneum && echo `"started && ./run.sh" +& wsl.exe -d $distro -- bash -lc $cmd 2>&1 | ForEach-Object { "$_" } +# 4. a correct body beside them, reported ok +& wsl.exe -d $distro -- bash -lc 'id; uname -r' 2>&1 | ForEach-Object { "$_" } +exit 0 diff --git a/tools/ci/fixtures/bash-body-ok.ps1 b/tools/ci/fixtures/bash-body-ok.ps1 new file mode 100644 index 000000000..ff7062d9a --- /dev/null +++ b/tools/ci/fixtures/bash-body-ok.ps1 @@ -0,0 +1,53 @@ +# Fixture for tools/ci/bash-body-check.sh --self-test: every way a job script hands bash a body, all of them +# correct. The check must report 8 bodies ok and exit 0. Never run; a stand-in for a job script. +$ErrorActionPreference = 'Continue' +$distro = 'Ubuntu-24.04' +$job = Join-Path $env:TEMP 'igneum-fixture' +$pkgW = '/mnt/c/igneum-fixture/pkg' +$FIXTURES = 'block-56-transfers-3shards block-78-increment' +function WslPath($p) { '/mnt/c' + ($p.Substring(2) -replace '\\', '/') } +# 1. a double-quoted here-string written to a file and run with bash (the pc1-cpu-prove.ps1 shape, fixed version) +$bash = @" +set -uo pipefail +export PATH="`$HOME/.cargo/bin:`$PATH"; [ -f "`$HOME/.cargo/env" ] && . "`$HOME/.cargo/env" +stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } +PKG='$pkgW'; DEST="`$HOME/igneum-prove-cpu" +echo "RESULT wsl `$(stamp) cores `$(nproc) ram_total_mb `$(free -m | awk '/Mem:/ {print `$2}')" +for FX in $FIXTURES; do + awk -v fx="`$FX" '/Maximum resident set size/ {r=`$NF} END {print "RESULT " fx " max_rss_kb=" r}' "`$DEST/`$FX-time.txt" + echo "RESULT `$FX prove end `$(stamp) exit `${PIPESTATUS[0]}" +done +"@ +$bashFile = Join-Path $job 'cpu-prove.sh' +[IO.File]::WriteAllText($bashFile, ($bash -replace "`r`n", "`n"), (New-Object System.Text.UTF8Encoding $false)) +$chk = (& wsl.exe -d $distro -u root -- bash -n (WslPath $bashFile) 2>&1); if ($LASTEXITCODE -ne 0) { "RESULT syntax FAILED: $chk"; exit 1 } +& wsl.exe -d $distro -u root -- bash (WslPath $bashFile) 2>&1 | ForEach-Object { "$_" } +# 2. a single-quoted literal after -lc (the wsl-setup.ps1 shape) +& wsl.exe -d $distro -- bash -lc 'id; uname -r; nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null || echo "no GPU visible inside WSL"' 2>&1 | ForEach-Object { "$_" } +# 3. a double-quoted string in a variable, interpolation left as-is (the prover-setup.ps1 shape) +$linuxDir = '/mnt/c/igneum-prove/igneum-prove-wsl2' +$cmd = "echo igneum | sudo -S -v 2>/dev/null; sudo -n true || echo 'sudo still asks for a password'; cd $linuxDir && bash ./setup-wsl.sh" +& wsl.exe -d $distro -u igneum -- bash -lc $cmd 2>&1 | ForEach-Object { "$_" } +# 4. single-quoted pieces joined with + over three lines (the wsl-setup.ps1 shape) +$mk = 'id igneum >/dev/null 2>&1 || (useradd -m -s /bin/bash igneum && echo igneum:igneum | chpasswd); ' + + 'echo "igneum ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/igneum && chmod 440 /etc/sudoers.d/igneum; ' + + 'printf "[user]\ndefault=igneum\n" > /etc/wsl.conf; id igneum' +& wsl.exe -d $distro -u root -- bash -c $mk 2>&1 | ForEach-Object { "$_" } +# 5. a concatenation in parentheses with a variable in the middle (the shard-test.ps1 shape) +& wsl.exe -d $distro -u igneum -- bash -lc ("sudo -n true 2>/dev/null || echo 'sudo asks for a password'; cd " + $linuxDir + " && bash ./setup-wsl.sh") 2>&1 | ForEach-Object { "$_" } +# 6. the Start-Process argument list, -c as its own quoted argument +$p = Start-Process -FilePath 'wsl.exe' -ArgumentList @('-d', $distro, '--', 'bash', '-c', 'pkill -f igneum-prove-host; pkill -f prove-shard.sh; true') -NoNewWindow -PassThru +# 7. a single-quoted here-string piped to Set-Content, the file then run by a derived path +$body = @' +set -euo pipefail +cd "$HOME/igneum" && ./igneum-miner --help | sed 's/^/RESULT help /' +'@ +$sh = Join-Path $job 'body.sh' +$body | Set-Content -Path $sh -Encoding ascii +$shW = WslPath $sh +& wsl.exe -d $distro -- bash $shW 2>&1 | ForEach-Object { "$_" } +# 8. -ec with a doubled quote and backtick escapes in a double-quoted string +& wsl.exe -d $distro -- bash -ec "echo ""started""; printf '%s`n' `"done`"" 2>&1 | ForEach-Object { "$_" } +# not bodies: a script file by path, and bash -n on a file +& wsl.exe -d $distro -u igneum -- bash /mnt/c/igneum-prove/igneum-prove-wsl2/prove-block.sh fixture core 2>&1 | ForEach-Object { "$_" } +exit 0 diff --git a/tools/ci/fixtures/bash-body-unextractable.ps1 b/tools/ci/fixtures/bash-body-unextractable.ps1 new file mode 100644 index 000000000..d6c8a9b0a --- /dev/null +++ b/tools/ci/fixtures/bash-body-unextractable.ps1 @@ -0,0 +1,14 @@ +# Fixture for tools/ci/bash-body-check.sh --self-test: three bash -c bodies the extractor sees but cannot read. +# Each must be reported as "unextractable body" and the check must exit 1: a skip would be a hole in the class check +# (the coordinator's rule, 5 October 2026). Never run; a stand-in for a job script. +$ErrorActionPreference = 'Continue' +$distro = 'Ubuntu-24.04' +# 1. the body comes from a command, not a string +$cmd = (Get-Content 'C:\igneum\body.txt') -join '; ' +& wsl.exe -d $distro -- bash -c $cmd 2>&1 | ForEach-Object { "$_" } +# 2. a format operator inside the parentheses +$tpl = 'cd {0} && ./run.sh' +& wsl.exe -d $distro -- bash -c ($tpl -f '/opt/igneum') 2>&1 | ForEach-Object { "$_" } +# 3. a variable with no assignment above +& wsl.exe -d $distro -- bash -lc $fromElsewhere 2>&1 | ForEach-Object { "$_" } +exit 0 From 2ed3dabe665c3d07a8997240bc4828552f182820 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:14:19 +0000 Subject: [PATCH 2/3] Jobs publisher: the class checks run on the script before it is signed (row C27) A PC job is published from a worktree by packaging/ota/publish-jobs.sh and never passes CI before it runs; tonight the root-socket fault came back from a job on a branch without the check. `add --kind run` now runs, on the script being published and before anything is signed: tools/ci/bash-body-check.sh for a PowerShell script (every inline bash body parses; a body it cannot read fails, never skips), `bash -n` for a .sh script, and tools/ci/prover-socket-check.sh for both (a root prover run kills sp1-gpu-server and unlinks its socket). A failure refuses the publish with the check's output; a missing check file refuses too. Kinds without a script (fetch, collect, restart, update-now, shard-benchmark, build) are untouched. tools/ci/prover-socket-check.sh is copied from proving-v1 (344cba8; master lacks it) with two additions: file arguments check those files only (the publisher's call), and an allow list for packaging/ota/test-publish-jobs.sh, which carries a known-bad root prover script on purpose. Its ci.yml step is left to proving-v1 to avoid a duplicate. packaging/ota/test-publish-jobs.sh: four refusals (a lost quote in a PowerShell bash body, an unreadable body, a .sh with a lost quote, a root prover script without the cleanup) and the envelope unchanged after a refusal. 32 passed, 0 failed on this Mac with the main checkout's signer. packaging/README-ship.md: the publish-time gate. Co-Authored-By: Claude Fable 5.1 --- packaging/README-ship.md | 5 ++++- packaging/ota/publish-jobs.sh | 13 +++++++++++++ packaging/ota/test-publish-jobs.sh | 20 +++++++++++++++++++- tools/ci/prover-socket-check.sh | 24 ++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) create mode 100755 tools/ci/prover-socket-check.sh diff --git a/packaging/README-ship.md b/packaging/README-ship.md index 3da24790e..0d62e74a5 100644 --- a/packaging/README-ship.md +++ b/packaging/README-ship.md @@ -117,4 +117,7 @@ program) or failed in 4 s (the 0.3.10 installer job). The file shape keeps the b `pc1-cpu-prove.ps1` does that in the job itself. `tools/ci/bash-body-check.sh` fails CI on an inline body that does not parse: it finds every body handed to bash (`bash -c "..."`, `bash -lc $var`, a `+` concatenation, a here-string written to a file and run), unescapes it the way PowerShell would, and runs `bash -n` on it. A body it sees but cannot read fails too. -`--self-test` shows it firing on the fixtures under `tools/ci/fixtures/`. +`--self-test` shows it firing on the fixtures under `tools/ci/fixtures/`. A job script is published from a worktree and +never passes CI before it runs, so `packaging/ota/publish-jobs.sh add --kind run` runs the same check (and +`tools/ci/prover-socket-check.sh`, the root-socket class; `bash -n` for a `.sh` script) on the script before anything is +signed, and refuses the publish with the check's output. `packaging/ota/test-publish-jobs.sh` proves the refusals. diff --git a/packaging/ota/publish-jobs.sh b/packaging/ota/publish-jobs.sh index 4e26f0ed9..8d444d5ff 100755 --- a/packaging/ota/publish-jobs.sh +++ b/packaging/ota/publish-jobs.sh @@ -256,6 +256,19 @@ if [ "$CMD" = add ]; then [ -n "$SCRIPT" ] && [ -f "$SCRIPT" ] || { echo "run: --script is required" >&2; exit 2; } [ -n "$SHELL_KIND" ] || { case "$SCRIPT" in *.sh) SHELL_KIND=bash ;; *) SHELL_KIND=powershell ;; esac; } [ -n "$PLATFORM" ] || { [ "$SHELL_KIND" = powershell ] && PLATFORM=windows || PLATFORM=any; } + # A job script is published from a worktree and never passes CI before it runs (5 October 2026: the root-socket + # fault came back from a branch without the check), so the class checks run here on the script itself, before + # anything is signed. A failure refuses the publish with the check's output; a missing check refuses it too. + # PowerShell: every inline bash body must parse (tools/ci/bash-body-check.sh, the lost-quote class; a body it + # cannot read fails, never skips). Bash: the script itself must parse. Both: a root prover run kills the GPU + # server and unlinks its socket (tools/ci/prover-socket-check.sh, the root-socket class). A check file that is + # missing from this tree refuses too (bash exits 127 with the reason), so no job goes out unchecked. + if [ "$SHELL_KIND" = powershell ]; then + out="$(bash "$ROOT/tools/ci/bash-body-check.sh" "$SCRIPT" 2>&1)" || { echo "run: bash-body-check.sh refuses $SCRIPT:" >&2; printf '%s\n' "$out" >&2; exit 1; } + else + out="$(bash -n "$SCRIPT" 2>&1)" || { echo "run: bash -n refuses $SCRIPT (the lost-quote class):" >&2; printf '%s\n' "$out" >&2; exit 1; } + fi + out="$(bash "$ROOT/tools/ci/prover-socket-check.sh" "$SCRIPT" 2>&1)" || { echo "run: prover-socket-check.sh refuses $SCRIPT:" >&2; printf '%s\n' "$out" >&2; exit 1; } PARAMS="$(python3 -c 'import json,sys; print(json.dumps({"script": open(sys.argv[1]).read(), "shell": sys.argv[2], "elevated": sys.argv[3]=="1", "stop_miners_first": sys.argv[4]=="1", **({"timeout_minutes": int(sys.argv[5])} if sys.argv[5] else {})}))' "$SCRIPT" "$SHELL_KIND" "$ELEVATED" "$STOP_MINERS" "$TIMEOUT_MIN")" [ -n "$TITLE" ] || TITLE="run $(basename "$SCRIPT")" ;; diff --git a/packaging/ota/test-publish-jobs.sh b/packaging/ota/test-publish-jobs.sh index d3fbc9686..c5f8b545e 100755 --- a/packaging/ota/test-publish-jobs.sh +++ b/packaging/ota/test-publish-jobs.sh @@ -8,7 +8,10 @@ # byte and its sig IS the plain signature; the signer reads the envelope back; a second `add` (a new publish) gives # a new envelope; an envelope made by hand from the FIRST file and the SECOND signature (the stale pair an edge can # serve across a deploy) is REFUSED by the signer with the words the app logs; a tampered inner byte is refused; -# `sign` rewrites all three consistently; and the real OTA key is the key the app embeds. +# `sign` rewrites all three consistently; a `run` script that fails a class check (a lost quote in an inline bash +# body, a body the check cannot read, a bash script that does not parse, a root prover run without the socket cleanup) +# is REFUSED before anything is signed and the envelope is left as it was; and the real OTA key is the key the app +# embeds. # # A check is trusted only once it has fired on a known-good and a known-bad case (standing rule, 4 October 2026), so # every negative case here must FAIL for the run to pass. Needs ~/.config/igneum/ota-signing-key (the publisher's own @@ -96,6 +99,21 @@ PY grep -q "^inner-identical True$" "$T/inner2.txt" && grep -q "^sig-identical True$" "$T/inner2.txt" && ok "after sign: the envelope still holds the plain file and its signature" || bad "after sign: the envelope and the pair differ" expect_ok "list reads the folder" "$PUBLISH" list --dest "$D" +echo "== the class checks refuse a bad script before anything is signed (5 October 2026: a job never passes CI first)" +cp "$D/igneum-jobs.signed.json" "$T/before.signed" +printf '& wsl.exe -d Ubuntu-24.04 -- bash -c '"'"'echo "started; ls /opt/igneum'"'"' 2>&1\n' > "$T/lost-quote.ps1" +expect_fail "a PowerShell script with a lost quote in its bash body" "$PUBLISH" add --kind run --target 1ccfe586 --script "$T/lost-quote.ps1" --id test-lost-quote --dest "$D" --base-url https://example.invalid/dl/t +grep -q "unexpected EOF while looking for matching" "$T/out" && ok "refused with the bash -n error" || bad "another reason: $(tail -1 "$T/out")" +printf '$cmd = (Get-Content body.txt) -join "; "\n& wsl.exe -- bash -c $cmd\n' > "$T/unreadable.ps1" +expect_fail "a PowerShell script whose bash body the check cannot read" "$PUBLISH" add --kind run --target 1ccfe586 --script "$T/unreadable.ps1" --id test-unreadable --dest "$D" --base-url https://example.invalid/dl/t +grep -q "unextractable body" "$T/out" && ok "refused as unextractable, not skipped" || bad "another reason: $(tail -1 "$T/out")" +printf 'echo "started; ls /opt/igneum\n' > "$T/lost-quote.sh" +expect_fail "a bash script with a lost quote" "$PUBLISH" add --kind run --target 1ccfe586 --script "$T/lost-quote.sh" --id test-lost-quote-sh --dest "$D" --base-url https://example.invalid/dl/t +printf '& wsl.exe -d Ubuntu-24.04 -u root -- bash /mnt/c/prove.sh\n# igneum-prove-host --mode shard --shard 0\n' > "$T/root-prover.ps1" +expect_fail "a root prover script without the socket cleanup" "$PUBLISH" add --kind run --target 1ccfe586 --script "$T/root-prover.ps1" --id test-root-socket --dest "$D" --base-url https://example.invalid/dl/t +grep -q "prover-socket" "$T/out" && ok "refused by the socket check" || bad "another reason: $(tail -1 "$T/out")" +cmp -s "$T/before.signed" "$D/igneum-jobs.signed.json" && ok "a refused publish leaves the envelope untouched" || bad "a refused publish changed the envelope" + echo "== the key" EMB="$("$SIGNER" embedded | head -1)" [ "$EMB" = "$(tr -d '[:space:]' < "$PUB")" ] && ok "the embedded key is the Mac's OTA public key" || bad "the embedded key is not $PUB" diff --git a/tools/ci/prover-socket-check.sh b/tools/ci/prover-socket-check.sh new file mode 100755 index 000000000..ddd10e561 --- /dev/null +++ b/tools/ci/prover-socket-check.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# The root-socket class (5 October 2026, 20:00Z): a PC 2 job ran igneum-prove-host as root inside WSL2, which started +# an sp1-gpu-server whose socket /tmp/sp1-cuda-0.sock stayed root-owned after the job; the live prover (the app's user) +# then failed every shard with "CudaClientError: Connect(PermissionDenied)" until the socket was gone. Rule: every +# playbook that runs the prover host as root on a shared card kills the server AND unlinks its socket (at the start +# and at the end), or runs as the app's user. This check fails CI when a script runs `igneum-prove-host` under a +# `-u root` WSL session without both lines. With file arguments it checks those files only (the jobs publisher runs it +# on the script being published, packaging/ota/publish-jobs.sh add --kind run; a PC job never passes CI before it runs). +set -euo pipefail +cd "$(dirname "$0")/../.." +fail=0 +# the publisher's test writes a known-bad root prover script on purpose, to prove the publish refuses it +ALLOW='^packaging/ota/test-publish-jobs\.sh$' +list_files() { if [ $# -gt 0 ]; then printf '%s\n' "$@"; else git ls-files 'tools/**' 'relay/playbooks/**' 'proving/**' 'packaging/**' | grep -E '\.(sh|ps1|mjs)$'; fi; } +while IFS= read -r f; do + [[ "$f" =~ $ALLOW ]] && continue + # a playbook that only runs `--mode id` or `--mode verify` starts no GPU server; the prove modes do + if grep -qE 'igneum-prove-host' "$f" && grep -qE -- '--mode (compressed|chain|aggregate|block|shard|all)\b' "$f" && grep -qE -- '-u root' "$f"; then + if ! grep -qE 'pkill -f sp1-gpu-server' "$f"; then echo "prover-socket: $f runs the prover host as root without killing sp1-gpu-server"; fail=1; fi + if ! grep -qE 'rm -f /tmp/sp1-cuda-' "$f"; then echo "prover-socket: $f runs the prover host as root without unlinking /tmp/sp1-cuda-*.sock"; fail=1; fi + fi +done < <(list_files "$@") +[ "$fail" = 0 ] && echo "prover-socket: every root prover playbook kills the GPU server and unlinks its socket" +exit $fail From b2e6d6159e458c4257ead09c93f348d323791d5a Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:37:29 +0000 Subject: [PATCH 3/3] CI: run jobs test their fetched kit before use (the wiped-jobs-folder class) The app's install clears the jobs folder on a PC, so a run job whose kit was fetched by an earlier fetch job finds nothing after an update and fails in seconds (5 October 2026, 21:49Z, the AMD kit; bench-log 4df339f). Rule: a run playbook that reaches a path under the jobs folder other than its own tests the kit is there before its first use, and the fetch is republished under a new id after any app update. tools/ci/kit-path-check.sh reads every *.ps1 under relay/playbooks/ and tools/. A kit root is a path derived from the jobs folder (`$jobs = Split-Path $env:IGNEUM_JOB_DIR` then `Join-Path $jobs ''`, the race-5090.ps1 shape) or one carrying a literal `jobs\` (the amd-card-test.ps1 shape); every path built from it belongs to that kit. A presence check (Test-Path, [IO.File]::Exists, [IO.Directory]::Exists, Get-Item or Get-ChildItem with -ErrorAction) on the root or anything under it covers the whole kit. A use before that line fails with "kit path used before a presence check: republish the fetch after any app update", as does a literal jobs\ path in a command with no check. The job's own folder ($env:IGNEUM_JOB_DIR) is not a kit path. Fixtures: kit-path-ok.ps1 (both shapes, checked; a sibling pack file covered by the worker's check) and kit-path-unchecked.ps1 (the worker run before its check, a literal never checked); --self-test asserts the lines. Wired into ci.yml after the bash-body step, and into publish-jobs.sh add --kind run beside the other two checks; test-publish-jobs.sh gains the refusal (34 passed, 0 failed). The current tree: race-5090.ps1 is the one playbook with a kit, checked before use. README-ship.md: the rule. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 2 + packaging/README-ship.md | 4 + packaging/ota/publish-jobs.sh | 7 +- packaging/ota/test-publish-jobs.sh | 6 +- tools/ci/fixtures/kit-path-ok.ps1 | 25 ++++++ tools/ci/fixtures/kit-path-unchecked.ps1 | 14 +++ tools/ci/kit-path-check.sh | 110 +++++++++++++++++++++++ 7 files changed, 165 insertions(+), 3 deletions(-) create mode 100644 tools/ci/fixtures/kit-path-ok.ps1 create mode 100644 tools/ci/fixtures/kit-path-unchecked.ps1 create mode 100755 tools/ci/kit-path-check.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1da843580..95d0c9661 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,8 @@ jobs: run: bash tools/ci/copied-sources-check.sh - name: bash bodies in PowerShell job scripts pass bash -n, the lost-quote class (self-test first, then the tree) run: bash tools/ci/bash-body-check.sh --self-test && bash tools/ci/bash-body-check.sh + - name: run jobs test their fetched kit before use, the wiped-jobs-folder class (self-test first, then the tree) + run: bash tools/ci/kit-path-check.sh --self-test && bash tools/ci/kit-path-check.sh - name: pinned guest programs match their manifest and are built only by pin-guests.sh run: bash tools/ci/pinned-guests-check.sh - name: no secret file names and no 64-hex secrets in the tree (self-test first, then the tree) diff --git a/packaging/README-ship.md b/packaging/README-ship.md index 0d62e74a5..ceb69dae2 100644 --- a/packaging/README-ship.md +++ b/packaging/README-ship.md @@ -121,3 +121,7 @@ file and run), unescapes it the way PowerShell would, and runs `bash -n` on it. never passes CI before it runs, so `packaging/ota/publish-jobs.sh add --kind run` runs the same check (and `tools/ci/prover-socket-check.sh`, the root-socket class; `bash -n` for a `.sh` script) on the script before anything is signed, and refuses the publish with the check's output. `packaging/ota/test-publish-jobs.sh` proves the refusals. +The wiped-jobs-folder class (5 October 2026, 21:49Z): an app install clears the jobs folder, so a run job that uses a kit +fetched by an earlier fetch job tests the kit is there (Test-Path on the kit root or any file under it) before its first +use, and the fetch is republished under a new id after any app update. `tools/ci/kit-path-check.sh` fails CI and the +publish on a kit path used before that check. diff --git a/packaging/ota/publish-jobs.sh b/packaging/ota/publish-jobs.sh index 8d444d5ff..1d3c9d7d0 100755 --- a/packaging/ota/publish-jobs.sh +++ b/packaging/ota/publish-jobs.sh @@ -261,14 +261,17 @@ if [ "$CMD" = add ]; then # anything is signed. A failure refuses the publish with the check's output; a missing check refuses it too. # PowerShell: every inline bash body must parse (tools/ci/bash-body-check.sh, the lost-quote class; a body it # cannot read fails, never skips). Bash: the script itself must parse. Both: a root prover run kills the GPU - # server and unlinks its socket (tools/ci/prover-socket-check.sh, the root-socket class). A check file that is - # missing from this tree refuses too (bash exits 127 with the reason), so no job goes out unchecked. + # server and unlinks its socket (tools/ci/prover-socket-check.sh, the root-socket class), and a fetched kit under + # the jobs folder is tested before its first use (tools/ci/kit-path-check.sh, the wiped-jobs-folder class). A + # check file that is missing from this tree refuses too (bash exits 127 with the reason), so no job goes out + # unchecked. if [ "$SHELL_KIND" = powershell ]; then out="$(bash "$ROOT/tools/ci/bash-body-check.sh" "$SCRIPT" 2>&1)" || { echo "run: bash-body-check.sh refuses $SCRIPT:" >&2; printf '%s\n' "$out" >&2; exit 1; } else out="$(bash -n "$SCRIPT" 2>&1)" || { echo "run: bash -n refuses $SCRIPT (the lost-quote class):" >&2; printf '%s\n' "$out" >&2; exit 1; } fi out="$(bash "$ROOT/tools/ci/prover-socket-check.sh" "$SCRIPT" 2>&1)" || { echo "run: prover-socket-check.sh refuses $SCRIPT:" >&2; printf '%s\n' "$out" >&2; exit 1; } + out="$(bash "$ROOT/tools/ci/kit-path-check.sh" "$SCRIPT" 2>&1)" || { echo "run: kit-path-check.sh refuses $SCRIPT:" >&2; printf '%s\n' "$out" >&2; exit 1; } PARAMS="$(python3 -c 'import json,sys; print(json.dumps({"script": open(sys.argv[1]).read(), "shell": sys.argv[2], "elevated": sys.argv[3]=="1", "stop_miners_first": sys.argv[4]=="1", **({"timeout_minutes": int(sys.argv[5])} if sys.argv[5] else {})}))' "$SCRIPT" "$SHELL_KIND" "$ELEVATED" "$STOP_MINERS" "$TIMEOUT_MIN")" [ -n "$TITLE" ] || TITLE="run $(basename "$SCRIPT")" ;; diff --git a/packaging/ota/test-publish-jobs.sh b/packaging/ota/test-publish-jobs.sh index c5f8b545e..3f149f8fd 100755 --- a/packaging/ota/test-publish-jobs.sh +++ b/packaging/ota/test-publish-jobs.sh @@ -9,7 +9,8 @@ # a new envelope; an envelope made by hand from the FIRST file and the SECOND signature (the stale pair an edge can # serve across a deploy) is REFUSED by the signer with the words the app logs; a tampered inner byte is refused; # `sign` rewrites all three consistently; a `run` script that fails a class check (a lost quote in an inline bash -# body, a body the check cannot read, a bash script that does not parse, a root prover run without the socket cleanup) +# body, a body the check cannot read, a bash script that does not parse, a root prover run without the socket cleanup, +# a fetched kit used before a presence check) # is REFUSED before anything is signed and the envelope is left as it was; and the real OTA key is the key the app # embeds. # @@ -112,6 +113,9 @@ expect_fail "a bash script with a lost quote" "$PUBLISH" add --kind run --target printf '& wsl.exe -d Ubuntu-24.04 -u root -- bash /mnt/c/prove.sh\n# igneum-prove-host --mode shard --shard 0\n' > "$T/root-prover.ps1" expect_fail "a root prover script without the socket cleanup" "$PUBLISH" add --kind run --target 1ccfe586 --script "$T/root-prover.ps1" --id test-root-socket --dest "$D" --base-url https://example.invalid/dl/t grep -q "prover-socket" "$T/out" && ok "refused by the socket check" || bad "another reason: $(tail -1 "$T/out")" +printf '$jobs = Split-Path $env:IGNEUM_JOB_DIR\n$exe = Join-Path (Join-Path $jobs "fetch-kit-1") "worker.exe"\n& $exe --list\n' > "$T/kit-unchecked.ps1" +expect_fail "a run script that uses a fetched kit before a presence check" "$PUBLISH" add --kind run --target 1ccfe586 --script "$T/kit-unchecked.ps1" --id test-kit-unchecked --dest "$D" --base-url https://example.invalid/dl/t +grep -q "kit path used before a presence check" "$T/out" && ok "refused by the kit-path check" || bad "another reason: $(tail -1 "$T/out")" cmp -s "$T/before.signed" "$D/igneum-jobs.signed.json" && ok "a refused publish leaves the envelope untouched" || bad "a refused publish changed the envelope" echo "== the key" diff --git a/tools/ci/fixtures/kit-path-ok.ps1 b/tools/ci/fixtures/kit-path-ok.ps1 new file mode 100644 index 000000000..14904f369 --- /dev/null +++ b/tools/ci/fixtures/kit-path-ok.ps1 @@ -0,0 +1,25 @@ +# Fixture for tools/ci/kit-path-check.sh --self-test: both ways a run job reaches a fetched kit under the app's jobs +# folder, each checked for presence before its first use. Must pass (2 kit paths). Never run; a stand-in for a job. +$ErrorActionPreference = 'Continue' +# the job's own folder always exists (the app made it for this run): not a kit path +$job = $env:IGNEUM_JOB_DIR +$jobFile = Join-Path $env:IGNEUM_JOB_DIR 'jobs.txt' +# 1. the race-5090.ps1 shape: the jobs folder is the parent of this job's folder, the kit is a sibling job's folder +$jobs = Split-Path $env:IGNEUM_JOB_DIR +$fetched = Join-Path $jobs 'fetch-race-worker-20261004' +$exe = Join-Path $fetched 'igneum-worker-cuda.exe' +if (-not (Test-Path $exe)) { Write-Output "RESULT race worker missing at $exe (the fetch job runs first)"; exit 2 } +& $exe --race --race-rounds 3 2>&1 | ForEach-Object { "RESULT $_" } +# 2. the amd-card-test.ps1 shape: jobs\\kit under the app folder, a wait loop then the check; a sibling file +# under the same kit (pack-a) is covered by the check on the worker +$KitJob = 'amd-kit-1' +$kit = Join-Path $env:IGNEUM_APP_DIR ("jobs\" + $KitJob + "\kit") +$worker = Join-Path $kit 'igneum-worker-opencl.exe' +$tele = Join-Path $kit 'igneum-gpu-telemetry.exe' +$pack = Join-Path $kit 'pack-a' +$deadline = (Get-Date).AddMinutes(2) +while (-not ((Test-Path $worker) -and (Test-Path $tele)) -and (Get-Date) -lt $deadline) { Start-Sleep -Seconds 5 } +if (-not (Test-Path $worker)) { "RESULT no worker at $worker"; exit 1 } +$list = & $worker --list 2>&1 +$serve = Start-Process -FilePath $worker -ArgumentList '--serve','--pack',"`"$pack`"" -RedirectStandardInput $jobFile -NoNewWindow -PassThru +exit 0 diff --git a/tools/ci/fixtures/kit-path-unchecked.ps1 b/tools/ci/fixtures/kit-path-unchecked.ps1 new file mode 100644 index 000000000..410235371 --- /dev/null +++ b/tools/ci/fixtures/kit-path-unchecked.ps1 @@ -0,0 +1,14 @@ +# Fixture for tools/ci/kit-path-check.sh --self-test: the wiped-jobs-folder class (5 October 2026, 21:49Z: the 0.3.10 +# install cleared the jobs folder and the AMD kit job failed in seconds). Line 9 runs the worker before its check at +# line 10; line 13 runs a literal jobs\ path that is never checked. Must fail twice. Never run; a stand-in for a job. +$ErrorActionPreference = 'Continue' +# 1. the worker is run first and tested after +$jobs = Split-Path $env:IGNEUM_JOB_DIR +$fetched = Join-Path $jobs 'fetch-race-worker-20261004' +$exe = Join-Path $fetched 'igneum-worker-cuda.exe' +& $exe --race 2>&1 | ForEach-Object { "RESULT $_" } +if (-not (Test-Path $exe)) { Write-Output "RESULT race worker missing at $exe"; exit 2 } +# 2. an inline literal under the jobs folder, never checked +$KitJob = 'amd-kit-1' +& "$env:IGNEUM_APP_DIR\jobs\$KitJob\kit\igneum-worker-opencl.exe" --list 2>&1 +exit 0 diff --git a/tools/ci/kit-path-check.sh b/tools/ci/kit-path-check.sh new file mode 100755 index 000000000..693791f8e --- /dev/null +++ b/tools/ci/kit-path-check.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# The wiped-jobs-folder class (5 October 2026, 21:49Z, bench-log 39f02ff): the app's install clears the jobs folder on +# a PC, so a run job whose kit was fetched by an earlier fetch job finds nothing after an update and fails in seconds. +# Rule: a run playbook that reaches a path under the app's jobs folder other than its own +# (`$jobs = Split-Path $env:IGNEUM_JOB_DIR` then `Join-Path $jobs ''`, the race-5090.ps1 shape; or a literal +# `jobs\\...`, the amd-card-test.ps1 shape) tests that the kit is there (Test-Path, [IO.File]::Exists, +# [IO.Directory]::Exists, Get-Item or Get-ChildItem with -ErrorAction) before its first use, and republishes the fetch +# after any app update. A check on the kit's root or on anything under it covers the whole kit. This check reads every +# *.ps1 under relay/playbooks/ and tools/ and fails on a kit path used before a presence check. The job's own folder +# ($env:IGNEUM_JOB_DIR, made by the app for the run) is not a kit path. +# +# Usage: tools/ci/kit-path-check.sh # the tree (tools/ci/fixtures/ left out) +# tools/ci/kit-path-check.sh ... # named files (the jobs publisher runs it on the script it publishes) +# tools/ci/kit-path-check.sh --self-test # must fire on the unchecked fixture and stay quiet on the correct one +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../.." && pwd)" + +check_files() { + python3 - "$@" <<'PY' +import re, sys + +SEED_LIT = re.compile(r'jobs\\') # a literal path under the jobs folder +SEED_FOLDER = re.compile(r'Split-Path\s+\$env:IGNEUM_JOB_DIR') # the jobs folder itself: the parent of this job's folder +CHECK = re.compile(r'Test-Path|\[IO\.(File|Directory)\]::Exists|Get-(Item|ChildItem)\b[^|]*-ErrorAction') +ASSIGN = re.compile(r'^\s*\$([A-Za-z_][A-Za-z0-9_]*)\s*=(?!=)\s*(.*)$') +# a right-hand side that only builds a path (the kit var it names is then a path, not a use of one) +DERIV = re.compile(r'^(Join-Path\b|Split-Path\b|\(|"|\'|\[IO\.Path\]::Combine|\$env:|\$[A-Za-z_][A-Za-z0-9_]*\s*(\+|\.(TrimEnd|Replace)|$))') +TEXT_ONLY = re.compile(r'^\s*(Write-Output|Write-Host|Write-Verbose|Say|Log)\b|^\s*"') +VAR = re.compile(r'\$([A-Za-z_][A-Za-z0-9_]*)\b(?!:)') +MSG = 'kit path used before a presence check: republish the fetch after any app update' + +rc = 0 +files = sys.argv[1:] +with_kits = 0 +for path in files: + try: + lines = open(path, encoding='utf-8', errors='replace').read().split('\n') + except OSError as e: + print('FAIL %s: %s' % (path, e)); rc = 1; continue + folder = set() # vars that are the jobs folder (always there) + root_of = {} # kit var -> its root var + root_line = {} # root var -> the line it is defined on + checked = set() # roots with a presence check so far + reported = set() + inline_checked = False + fails = [] + for ln, raw in enumerate(lines, 1): + s = raw.strip() + if not s or s.startswith('#'): continue + refs = set(VAR.findall(raw)) + kit_refs = refs & set(root_of) + m = ASSIGN.match(raw) + if m: + name, rhs = m.group(1), m.group(2).strip() + if SEED_FOLDER.search(rhs) and not SEED_LIT.search(rhs): + folder.add(name); continue + deriv = bool(DERIV.match(rhs)) + if deriv and (SEED_LIT.search(rhs) or (refs & folder)): + root_of[name] = name; root_line[name] = ln; continue + if deriv and kit_refs: + root_of[name] = root_of[sorted(kit_refs)[0]]; continue + if CHECK.search(raw): + for v in kit_refs: checked.add(root_of[v]) + if SEED_LIT.search(raw): inline_checked = True + continue + for v in sorted(kit_refs): + r = root_of[v] + if r in checked or r in reported: continue + reported.add(r) + fails.append('FAIL %s:%d %s ($%s, kit path $%s defined at line %d)' % (path, ln, MSG, v, r, root_line[r])) + if SEED_LIT.search(raw) and not inline_checked and not TEXT_ONLY.match(raw): + inline_checked = True + fails.append('FAIL %s:%d %s (a literal jobs\\ path in a command, never checked)' % (path, ln, MSG)) + if fails: + rc = 1 + for f in fails: print(f) + elif root_of or inline_checked: + with_kits += 1 + print('ok %s (%d kit path%s, checked before use)' % (path, len(root_line), '' if len(root_line) == 1 else 's')) +print('kit-path-check: %d files, %d with a fetched kit, %s' % (len(files), with_kits, 'all checked before use' if rc == 0 else 'FAILURES above')) +sys.exit(rc) +PY +} + +if [ "${1:-}" = "--self-test" ]; then + F="$HERE/fixtures" + ok_out="$(check_files "$F/kit-path-ok.ps1" 2>&1)"; ok_rc=$? + bad_out="$(check_files "$F/kit-path-unchecked.ps1" 2>&1)"; bad_rc=$? + echo "self-test correct fixture: rc $ok_rc"; printf '%s\n' "$ok_out" | sed 's/^/ /' + echo "self-test unchecked fixture: rc $bad_rc"; printf '%s\n' "$bad_out" | sed 's/^/ /' + [ $ok_rc -eq 0 ] || { echo "SELF-TEST FAILED: the correct fixture was flagged"; exit 1; } + printf '%s\n' "$ok_out" | grep -q '^ok .*2 kit paths' || { echo "SELF-TEST FAILED: the correct fixture holds 2 kit paths, a different number was found"; exit 1; } + [ $bad_rc -ne 0 ] || { echo "SELF-TEST FAILED: the unchecked fixture passed"; exit 1; } + for want in 'kit-path-unchecked.ps1:9 ' 'kit-path-unchecked.ps1:13 '; do + printf '%s\n' "$bad_out" | grep -q "^FAIL .*$want.*republish the fetch after any app update" || { echo "SELF-TEST FAILED: the unchecked use at $want was not reported"; exit 1; } + done + [ "$(printf '%s\n' "$bad_out" | grep -c '^FAIL ')" -eq 2 ] || { echo "SELF-TEST FAILED: 2 failures expected"; exit 1; } + echo "self-test passed: the unchecked kit paths fail, the checked ones pass" + exit 0 +fi + +if [ $# -gt 0 ]; then + check_files "$@"; exit $? +fi +cd "$REPO" +files=$(git ls-files 'relay/playbooks/**' 'tools/**' | grep -E '\.ps1$' | grep -v '^tools/ci/fixtures/' || true) +if [ -z "$files" ]; then echo "kit-path-check: no .ps1 files under relay/playbooks/ or tools/"; exit 0; fi +# shellcheck disable=SC2086 +check_files $files