From 11888df9a76fcede68620a290a508490c3d6d6d3 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:10:18 +0000 Subject: [PATCH] Jobs publisher: a remove refuses a job the target's latest report shows running, and any job published with --installs-app, unless --force \"\" (7 Oct 2026 18:53 BST: a removal reached PC 2 one second after its job launched a silent installer over the running app; the runner's abort ended the process tree and the app went dark); tools/ci/publish-jobs-check.sh in the gate Co-Authored-By: Claude Fable 5.1 --- packaging/ota/publish-jobs.sh | 31 ++++++++++++++++++++++++++----- tools/ci/pre-push.sh | 1 + tools/ci/publish-jobs-check.sh | 30 ++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 5 deletions(-) create mode 100755 tools/ci/publish-jobs-check.sh diff --git a/packaging/ota/publish-jobs.sh b/packaging/ota/publish-jobs.sh index 38d2fdc2e..055304c1c 100755 --- a/packaging/ota/publish-jobs.sh +++ b/packaging/ota/publish-jobs.sh @@ -54,7 +54,7 @@ KIND="" TARGET="" PLATFORM="" REQUIRES="" REQUIRES_SET=0 ID="" TITLE="" EXPIRES_ SCRIPT="" SHELL_KIND="" ELEVATED=0 STOP_MINERS=0 TIMEOUT_MIN="" CARDS_OFF="" FILE="" URL="" SHA="" SIZE="" DIR="" TO="" EXTRACT=0 EXTRACT_DIR="" FRESH=0 GLOBS=() COMMAND="" WHAT="" -ZIP="" FIXTURES="" CAP_MIN="" DISTRO="" WSL_USER="" REMOVE_ID="" +ZIP="" FIXTURES="" CAP_MIN="" DISTRO="" WSL_USER="" REMOVE_ID="" FORCE="" INSTALLS_APP=0 TARGETS="" BUDGET_MIN="" STAGE_MIN="" MIN_FREE_GB="" TESTS=1 RELAY_URL="" NICE="" CARGO_JOBS="" case "$CMD" in remove) REMOVE_ID="${1:-}"; [ -n "$REMOVE_ID" ] || { echo "remove " >&2; exit 2; }; shift ;; @@ -67,6 +67,8 @@ while [ $# -gt 0 ]; do --requires) REQUIRES="$2"; REQUIRES_SET=1; [ "$REQUIRES" = none ] && REQUIRES=""; shift 2 ;; --id) ID="$2"; shift 2 ;; --title) TITLE="$2"; shift 2 ;; + --force) FORCE="$2"; shift 2 ;; # remove: override the running-job refusal, with the reason (7 Oct 2026) + --installs-app) INSTALLS_APP=1; shift ;; # add --kind run: the script installs over the app; never removed without --force --expires-hours) EXPIRES_H="$2"; shift 2 ;; --script) SCRIPT="$2"; shift 2 ;; --shell) SHELL_KIND="$2"; shift 2 ;; @@ -227,8 +229,8 @@ if [ "$CMD" = list ]; then [ -f "$JOBS" ] || { echo "no jobs file in $DEST"; exit 0; } "$SIGNER" verify-jobs "$PUB" "$JOBS" "$JOBS.sig" || { echo "the file in $DEST does not verify; run: $0 sign" >&2; exit 1; } if [ -f "$SIGNED" ]; then "$SIGNER" verify-signed-jobs "$PUB" "$SIGNED" >/dev/null || { echo "the envelope in $DEST does not verify; run: $0 sign" >&2; exit 1; }; else echo "(no igneum-jobs.signed.json yet; the next write makes one)"; fi - python3 - "$JOBS" <<'PY' -import json, sys, datetime + INSTALLS_APP="$INSTALLS_APP" python3 - "$JOBS" <<'PY' +import json, sys, datetime, os f = json.load(open(sys.argv[1])) now = datetime.datetime.now(datetime.timezone.utc) for j in f.get("jobs", []): @@ -353,17 +355,36 @@ except Exception: print("")' "${ZIP%.zip}.json" 2>/dev/null || true)" esac [ -n "$PLATFORM" ] || PLATFORM=any [ -n "$ID" ] || ID="$KIND-$(date -u +%Y%m%d-%H%M%S)" - NEW_JOB="$(python3 -c 'import json,sys,datetime + NEW_JOB="$(INSTALLS_APP="$INSTALLS_APP" python3 -c 'import json,sys,datetime,os a=sys.argv now=datetime.datetime.now(datetime.timezone.utc) t={"machine_ids": "all" if a[2]=="all" else [x.strip().lower() for x in a[2].split(",") if x.strip()], "platform": a[3]} +if os.environ.get("INSTALLS_APP")=="1" and a[5]=="run": t.setdefault("params",{})["installs_app"]=True if a[4]: t["requires"]=[x.strip() for x in a[4].split(",") if x.strip()] print(json.dumps({"id": a[1], "kind": a[5], "title": a[6], "created_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "expires_at": (now+datetime.timedelta(hours=float(a[7]))).strftime("%Y-%m-%dT%H:%M:%SZ"), "target": t, "params": json.loads(a[8]), "report": "log-intake"}))' "$ID" "$TARGET" "$PLATFORM" "$REQUIRES" "$KIND" "$TITLE" "$EXPIRES_H" "$PARAMS")" fi +# ---- the removal guard (7 October 2026, 18:53 BST: a removal reached PC 2 one second after its job had launched a silent +# installer over the running app; the runner's abort-on-removal ended the process tree and the app went dark). A remove refuses +# when any target's latest report for the id has started and carries no final line (the job is running), or when the job was +# published with --installs-app; `--force ""` overrides, and the reason is printed. The read is tools/jobs.mjs . +# REMOVE_GUARD_READ= replaces the read for the self-test (tools/ci/publish-jobs-check.sh). +if [ -n "$REMOVE_ID" ]; then + read_out="$( if [ -n "${REMOVE_GUARD_READ:-}" ]; then cat "$REMOVE_GUARD_READ"; else node "$ROOT/tools/jobs.mjs" "$REMOVE_ID" 2>/dev/null || true; fi )" + running="$(printf '%s\n' "$read_out" | grep -cE '^(job [^ ]+ \(.*\) on .* started|== running the)' || true)" + final="$(printf '%s\n' "$read_out" | grep -cE '^(SUMMARY: |job [^ ]+: (done|failed|aborted|timeout))' || true)" + jobs_now="$JOBS"; [ -f "$DEST/igneum-jobs.json" ] && jobs_now="$DEST/igneum-jobs.json" # the file at the destination this run writes + installs="$(python3 -c 'import json,sys; j=[x for x in json.load(open(sys.argv[1])).get("jobs",[]) if x.get("id")==sys.argv[2]]; print(1 if j and "\"installs_app\": true" in json.dumps(j[0]) else 0)' "$jobs_now" "$REMOVE_ID" 2>/dev/null || echo 0)" + if [ -z "$FORCE" ]; then + if [ "$running" -gt 0 ] && [ "$final" = 0 ]; then echo "remove refused: $REMOVE_ID has started on a machine and has no final line yet (a removal ends the running job's process tree); wait for its SUMMARY, or --force \"\"" >&2; exit 3; fi + if [ "$installs" = 1 ]; then echo "remove refused: $REMOVE_ID was published with --installs-app (it installs over the app); --force \"\" to remove it anyway" >&2; exit 3; fi + else + echo "remove: --force given ($FORCE); running=$running final=$final installs_app=$installs" + fi +fi # ---- merge: current jobs minus expired (minus a removed id), plus the new one; canonical JSON --------------------- NEW="$JOBS.new" -python3 - "$JOBS" "$NEW" "$NEW_JOB" "$REMOVE_ID" <<'PY' +INSTALLS_APP="$INSTALLS_APP" python3 - "$JOBS" "$NEW" "$NEW_JOB" "$REMOVE_ID" <<'PY' import json, sys, datetime, os cur, out, new_job, remove = sys.argv[1:5] now = datetime.datetime.now(datetime.timezone.utc) diff --git a/tools/ci/pre-push.sh b/tools/ci/pre-push.sh index 266cd7122..3428f51b9 100755 --- a/tools/ci/pre-push.sh +++ b/tools/ci/pre-push.sh @@ -106,6 +106,7 @@ tree_checks() { run "the class router is a preference with spill-over (a held or overloaded box hands the job to the other one)" bash tools/ci/route-spill-check.sh run "per-core leases, the quiet class and the reaper pass on the box (lease.sh and remote-run.sh self-tests over ssh)" bash tools/ci/box-locks-check.sh $( [ "$MODE" = ci ] && echo --ci ) run "the simulators job runs on master and release-* pushes and pull requests into them only" bash tools/ci/sims-branch-check.sh + run "publish-jobs.sh never removes a running or installs-app job without --force (the PC 2 abort class)" bash tools/ci/publish-jobs-check.sh run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh' run "no script kills or finds a process by a plain name or a file name (pgrep/pkill -f literals, ps | grep)" bash -c 'bash tools/ci/kill-by-name-check.sh --self-test && bash tools/ci/kill-by-name-check.sh' run "the identity check's own self-test (excluded research path passes, exported leak fails)" bash tools/ci/identity-check.sh --self-test diff --git a/tools/ci/publish-jobs-check.sh b/tools/ci/publish-jobs-check.sh new file mode 100755 index 000000000..3334e94b7 --- /dev/null +++ b/tools/ci/publish-jobs-check.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# The removal guard of packaging/ota/publish-jobs.sh (7 October 2026, 18:53 BST: a removal reached PC 2 one second after its job had +# launched a silent installer over the running app; the runner's abort-on-removal ended the process tree and the app went dark). +# A remove must refuse while any target's report shows the job started with no final line, and refuse a job published with +# --installs-app; --force "" overrides. The check runs the script against a scratch destination (--dest) with the +# machine read replaced by a fixture (REMOVE_GUARD_READ), so nothing is published and no machine is read. +# +# tools/ci/publish-jobs-check.sh # exit 1 with the case that resolved wrongly (the check IS its self-test) +set -euo pipefail +cd "$(dirname "$0")/../.." +t=$(mktemp -d); trap 'rm -rf "$t"' EXIT +mkdir -p "$t/dest"; export IGNEUM_DLSITE="$t/site"; mkdir -p "$t/site/dl/testtoken" +P="packaging/ota/publish-jobs.sh" +# the real OTA key signs into the scratch --dest (as packaging/ota/test-publish-jobs.sh does; nothing is deployed, the downloads +# folder is untouched); a machine without the key or the signer skips the check as not applicable +[ -f "$HOME/.config/igneum/ota-signing-key" ] && [ -x app/igneum-app/target/release/igneum-ota-sign ] || { echo "publish-jobs-check: no OTA key or signer here; skipped as not applicable"; exit 0; } +printf 'echo hi\n' > "$t/s.ps1" +bash "$P" add --kind run --target 00000001 --script "$t/s.ps1" --id guard-run --title t --requires none --dest "$t/dest" >/dev/null 2>&1 || { echo "publish-jobs-check: add failed"; exit 1; } +bash "$P" add --kind run --target 00000001 --script "$t/s.ps1" --id guard-install --installs-app --title t --requires none --dest "$t/dest" >/dev/null 2>&1 || { echo "publish-jobs-check: add --installs-app failed"; exit 1; } +grep -q '"installs_app":true' "$t/dest/igneum-jobs.json" || { echo "publish-jobs-check: --installs-app did not write the param"; exit 1; } +fail=0 +printf 'job guard-run (run) on PC machine 0000000100000000 run job-guard-run-00000001, started 2026-10-07T18:00:00Z\n== running the powershell script (cap 40 min) ==\nRESULT start\n' > "$t/running.txt" +printf 'SUMMARY: done exit 0, started 2026-10-07T18:00:00Z, finished 2026-10-07T18:00:09Z, 9 s: script finished, exit 0\njob guard-run: done (exit 0) after 9 s: script finished\n' > "$t/done.txt" +if REMOVE_GUARD_READ="$t/running.txt" bash "$P" remove guard-run --dest "$t/dest" >/dev/null 2>&1; then echo "publish-jobs-check: a RUNNING job was removed"; fail=1; else echo "publish-jobs-check: a running job's removal is refused"; fi +if REMOVE_GUARD_READ="$t/running.txt" bash "$P" remove guard-run --dest "$t/dest" --force "test" >/dev/null 2>&1; then echo "publish-jobs-check: --force removes a running job (with the reason)"; else echo "publish-jobs-check: --force did not remove"; fail=1; fi +if REMOVE_GUARD_READ="$t/done.txt" bash "$P" remove guard-install --dest "$t/dest" >/dev/null 2>&1; then echo "publish-jobs-check: an --installs-app job was removed without --force"; fail=1; else echo "publish-jobs-check: an --installs-app job's removal is refused"; fi +bash "$P" add --kind run --target 00000001 --script "$t/s.ps1" --id guard-run2 --title t --requires none --dest "$t/dest" >/dev/null 2>&1 +if REMOVE_GUARD_READ="$t/done.txt" bash "$P" remove guard-run2 --dest "$t/dest" >/dev/null 2>&1; then echo "publish-jobs-check: a finished job's removal passes"; else echo "publish-jobs-check: a finished job's removal was refused"; fail=1; fi +[ "$fail" = 0 ] && echo "publish-jobs-check: a running or installs-app job is never removed without --force; a finished one is" +exit $fail