Jobs publisher: a remove refuses a job the target's latest report shows running, and any job published with --installs-app, unless --force \"<reason>\" (7 Oct 2026 18:53 BST: a removal reached PC 2 one second after its job launched a silent installer over the running app; the runner's abort ended the process tree and the app went dark); tools/ci/publish-jobs-check.sh in the gate
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs
parent
f070417c7b
commit
11888df9a7
3 changed files with 57 additions and 5 deletions
|
|
@ -54,7 +54,7 @@ KIND="" TARGET="" PLATFORM="" REQUIRES="" REQUIRES_SET=0 ID="" TITLE="" EXPIRES_
|
|||
SCRIPT="" SHELL_KIND="" ELEVATED=0 STOP_MINERS=0 TIMEOUT_MIN="" CARDS_OFF=""
|
||||
FILE="" URL="" SHA="" SIZE="" DIR="" TO="" EXTRACT=0 EXTRACT_DIR="" FRESH=0
|
||||
GLOBS=() COMMAND="" WHAT=""
|
||||
ZIP="" FIXTURES="" CAP_MIN="" DISTRO="" WSL_USER="" REMOVE_ID=""
|
||||
ZIP="" FIXTURES="" CAP_MIN="" DISTRO="" WSL_USER="" REMOVE_ID="" FORCE="" INSTALLS_APP=0
|
||||
TARGETS="" BUDGET_MIN="" STAGE_MIN="" MIN_FREE_GB="" TESTS=1 RELAY_URL="" NICE="" CARGO_JOBS=""
|
||||
case "$CMD" in
|
||||
remove) REMOVE_ID="${1:-}"; [ -n "$REMOVE_ID" ] || { echo "remove <id>" >&2; exit 2; }; shift ;;
|
||||
|
|
@ -67,6 +67,8 @@ while [ $# -gt 0 ]; do
|
|||
--requires) REQUIRES="$2"; REQUIRES_SET=1; [ "$REQUIRES" = none ] && REQUIRES=""; shift 2 ;;
|
||||
--id) ID="$2"; shift 2 ;;
|
||||
--title) TITLE="$2"; shift 2 ;;
|
||||
--force) FORCE="$2"; shift 2 ;; # remove: override the running-job refusal, with the reason (7 Oct 2026)
|
||||
--installs-app) INSTALLS_APP=1; shift ;; # add --kind run: the script installs over the app; never removed without --force
|
||||
--expires-hours) EXPIRES_H="$2"; shift 2 ;;
|
||||
--script) SCRIPT="$2"; shift 2 ;;
|
||||
--shell) SHELL_KIND="$2"; shift 2 ;;
|
||||
|
|
@ -227,8 +229,8 @@ if [ "$CMD" = list ]; then
|
|||
[ -f "$JOBS" ] || { echo "no jobs file in $DEST"; exit 0; }
|
||||
"$SIGNER" verify-jobs "$PUB" "$JOBS" "$JOBS.sig" || { echo "the file in $DEST does not verify; run: $0 sign" >&2; exit 1; }
|
||||
if [ -f "$SIGNED" ]; then "$SIGNER" verify-signed-jobs "$PUB" "$SIGNED" >/dev/null || { echo "the envelope in $DEST does not verify; run: $0 sign" >&2; exit 1; }; else echo "(no igneum-jobs.signed.json yet; the next write makes one)"; fi
|
||||
python3 - "$JOBS" <<'PY'
|
||||
import json, sys, datetime
|
||||
INSTALLS_APP="$INSTALLS_APP" python3 - "$JOBS" <<'PY'
|
||||
import json, sys, datetime, os
|
||||
f = json.load(open(sys.argv[1]))
|
||||
now = datetime.datetime.now(datetime.timezone.utc)
|
||||
for j in f.get("jobs", []):
|
||||
|
|
@ -353,17 +355,36 @@ except Exception: print("")' "${ZIP%.zip}.json" 2>/dev/null || true)"
|
|||
esac
|
||||
[ -n "$PLATFORM" ] || PLATFORM=any
|
||||
[ -n "$ID" ] || ID="$KIND-$(date -u +%Y%m%d-%H%M%S)"
|
||||
NEW_JOB="$(python3 -c 'import json,sys,datetime
|
||||
NEW_JOB="$(INSTALLS_APP="$INSTALLS_APP" python3 -c 'import json,sys,datetime,os
|
||||
a=sys.argv
|
||||
now=datetime.datetime.now(datetime.timezone.utc)
|
||||
t={"machine_ids": "all" if a[2]=="all" else [x.strip().lower() for x in a[2].split(",") if x.strip()], "platform": a[3]}
|
||||
if os.environ.get("INSTALLS_APP")=="1" and a[5]=="run": t.setdefault("params",{})["installs_app"]=True
|
||||
if a[4]: t["requires"]=[x.strip() for x in a[4].split(",") if x.strip()]
|
||||
print(json.dumps({"id": a[1], "kind": a[5], "title": a[6], "created_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "expires_at": (now+datetime.timedelta(hours=float(a[7]))).strftime("%Y-%m-%dT%H:%M:%SZ"), "target": t, "params": json.loads(a[8]), "report": "log-intake"}))' "$ID" "$TARGET" "$PLATFORM" "$REQUIRES" "$KIND" "$TITLE" "$EXPIRES_H" "$PARAMS")"
|
||||
fi
|
||||
|
||||
# ---- the removal guard (7 October 2026, 18:53 BST: a removal reached PC 2 one second after its job had launched a silent
|
||||
# installer over the running app; the runner's abort-on-removal ended the process tree and the app went dark). A remove refuses
|
||||
# when any target's latest report for the id has started and carries no final line (the job is running), or when the job was
|
||||
# published with --installs-app; `--force "<reason>"` overrides, and the reason is printed. The read is tools/jobs.mjs <id>.
|
||||
# REMOVE_GUARD_READ=<file> replaces the read for the self-test (tools/ci/publish-jobs-check.sh).
|
||||
if [ -n "$REMOVE_ID" ]; then
|
||||
read_out="$( if [ -n "${REMOVE_GUARD_READ:-}" ]; then cat "$REMOVE_GUARD_READ"; else node "$ROOT/tools/jobs.mjs" "$REMOVE_ID" 2>/dev/null || true; fi )"
|
||||
running="$(printf '%s\n' "$read_out" | grep -cE '^(job [^ ]+ \(.*\) on .* started|== running the)' || true)"
|
||||
final="$(printf '%s\n' "$read_out" | grep -cE '^(SUMMARY: |job [^ ]+: (done|failed|aborted|timeout))' || true)"
|
||||
jobs_now="$JOBS"; [ -f "$DEST/igneum-jobs.json" ] && jobs_now="$DEST/igneum-jobs.json" # the file at the destination this run writes
|
||||
installs="$(python3 -c 'import json,sys; j=[x for x in json.load(open(sys.argv[1])).get("jobs",[]) if x.get("id")==sys.argv[2]]; print(1 if j and "\"installs_app\": true" in json.dumps(j[0]) else 0)' "$jobs_now" "$REMOVE_ID" 2>/dev/null || echo 0)"
|
||||
if [ -z "$FORCE" ]; then
|
||||
if [ "$running" -gt 0 ] && [ "$final" = 0 ]; then echo "remove refused: $REMOVE_ID has started on a machine and has no final line yet (a removal ends the running job's process tree); wait for its SUMMARY, or --force \"<reason>\"" >&2; exit 3; fi
|
||||
if [ "$installs" = 1 ]; then echo "remove refused: $REMOVE_ID was published with --installs-app (it installs over the app); --force \"<reason>\" to remove it anyway" >&2; exit 3; fi
|
||||
else
|
||||
echo "remove: --force given ($FORCE); running=$running final=$final installs_app=$installs"
|
||||
fi
|
||||
fi
|
||||
# ---- merge: current jobs minus expired (minus a removed id), plus the new one; canonical JSON ---------------------
|
||||
NEW="$JOBS.new"
|
||||
python3 - "$JOBS" "$NEW" "$NEW_JOB" "$REMOVE_ID" <<'PY'
|
||||
INSTALLS_APP="$INSTALLS_APP" python3 - "$JOBS" "$NEW" "$NEW_JOB" "$REMOVE_ID" <<'PY'
|
||||
import json, sys, datetime, os
|
||||
cur, out, new_job, remove = sys.argv[1:5]
|
||||
now = datetime.datetime.now(datetime.timezone.utc)
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@ tree_checks() {
|
|||
run "the class router is a preference with spill-over (a held or overloaded box hands the job to the other one)" bash tools/ci/route-spill-check.sh
|
||||
run "per-core leases, the quiet class and the reaper pass on the box (lease.sh and remote-run.sh self-tests over ssh)" bash tools/ci/box-locks-check.sh $( [ "$MODE" = ci ] && echo --ci )
|
||||
run "the simulators job runs on master and release-* pushes and pull requests into them only" bash tools/ci/sims-branch-check.sh
|
||||
run "publish-jobs.sh never removes a running or installs-app job without --force (the PC 2 abort class)" bash tools/ci/publish-jobs-check.sh
|
||||
run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh'
|
||||
run "no script kills or finds a process by a plain name or a file name (pgrep/pkill -f literals, ps | grep)" bash -c 'bash tools/ci/kill-by-name-check.sh --self-test && bash tools/ci/kill-by-name-check.sh'
|
||||
run "the identity check's own self-test (excluded research path passes, exported leak fails)" bash tools/ci/identity-check.sh --self-test
|
||||
|
|
|
|||
30
tools/ci/publish-jobs-check.sh
Executable file
30
tools/ci/publish-jobs-check.sh
Executable file
|
|
@ -0,0 +1,30 @@
|
|||
#!/usr/bin/env bash
|
||||
# The removal guard of packaging/ota/publish-jobs.sh (7 October 2026, 18:53 BST: a removal reached PC 2 one second after its job had
|
||||
# launched a silent installer over the running app; the runner's abort-on-removal ended the process tree and the app went dark).
|
||||
# A remove must refuse while any target's report shows the job started with no final line, and refuse a job published with
|
||||
# --installs-app; --force "<reason>" overrides. The check runs the script against a scratch destination (--dest) with the
|
||||
# machine read replaced by a fixture (REMOVE_GUARD_READ), so nothing is published and no machine is read.
|
||||
#
|
||||
# tools/ci/publish-jobs-check.sh # exit 1 with the case that resolved wrongly (the check IS its self-test)
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/../.."
|
||||
t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
|
||||
mkdir -p "$t/dest"; export IGNEUM_DLSITE="$t/site"; mkdir -p "$t/site/dl/testtoken"
|
||||
P="packaging/ota/publish-jobs.sh"
|
||||
# the real OTA key signs into the scratch --dest (as packaging/ota/test-publish-jobs.sh does; nothing is deployed, the downloads
|
||||
# folder is untouched); a machine without the key or the signer skips the check as not applicable
|
||||
[ -f "$HOME/.config/igneum/ota-signing-key" ] && [ -x app/igneum-app/target/release/igneum-ota-sign ] || { echo "publish-jobs-check: no OTA key or signer here; skipped as not applicable"; exit 0; }
|
||||
printf 'echo hi\n' > "$t/s.ps1"
|
||||
bash "$P" add --kind run --target 00000001 --script "$t/s.ps1" --id guard-run --title t --requires none --dest "$t/dest" >/dev/null 2>&1 || { echo "publish-jobs-check: add failed"; exit 1; }
|
||||
bash "$P" add --kind run --target 00000001 --script "$t/s.ps1" --id guard-install --installs-app --title t --requires none --dest "$t/dest" >/dev/null 2>&1 || { echo "publish-jobs-check: add --installs-app failed"; exit 1; }
|
||||
grep -q '"installs_app":true' "$t/dest/igneum-jobs.json" || { echo "publish-jobs-check: --installs-app did not write the param"; exit 1; }
|
||||
fail=0
|
||||
printf 'job guard-run (run) on PC machine 0000000100000000 run job-guard-run-00000001, started 2026-10-07T18:00:00Z\n== running the powershell script (cap 40 min) ==\nRESULT start\n' > "$t/running.txt"
|
||||
printf 'SUMMARY: done exit 0, started 2026-10-07T18:00:00Z, finished 2026-10-07T18:00:09Z, 9 s: script finished, exit 0\njob guard-run: done (exit 0) after 9 s: script finished\n' > "$t/done.txt"
|
||||
if REMOVE_GUARD_READ="$t/running.txt" bash "$P" remove guard-run --dest "$t/dest" >/dev/null 2>&1; then echo "publish-jobs-check: a RUNNING job was removed"; fail=1; else echo "publish-jobs-check: a running job's removal is refused"; fi
|
||||
if REMOVE_GUARD_READ="$t/running.txt" bash "$P" remove guard-run --dest "$t/dest" --force "test" >/dev/null 2>&1; then echo "publish-jobs-check: --force removes a running job (with the reason)"; else echo "publish-jobs-check: --force did not remove"; fail=1; fi
|
||||
if REMOVE_GUARD_READ="$t/done.txt" bash "$P" remove guard-install --dest "$t/dest" >/dev/null 2>&1; then echo "publish-jobs-check: an --installs-app job was removed without --force"; fail=1; else echo "publish-jobs-check: an --installs-app job's removal is refused"; fi
|
||||
bash "$P" add --kind run --target 00000001 --script "$t/s.ps1" --id guard-run2 --title t --requires none --dest "$t/dest" >/dev/null 2>&1
|
||||
if REMOVE_GUARD_READ="$t/done.txt" bash "$P" remove guard-run2 --dest "$t/dest" >/dev/null 2>&1; then echo "publish-jobs-check: a finished job's removal passes"; else echo "publish-jobs-check: a finished job's removal was refused"; fail=1; fi
|
||||
[ "$fail" = 0 ] && echo "publish-jobs-check: a running or installs-app job is never removed without --force; a finished one is"
|
||||
exit $fail
|
||||
Loading…
Reference in a new issue