igneum/tools/observer/autosync.sh
igneum-labs 11c4426a96 Kill by exact command line or pid file, never by a name: tools/ci/kill-by-name-check.sh in the gate; the 36 pgrep/pkill literals in the tree fixed
The fleet's 22:09 UK incident (a Mac-side pkill -f <log file name> matched nothing, the roll-everything script lived on and wiped a held box) and the day's two pgrep self-matches are one class. The check flags pgrep -f / pkill -f with a plain literal (every one on a line), any pgrep/pkill on a file-name shape, and ps | grep with a literal; it allows the bracket form, -x, -F pidfile, kill $(cat pidfile), a variable and a full path; 11 banned and 16 allowed shapes in its self-test; 0.15 s over the tree. The 25 pkill -f sp1-gpu-server inside bash -c bodies (which matched the calling bash) are pkill -x; the other 11 literals take the bracket form; prover-socket-check accepts both. Row R in the record; the CLAUDE.md rule names the check and covers pkill and file names.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 22:10:31 +00:00

51 lines
3.6 KiB
Bash
Executable file

#!/usr/bin/env bash
# Keeps the shared checkout's observer and public API in step with origin/master, so a change an agent pushed to
# tools/observer or site/api is running here within five minutes (4 October 2026: the live API said "observer has
# not reported the proving layer" for an hour because the shared checkout was three commits behind origin).
# Fast-forwards only; if the shared checkout is ahead or dirty in those paths it logs and waits for a human.
# The observer is restarted whenever the checked-out observer.mjs or run.sh differs from what the running observer
# started from, whoever moved HEAD (4 October 2026, 19:35: a pull by hand moved HEAD past an observer fix and the
# observer kept the old code for ten minutes, because only this loop's own fast-forward used to restart it).
# Loop: tools/observer/autosync.sh (log: /tmp/igneum-devnet/autosync.out)
# Check: tools/observer/autosync.sh check prints the observer tree, the marker and whether a restart is due; exit 3 when due
cd "$(dirname "$0")/../.." || exit 1
LOG=/tmp/igneum-devnet/autosync.out
MARK="${OBSERVER_TREE_MARK:-/tmp/igneum-devnet/observer.tree}" # tree id of tools/observer the running observer was started from
PATHS="tools/observer site/api"
# the code the running observer executes: observer.mjs and its restart loop (not autosync.sh or the README, whose
# changes used to restart it for nothing, 20:15 on 4 October 2026)
observer_tree() { git rev-parse "HEAD:tools/observer/observer.mjs" "HEAD:tools/observer/run.sh" 2>/dev/null | tr -d '\n' | shasum | cut -c1-40; }
marker() { cat "$MARK" 2>/dev/null || echo none; }
restart_due() { local t; t="$(observer_tree)"; [ -n "$t" ] && [ "$(marker)" != "$t" ]; }
restart_observer() { # <reason>
local pid
pid=$(pgrep -f "[n]ode tools/observer/observer.mjs" | head -1)
if [ -n "$pid" ]; then kill "$pid" && echo "$(date -u +%FT%TZ) observer $pid restarted: $1"
else echo "$(date -u +%FT%TZ) no observer process to restart ($1): is tools/observer/run.sh running?"; fi
observer_tree > "$MARK"
}
if [ "${1:-}" = check ]; then
echo "observer tree $(observer_tree | cut -c1-12), marker $(marker | cut -c1-12), restart $(restart_due && echo due || echo "not due")"
restart_due && exit 3; exit 0
fi
mkdir -p "$(dirname "$LOG")"; exec >>"$LOG" 2>&1
while :; do
git fetch -q origin master 2>/dev/null
local_sha=$(git rev-parse HEAD); remote_sha=$(git rev-parse origin/master)
if [ "$local_sha" != "$remote_sha" ]; then
if [ -z "$(git status --porcelain -- $PATHS)" ] && git merge-base --is-ancestor HEAD origin/master; then
if git merge -q --ff-only origin/master 2>/tmp/igneum-devnet/autosync-ff.err; then
echo "$(date -u +%FT%TZ) fast-forwarded to $(git rev-parse --short HEAD)"
else
# name what blocks it: files dirty here that the incoming commits also change (4 October 2026, 20:05: results
# files edited in the shared checkout against a commit of the same files; the observer sat behind master)
blocking=$(comm -12 <(git status --porcelain | awk '{print $2}' | sort) <(git diff --name-only HEAD origin/master | sort) | head -5 | tr '\n' ' ')
echo "$(date -u +%FT%TZ) fast-forward failed (local ${local_sha:0:7}, origin ${remote_sha:0:7}): $(head -c 200 /tmp/igneum-devnet/autosync-ff.err | tr '\n' ' ') blocking: ${blocking:-none found}"
fi
else
echo "$(date -u +%FT%TZ) not fast-forwardable or dirty in $PATHS (local $local_sha, origin $remote_sha); waiting"
fi
fi
if restart_due; then restart_observer "tools/observer tree $(observer_tree | cut -c1-12) (was $(marker | cut -c1-12))"; fi
sleep 300
done