From 73fc2fd5785accaba2dbc81f1c049bf8573a936d Mon Sep 17 00:00:00 2001 From: igneum-josh <337424239+igneum-josh@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:48:44 +0100 Subject: [PATCH] Observer autosync restarts the observer whenever the checked-out tools/observer tree changes (marker + check mode); console stale mark at 180 s (one missed upload is not stale); bugs.md rows Co-Authored-By: Claude Fable 5.1 --- docs/bugs.md | 3 +++ relay/lib/parse.mjs | 10 ++++++---- relay/test/parse.test.mjs | 4 +++- tools/observer/autosync.sh | 28 ++++++++++++++++++++++------ 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/docs/bugs.md b/docs/bugs.md index 69f945806..a32c627e6 100644 --- a/docs/bugs.md +++ b/docs/bugs.md @@ -15,6 +15,9 @@ shard run reported as exit 0, 7a7e873). | 4 Oct 2026 | `vercel env add` from `site/` fails with "Could not retrieve Project Settings" | `site/.vercel/project.json` links the vivanmn team's `igneum` project; the live site (igneum.network, igneum.com, the GitHub integration) is the `igneum` team's project of the same name, which the igneum login reads and the vivanmn link does not | documented in `packaging/README-ship.md` (link, env ls, env add, deploy); no env set | `env ls` from a scratch link to the igneum-team project lists the two names; a branch push produced `igneum-git-` | | 4 Oct 2026 | `tools/jobs.mjs` and `publish-jobs.sh` print the downloads-folder token inside URLs on every run (X24 pattern) | the tokened base URL is echoed as is | the token masked as `` in every printed URL | by eye, this log's own transcript | | 4 Oct 2026 | round 4 X28 and X24, the parts under an hour: `===` on secrets, no HSTS on the relay, the relay token printed by `tools/relay.mjs list` and `watch` | as the review said | c1f59fb: `sameSecret` (timingSafeEqual, `relay/lib/auth.mjs`, test in CI), `Strict-Transport-Security` in `relay/vercel.json`, `/r/` printed (only `url` prints the real one) | relay deployed: key auth 200, wrong key and token 401, token path 200, HSTS header present | +| 4 Oct 2026 | the live feed showed two "checkpoint N locked" events 30 ms apart (1122, 1172, 1230, 1258, 1259 in 400 observer lines), and 708 of 764 locked checkpoints in `live_checkpoints` had `votes_seen` 0 | `finalityTick` read the state, awaited two SQL writes, then set the map; the `finalityLockNotification` handler checked the same map synchronously in between and recorded the lock too; a lock claimed by the notification was never upserted again, so the poll's `votes_seen` never landed | 7de1bdb: the poll claims the state before its first await; a `checkpointDetailed` set makes the poll fill `votes_seen` once | duplicates before the restart: 5 in 400 lines; after: see the next row's restart and pass 5 | +| 4 Oct 2026 | the observer kept running old code after its fix was on master and HEAD had moved past it | `autosync.sh` restarted the observer only when its own fast-forward moved HEAD; a pull by hand (19:35 UTC, HEAD to 8a77b85) bypassed it | autosync compares the checked-out `tools/observer` tree id with a marker written at each restart and restarts on any difference; `autosync.sh check` says what it would do | `check` with no marker: "restart due", exit 3; with the marker equal to the tree: "not due", exit 0; the shared checkout at that moment: due | +| 4 Oct 2026 | the Mac card read 0.0 MH/s for a minute while its blocks were still accepted (after the stale mark shipped) | STALE_S 120 s left no margin: one missed 60 s upload (a 120 s gap at 19:45:59 UTC) plus a 30 s STATUS age plus the 10 s cache | STALE_S 180 s (one missed upload is not stale; PC 2's real case was a 1,860 s gap) | test: 150 s is fresh, 240 s stale | ## Open diff --git a/relay/lib/parse.mjs b/relay/lib/parse.mjs index 1b8dc49a7..e185bd489 100644 --- a/relay/lib/parse.mjs +++ b/relay/lib/parse.mjs @@ -5,10 +5,12 @@ export const kvNum = (text, k) => { const v = kv(text, k); return v === null ? n export const lastMatch = (lines, re) => { for (let i = lines.length - 1; i >= 0; i--) { const m = re.exec(lines[i]); if (m) return m; } return null; }; export const FAULT = /WORKER MISMATCH|worker error|worker exited|panicked|CUDA error|submit error/; -// A card's hash is "now" only while its worker's STATUS line (every 30 s) is this fresh; older than this the card is -// marked stale and its hash leaves the machine total (4 October 2026: PC 2 showed 117 MH/s "now" from a STATUS line -// four minutes old while the shard prover held the GPU). -export const STALE_S = 120; +// A card's hash is "now" only while its worker's STATUS line (every 30 s, uploaded every 60 s) is this fresh; older +// than this the card is marked stale and its hash leaves the machine total (4 October 2026: PC 2 showed 117 MH/s "now" +// from a STATUS line four minutes old while the shard prover held the GPU; its upload gap was 1,860 s). One missed +// upload (120 s gap, seen on the Mac the same evening) plus a 30 s STATUS age plus the 10 s API cache is 160 s, so the +// mark starts at 180 s, the same as SILENT_S for a machine. +export const STALE_S = 180; export function markStale(cards, now = Date.now()) { for (const c of cards) { const age = c.status_at ? Math.round((now - Date.parse(c.status_at)) / 1000) : null; diff --git a/relay/test/parse.test.mjs b/relay/test/parse.test.mjs index 7d7b84edc..ade4eadf5 100644 --- a/relay/test/parse.test.mjs +++ b/relay/test/parse.test.mjs @@ -37,5 +37,7 @@ test('stale: a card is "now" within STALE_S of its STATUS line and stale after i assert.equal(fresh.stale, false); assert.equal(fresh.status_age_s, 58); assert.equal(old.stale, true); assert.equal(old.status_age_s, 240); assert.equal(none.stale, true); assert.equal(none.status_age_s, null); - assert.ok(STALE_S >= 90, 'two missed 30 s STATUS lines are not stale yet'); + assert.ok(STALE_S >= 160, 'one missed 60 s upload plus a 30 s STATUS age plus the 10 s cache is not stale'); + const oneMissed = { status_at: new Date(now - 150_000).toISOString(), hash_now: 18 }; + markStale([oneMissed], now); assert.equal(oneMissed.stale, false); }); diff --git a/tools/observer/autosync.sh b/tools/observer/autosync.sh index 48cb9b05f..78df973e7 100755 --- a/tools/observer/autosync.sh +++ b/tools/observer/autosync.sh @@ -3,22 +3,37 @@ # tools/observer or site/api is running here within five minutes (4 October 2026: the live API said "observer has # not reported the proving layer" for an hour because the shared checkout was three commits behind origin). # Fast-forwards only; if the shared checkout is ahead or dirty in those paths it logs and waits for a human. +# The observer is restarted whenever the checked-out tools/observer tree differs from the one the running observer +# started from, whoever moved HEAD (4 October 2026, 19:35: a pull by hand moved HEAD past an observer fix and the +# observer kept the old code for ten minutes, because only this loop's own fast-forward used to restart it). # Loop: tools/observer/autosync.sh (log: /tmp/igneum-devnet/autosync.out) +# Check: tools/observer/autosync.sh check prints the observer tree, the marker and whether a restart is due; exit 3 when due cd "$(dirname "$0")/../.." || exit 1 -LOG=/tmp/igneum-devnet/autosync.out; mkdir -p "$(dirname "$LOG")"; exec >>"$LOG" 2>&1 +LOG=/tmp/igneum-devnet/autosync.out +MARK="${OBSERVER_TREE_MARK:-/tmp/igneum-devnet/observer.tree}" # tree id of tools/observer the running observer was started from PATHS="tools/observer site/api" +observer_tree() { git rev-parse "HEAD:tools/observer" 2>/dev/null; } +marker() { cat "$MARK" 2>/dev/null || echo none; } +restart_due() { local t; t="$(observer_tree)"; [ -n "$t" ] && [ "$(marker)" != "$t" ]; } +restart_observer() { # + local pid + pid=$(pgrep -f "node tools/observer/observer.mjs" | head -1) + if [ -n "$pid" ]; then kill "$pid" && echo "$(date -u +%FT%TZ) observer $pid restarted: $1" + else echo "$(date -u +%FT%TZ) no observer process to restart ($1): is tools/observer/run.sh running?"; fi + observer_tree > "$MARK" +} +if [ "${1:-}" = check ]; then + echo "observer tree $(observer_tree | cut -c1-12), marker $(marker | cut -c1-12), restart $(restart_due && echo due || echo "not due")" + restart_due && exit 3; exit 0 +fi +mkdir -p "$(dirname "$LOG")"; exec >>"$LOG" 2>&1 while :; do git fetch -q origin master 2>/dev/null local_sha=$(git rev-parse HEAD); remote_sha=$(git rev-parse origin/master) if [ "$local_sha" != "$remote_sha" ]; then if [ -z "$(git status --porcelain -- $PATHS)" ] && git merge-base --is-ancestor HEAD origin/master; then - changed=$(git diff --name-only HEAD origin/master -- $PATHS) if git merge -q --ff-only origin/master; then echo "$(date -u +%FT%TZ) fast-forwarded to $(git rev-parse --short HEAD)" - if [ -n "$changed" ]; then - pid=$(pgrep -f "node tools/observer/observer.mjs" | head -1) - if [ -n "$pid" ]; then kill "$pid" && echo "$(date -u +%FT%TZ) observer $pid restarted for: $(echo "$changed" | tr '\n' ' ')"; fi - fi else echo "$(date -u +%FT%TZ) fast-forward failed (local $local_sha, origin $remote_sha)" fi @@ -26,5 +41,6 @@ while :; do echo "$(date -u +%FT%TZ) not fast-forwardable or dirty in $PATHS (local $local_sha, origin $remote_sha); waiting" fi fi + if restart_due; then restart_observer "tools/observer tree $(observer_tree | cut -c1-12) (was $(marker | cut -c1-12))"; fi sleep 300 done