Lock: three build slots, exclusive measure, run mode, status command; replaced atomically

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tools/lock/with-lock.sh

@@ -1,36 +1,82 @@
 #!/usr/bin/env bash
 # Serialises the two things that must not overlap on this Mac: BUILDS (cargo, swiftc, zigbuild, large cross-compiles) and
-# MEASUREMENTS (test networks, hash-rate samples, simulators, anything whose numbers go in the bench log).
-#   tools/lock/with-lock.sh build   <command...>      one build at a time (builds also nice -n 19, 4 jobs)
-#   tools/lock/with-lock.sh measure <command...>      one measurement at a time, and no build may start meanwhile
+# MEASUREMENTS (hash-rate samples, millisecond latencies, power: anything a build on the same machine could disturb).
+#   tools/lock/with-lock.sh build   <command...>      one of THREE build slots (each build is nice -n 19 with 4 jobs, so
+#                                                     three at once use 12 of the 16 cores); waits when all are busy
+#   tools/lock/with-lock.sh measure <command...>      exclusive: waits for every build slot and blocks new builds
 #   tools/lock/with-lock.sh run     <command...>      a functional run (test network, attack harness, simulator whose
-#                                                     outputs are counts, locks, forks, seconds): one at a time, but
-#                                                     builds may continue; nothing hash-rate, latency-in-ms or power
-# Rule (4 October 2026): code and documents run in parallel; builds and measurements queue. Waits up to 2 hours.
-# Evening of 4 October: a 3-hour finality network run held the build lock with six agents queued behind it, so
-# functional runs now take the measure lock only (`run`); `measure` stays for numbers a nice-19 build could disturb.
+#                                                     outputs are counts, locks, forks, seconds): one at a time, builds continue
+#   tools/lock/with-lock.sh status                    who holds what, and for how long
+# Rule (4 October 2026): code and documents run in parallel; builds share three slots; measurements are exclusive.
+# Evening of 4 October: one build slot and a 3-hour network run under `measure` left twelve agents queued; hence the
+# three slots, the `run` mode and `status`. Waits up to 2 hours. This file is replaced atomically (never edited in
+# place) because waiting instances are still reading it.
 set -euo pipefail
-kind="$1"; shift
+kind="${1:-}"; shift || true
 dir=/tmp/igneum-locks; mkdir -p "$dir"
-case "$kind" in
-  build)   locks=("$dir/build");;
-  measure) locks=("$dir/measure" "$dir/build");;
-  run)     locks=("$dir/measure");;
-  *) echo "usage: with-lock.sh build|measure|run <command...>" >&2; exit 2;;
-esac
-fds=(); i=9
-for l in "${locks[@]}"; do
-  i=$((i+1)); eval "exec $i>\"$l\""
-  if ! python3 - "$i" "$l" <<'PY'
-import fcntl, sys, time, os
-fd=int(sys.argv[1]); name=sys.argv[2]; t=time.time()
-while True:
-    try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB); break
-    except BlockingIOError:
-        if time.time()-t > 7200: print(f"with-lock: gave up waiting for {name} after 2 h", file=sys.stderr); sys.exit(1)
-        time.sleep(5)
+if [ "$kind" = status ]; then
+  for f in measure build build-0 build-1 build-2; do
+    [ -e "$dir/$f" ] || continue
+    holder=$(python3 - "$dir/$f" <<'PY'
+import fcntl, sys, os
+fd=os.open(sys.argv[1], os.O_RDWR)
+try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB); fcntl.flock(fd, fcntl.LOCK_UN); print("free")
+except BlockingIOError: print("HELD")
+PY
+)
+    printf '%-8s %s' "$f" "$holder"
+    if [ "$holder" = HELD ]; then
+      # the holder wrote its pid and start time into the lock file
+      printf '  %s' "$(tr '\n' ' ' < "$dir/$f" | cut -c1-140)"
+    fi
+    echo
+  done
+  echo "waiting:"; ps -eo pid,etime,command | grep -E "with-lock.sh (build|measure|run) " | grep -v grep | awk '{printf "  %s %s %s %s\n", $1, $2, $4, $5}' | head -20
+  exit 0
+fi
+case "$kind" in
+  build)   mode=build;;
+  measure) mode=measure;;
+  run)     mode=run;;
+  *) echo "usage: with-lock.sh build|measure|run <command...> | status" >&2; exit 2;;
+esac
+# Python holds the locks for the lifetime of this process: it writes the holder line, then execs the command with the
+# file descriptors inherited (flock locks follow the open file, so the command keeps them until it exits).
+exec python3 - "$mode" "$dir" "$@" <<'PY'
+import fcntl, sys, time, os
+mode, d = sys.argv[1], sys.argv[2]; cmd = sys.argv[3:]
+t0 = time.time()
+def opened(name):
+    fd = os.open(os.path.join(d, name), os.O_RDWR | os.O_CREAT, 0o644)
+    os.set_inheritable(fd, True); return fd
+def try_lock(fd):
+    try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB); return True
+    except BlockingIOError: return False
+def wait_lock(fd, name):
+    while not try_lock(fd):
+        if time.time() - t0 > 7200: print(f"with-lock: gave up waiting for {name} after 2 h", file=sys.stderr); sys.exit(1)
+        time.sleep(5)
+held = []
+if mode == 'build':
+    slots = [opened(f"build-{k}") for k in range(3)]
+    got = None
+    while got is None:
+        for k, fd in enumerate(slots):
+            if try_lock(fd): got = (k, fd); break
+        if got is None:
+            if time.time() - t0 > 7200: print("with-lock: gave up waiting for a build slot after 2 h", file=sys.stderr); sys.exit(1)
+            time.sleep(5)
+    for k, fd in enumerate(slots):
+        if got[0] != k: os.close(fd)
+    held = [(f"build-{got[0]}", got[1])]
+elif mode == 'measure':
+    for name in ['measure', 'build', 'build-0', 'build-1', 'build-2']:
+        fd = opened(name); wait_lock(fd, name); held.append((name, fd))
+else:
+    fd = opened('measure'); wait_lock(fd, 'measure'); held.append(('measure', fd))
+for name, fd in held:
+    os.ftruncate(fd, 0); os.lseek(fd, 0, 0)
+    os.write(fd, f"pid {os.getpid()} since {time.strftime('%H:%M:%S', time.gmtime())}Z waited {int(time.time()-t0)} s: {' '.join(cmd)[:120]}\n".encode())
+print(f"with-lock: holding {' '.join(n for n, _ in held)} for: {' '.join(cmd)}", file=sys.stderr)
+os.execvp(cmd[0], cmd)
 PY
-  then exit 1; fi
-done
-echo "with-lock: holding ${locks[*]##*/} for: $*" >&2
-exec "$@"