diff --git a/tools/lock/with-lock.sh b/tools/lock/with-lock.sh index 33e9f8ad1..c75f7be3e 100755 --- a/tools/lock/with-lock.sh +++ b/tools/lock/with-lock.sh @@ -1,36 +1,82 @@ #!/usr/bin/env bash # Serialises the two things that must not overlap on this Mac: BUILDS (cargo, swiftc, zigbuild, large cross-compiles) and -# MEASUREMENTS (test networks, hash-rate samples, simulators, anything whose numbers go in the bench log). -# tools/lock/with-lock.sh build one build at a time (builds also nice -n 19, 4 jobs) -# tools/lock/with-lock.sh measure one measurement at a time, and no build may start meanwhile +# MEASUREMENTS (hash-rate samples, millisecond latencies, power: anything a build on the same machine could disturb). +# tools/lock/with-lock.sh build one of THREE build slots (each build is nice -n 19 with 4 jobs, so +# three at once use 12 of the 16 cores); waits when all are busy +# tools/lock/with-lock.sh measure exclusive: waits for every build slot and blocks new builds # tools/lock/with-lock.sh run a functional run (test network, attack harness, simulator whose -# outputs are counts, locks, forks, seconds): one at a time, but -# builds may continue; nothing hash-rate, latency-in-ms or power -# Rule (4 October 2026): code and documents run in parallel; builds and measurements queue. Waits up to 2 hours. -# Evening of 4 October: a 3-hour finality network run held the build lock with six agents queued behind it, so -# functional runs now take the measure lock only (`run`); `measure` stays for numbers a nice-19 build could disturb. +# outputs are counts, locks, forks, seconds): one at a time, builds continue +# tools/lock/with-lock.sh status who holds what, and for how long +# Rule (4 October 2026): code and documents run in parallel; builds share three slots; measurements are exclusive. +# Evening of 4 October: one build slot and a 3-hour network run under `measure` left twelve agents queued; hence the +# three slots, the `run` mode and `status`. Waits up to 2 hours. This file is replaced atomically (never edited in +# place) because waiting instances are still reading it. set -euo pipefail -kind="$1"; shift +kind="${1:-}"; shift || true dir=/tmp/igneum-locks; mkdir -p "$dir" -case "$kind" in - build) locks=("$dir/build");; - measure) locks=("$dir/measure" "$dir/build");; - run) locks=("$dir/measure");; - *) echo "usage: with-lock.sh build|measure|run " >&2; exit 2;; -esac -fds=(); i=9 -for l in "${locks[@]}"; do - i=$((i+1)); eval "exec $i>\"$l\"" - if ! python3 - "$i" "$l" <<'PY' -import fcntl, sys, time, os -fd=int(sys.argv[1]); name=sys.argv[2]; t=time.time() -while True: - try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB); break - except BlockingIOError: - if time.time()-t > 7200: print(f"with-lock: gave up waiting for {name} after 2 h", file=sys.stderr); sys.exit(1) - time.sleep(5) +if [ "$kind" = status ]; then + for f in measure build build-0 build-1 build-2; do + [ -e "$dir/$f" ] || continue + holder=$(python3 - "$dir/$f" <<'PY' +import fcntl, sys, os +fd=os.open(sys.argv[1], os.O_RDWR) +try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB); fcntl.flock(fd, fcntl.LOCK_UN); print("free") +except BlockingIOError: print("HELD") +PY +) + printf '%-8s %s' "$f" "$holder" + if [ "$holder" = HELD ]; then + # the holder wrote its pid and start time into the lock file + printf ' %s' "$(tr '\n' ' ' < "$dir/$f" | cut -c1-140)" + fi + echo + done + echo "waiting:"; ps -eo pid,etime,command | grep -E "with-lock.sh (build|measure|run) " | grep -v grep | awk '{printf " %s %s %s %s\n", $1, $2, $4, $5}' | head -20 + exit 0 +fi +case "$kind" in + build) mode=build;; + measure) mode=measure;; + run) mode=run;; + *) echo "usage: with-lock.sh build|measure|run | status" >&2; exit 2;; +esac +# Python holds the locks for the lifetime of this process: it writes the holder line, then execs the command with the +# file descriptors inherited (flock locks follow the open file, so the command keeps them until it exits). +exec python3 - "$mode" "$dir" "$@" <<'PY' +import fcntl, sys, time, os +mode, d = sys.argv[1], sys.argv[2]; cmd = sys.argv[3:] +t0 = time.time() +def opened(name): + fd = os.open(os.path.join(d, name), os.O_RDWR | os.O_CREAT, 0o644) + os.set_inheritable(fd, True); return fd +def try_lock(fd): + try: fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB); return True + except BlockingIOError: return False +def wait_lock(fd, name): + while not try_lock(fd): + if time.time() - t0 > 7200: print(f"with-lock: gave up waiting for {name} after 2 h", file=sys.stderr); sys.exit(1) + time.sleep(5) +held = [] +if mode == 'build': + slots = [opened(f"build-{k}") for k in range(3)] + got = None + while got is None: + for k, fd in enumerate(slots): + if try_lock(fd): got = (k, fd); break + if got is None: + if time.time() - t0 > 7200: print("with-lock: gave up waiting for a build slot after 2 h", file=sys.stderr); sys.exit(1) + time.sleep(5) + for k, fd in enumerate(slots): + if got[0] != k: os.close(fd) + held = [(f"build-{got[0]}", got[1])] +elif mode == 'measure': + for name in ['measure', 'build', 'build-0', 'build-1', 'build-2']: + fd = opened(name); wait_lock(fd, name); held.append((name, fd)) +else: + fd = opened('measure'); wait_lock(fd, 'measure'); held.append(('measure', fd)) +for name, fd in held: + os.ftruncate(fd, 0); os.lseek(fd, 0, 0) + os.write(fd, f"pid {os.getpid()} since {time.strftime('%H:%M:%S', time.gmtime())}Z waited {int(time.time()-t0)} s: {' '.join(cmd)[:120]}\n".encode()) +print(f"with-lock: holding {' '.join(n for n, _ in held)} for: {' '.join(cmd)}", file=sys.stderr) +os.execvp(cmd[0], cmd) PY - then exit 1; fi -done -echo "with-lock: holding ${locks[*]##*/} for: $*" >&2 -exec "$@"