From 11c4426a96bbed80ef51574f5dbaec2ff157b6c9 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:10:31 +0000 Subject: [PATCH] Kill by exact command line or pid file, never by a name: tools/ci/kill-by-name-check.sh in the gate; the 36 pgrep/pkill literals in the tree fixed The fleet's 22:09 UK incident (a Mac-side pkill -f matched nothing, the roll-everything script lived on and wiped a held box) and the day's two pgrep self-matches are one class. The check flags pgrep -f / pkill -f with a plain literal (every one on a line), any pgrep/pkill on a file-name shape, and ps | grep with a literal; it allows the bracket form, -x, -F pidfile, kill $(cat pidfile), a variable and a full path; 11 banned and 16 allowed shapes in its self-test; 0.15 s over the tree. The 25 pkill -f sp1-gpu-server inside bash -c bodies (which matched the calling bash) are pkill -x; the other 11 literals take the bracket form; prover-socket-check accepts both. Row R in the record; the CLAUDE.md rule names the check and covers pkill and file names. Co-Authored-By: Claude Fable 5.1 --- docs/analysis/ci-failures-2026-10-06.md | 9 +- infra/build-server/hands/move-hand.sh | 4 +- infra/devnet/restart-hand-nodes.sh | 2 +- relay/playbooks/shard-test.ps1 | 2 +- tools/ci/fixtures/bash-body-ok.ps1 | 2 +- tools/ci/kill-by-name-check.sh | 99 +++++++++++++++++++ tools/ci/pre-push.sh | 1 + tools/ci/prover-socket-check.sh | 3 +- tools/exec-attacks/lib/common.mjs | 2 +- tools/exec-attacks/net.sh | 4 +- tools/lock/with-lock.sh | 2 +- tools/observer/autosync.sh | 2 +- tools/prover-floor/pc2-floor-measure.ps1 | 8 +- tools/prover-floor/pc2-floor-sweep1.ps1 | 8 +- tools/prover-floor/pc2-floor-sweep2.ps1 | 8 +- tools/prover-floor/pc2-floor-sweep3-miner.ps1 | 8 +- tools/proving-v1/pc2-agg-cost.ps1 | 2 +- tools/proving-v1/pc2-chain.ps1 | 2 +- tools/proving-v1/pc2-memory-miner-on.ps1 | 6 +- tools/proving-v1/pc2-memory-sweep.ps1 | 6 +- tools/proving-v1/pc2-segments.ps1 | 4 +- tools/proving-v1/pc2-socket-fix.ps1 | 2 +- tools/proving-v1/pc2-sp-curve.ps1 | 6 +- 23 files changed, 147 insertions(+), 45 deletions(-) create mode 100755 tools/ci/kill-by-name-check.sh diff --git a/docs/analysis/ci-failures-2026-10-06.md b/docs/analysis/ci-failures-2026-10-06.md index 324a5053..18c9e45e 100644 --- a/docs/analysis/ci-failures-2026-10-06.md +++ b/docs/analysis/ci-failures-2026-10-06.md @@ -41,6 +41,7 @@ One line per class. "Guard" is what now stops the class before it reaches master | N no-secrets | 2 | 06 15:56Z | 06 16:23Z | A 64-hex test vector next to `private_key` in `app/igneum-wallet/src/vault.rs` (wallet-0.1.5) | Allow-listed as a test value | The gate | | P swallowed defaults line (no CI run: a silent class) | 0 | 06 19:5xZ | 06 21:xxZ | A comment appended to a line of shell assignments in `tools/build-remote.sh` and then `tools/cross-remote.sh` turned every assignment after the `#` into comment text; `bash -n` and shellcheck are silent on it; the default cross-build never ran and its chain kept the previous exes from about 20:40 to 22:00 UK | 36e4ee7 on master: the lines split; `tools/ci/defaults-line-check.sh` with its self-test | In ci.yml at 36e4ee7 and in the gate from this change, so it runs on the pushing machine before the push | | Q gate checks that read the machine, not the fact (found by the gate itself, 22:1x to 22:3x UK) | 0 | 06 21:1xZ | 06 21:3xZ | Two pushes of this change to master were refused by the new hook: (1) git hands a hook `GIT_DIR`, and `remote-run.sh --self-test`'s nested `git init`, commits and reset then acted on THIS repository's worktree: it set `core.bare`, moved the local `master` to three fixture commits and broke the main checkout for ten minutes (restored from the reflog: master back to 36e4ee7, `core.bare false`; nothing was pushed, nothing lost); (2) the same self-test judged a stale `index.lock` by `pgrep -x git` over the whole machine, so the hook's own `git push` (or any other agent's git) made the fixture's checkout die with "index.lock: File exists" | The gate unsets `GIT_DIR` and the other hook variables before any check; the staleness test is the lock's age (over 30 s), and the self-test backdates its fixture lock | The gate itself: every self-test now runs inside a real hook before every master push, with a `git push` alive beside it | +| R kill by name (the fleet, no CI run) | 0 | 06 21:09Z | 06 21:09Z | A Mac-side `pkill -f ` matched nothing: the name was a shell redirect, not part of any command line; the roll-everything script lived on and wiped a box it had been told to hold. Earlier the same day, twice: a `pgrep -f ""` matched the calling shell's own command line. Six `pkill -f sp1-gpu-server` inside `bash -c '...'` bodies in tools/proving-v1 carried the same shape on master | The fleet runs Mac-side jobs under `tools/fleet/fleet-bg.sh` (a pid file per job) and anchors every on-box kill on the binary's full path and first argument; the six prover lines use `pkill -x` on the binary name | `tools/ci/kill-by-name-check.sh` in the gate: flags `pgrep -f` / `pkill -f` with a plain literal, any pgrep/pkill on a file-name shape (.log, .out, .pid, .json ...), and `ps | grep `; allows the bracket form, `-x`, `-F `, `kill $(cat pidfile)`, a variable, a full path; self-test of 9 banned and 14 allowed shapes | | O1 windows-ci cancelled | 16 | 04 10:42Z | 06 18:12Z | `concurrency: cancel-in-progress` on windows.yml: a newer master push superseded the run. Not a failure | None needed | None; they are listed because `gh run list` counts them as non-green | | O2 hosted runner not acquired | 8 | 05 19:26Z | 05 20:54Z | "The job was not acquired by Runner of type hosted even after multiple attempts" on release-0.3.10 (7) and master (1): GitHub capacity, retried by hand | Re-run | The `red` job reports it; the box runner for `pow` and `sims` (section 5) takes those jobs off the hosted pool | @@ -57,10 +58,10 @@ is the wrong place; it goes in the script. | Mode | When | What | |---|---|---| -| `--hook` on a push to master or release-* | installed by `tools/ci/install-hooks.sh` into the shared hooks directory (one set for every worktree) | all 31 checks; a red check refuses the push and prints its output | +| `--hook` on a push to master or release-* | installed by `tools/ci/install-hooks.sh` into the shared hooks directory (one set for every worktree) | all 32 checks; a red check refuses the push and prints its output | | `--hook` on any other ref | same | the two structural checks only (conflict markers, Windows paths) | -| `--ci` | the `site` job | all 31 checks, with the site built in place | -| default | by hand in any worktree | all 31 checks | +| `--ci` | the `site` job | all 32 checks, with the site built in place | +| default | by hand in any worktree | all 32 checks | | `--self-test` | in the gate itself | a known failure is RED and fails the gate; a known success is ok; master and release-* select the full gate, other refs the light one | Measured 6 October 2026, 21:5x UK, on the Mac: 30 checks, GREEN, 25 s (no-secrets 11 s, identity grep 3 s, the rest @@ -70,7 +71,7 @@ worktrees); `git status` before and after the full gate is identical. Checks that joined CI through the gate and were not in ci.yml before: the ledger sentence check (`ledger-text-check.mjs`), the workflow shell parse (`check-workflow-shell.mjs`), the Windows paths check, and the three -self-tests (identity, Windows paths, the red watcher). The swallowed-defaults check (36e4ee7) is in the gate too. +self-tests (identity, Windows paths, the red watcher). The swallowed-defaults check (36e4ee7) and the kill-by-name check are in the gate too. ## 4. The red watcher (`tools/ci/red-watch.mjs`, `infra/build-server/ci-red/`) diff --git a/infra/build-server/hands/move-hand.sh b/infra/build-server/hands/move-hand.sh index 170a076d..87f4ac57 100755 --- a/infra/build-server/hands/move-hand.sh +++ b/infra/build-server/hands/move-hand.sh @@ -124,10 +124,10 @@ case "$MODE" in run launchctl bootout "gui/$(id -u)/$label" 2>/dev/null || true run mv -f "$HOME/Library/LaunchAgents/$label.plist" "$HOME/Library/LaunchAgents/$label.plist.moved-to-build-1-$(date -u +%Y%m%d)" done - say "Mac igneumd processes now: $(pgrep -fl 'igneumd --' | grep -v Wallet | wc -l | tr -d ' ') (the wallet's own node is not a hand)" ;; + say "Mac igneumd processes now: $(pgrep -fl '[i]gneumd --' | grep -v Wallet | wc -l | tr -d ' ') (the wallet's own node is not a hand)" ;; status) echo "--- box:"; rssh "for u in igneum-node1 igneum-observer-node igneum-observer igneum-observer-sync.timer; do printf '%-26s %s\n' \$u \$(systemctl is-active \$u); done; journalctl -u igneum-node1 -o cat -n 2 --no-pager 2>/dev/null | cut -c1-160; journalctl -u igneum-observer -o cat -n 2 --no-pager 2>/dev/null | cut -c1-160" - echo "--- mac:"; launchctl print "gui/$(id -u)/network.igneum.devnet.node1" 2>/dev/null | grep -E 'state|pid' | head -2; launchctl print "gui/$(id -u)/network.igneum.devnet.observer" 2>/dev/null | grep -E 'state|pid' | head -2; pgrep -fl 'observer.mjs|observer/run.sh|autosync.sh' || echo "no observer processes on the Mac" ;; + echo "--- mac:"; launchctl print "gui/$(id -u)/network.igneum.devnet.node1" 2>/dev/null | grep -E 'state|pid' | head -2; launchctl print "gui/$(id -u)/network.igneum.devnet.observer" 2>/dev/null | grep -E 'state|pid' | head -2; pgrep -fl '[o]bserver.mjs|[o]bserver/run.sh|[a]utosync.sh' || echo "no observer processes on the Mac" ;; *) sed -n '2,20p' "$0"; exit 2 ;; esac diff --git a/infra/devnet/restart-hand-nodes.sh b/infra/devnet/restart-hand-nodes.sh index ae382ce5..2b6e4467 100755 --- a/infra/devnet/restart-hand-nodes.sh +++ b/infra/devnet/restart-hand-nodes.sh @@ -39,4 +39,4 @@ reset_exec /tmp/igneum-devnet/node1 nohup caffeinate -dims "$BIN" --devnet --nodnsseed --disable-upnp --enable-unsynced-mining --appdir=/tmp/igneum-devnet/node1 --rpclisten=0.0.0.0:26610 --listen=0.0.0.0:26611 --evm-rpclisten=127.0.0.1:26791 \ --addpeer=188.245.5.161:26611 --addpeer=192.168.68.67:26611 --override-params-file="$OV" $SNAPARG --nologfiles --yes >> /tmp/igneum-devnet/node1.out 2>&1 & lines /tmp/igneum-devnet/node1.out -echo "running now:"; ps -o pid=,lstart=,command= -p "$(pgrep -f 'igneumd --devnet' | tr '\n' ',' | sed 's/,$//')" | cut -c1-160 +echo "running now:"; ps -o pid=,lstart=,command= -p "$(pgrep -f '[i]gneumd --devnet' | tr '\n' ',' | sed 's/,$//')" | cut -c1-160 diff --git a/relay/playbooks/shard-test.ps1 b/relay/playbooks/shard-test.ps1 index 2535aa11..4a357c0e 100644 --- a/relay/playbooks/shard-test.ps1 +++ b/relay/playbooks/shard-test.ps1 @@ -125,7 +125,7 @@ try { if ((Get-Date) -ge $proveDeadline) { Say 'time budget reached: ending the proof run' Stop-Process -Id $p.Id -Force -ErrorAction SilentlyContinue - & wsl.exe -d $distro -u $wslUser -- bash -c 'pkill -f igneum-prove-host; pkill -f prove-shard.sh; true' 2>&1 | Out-Null + & wsl.exe -d $distro -u $wslUser -- bash -c 'pkill -f "[i]gneum-prove-host"; pkill -f "[p]rove-shard.sh"; true' 2>&1 | Out-Null break } Start-Sleep 30 diff --git a/tools/ci/fixtures/bash-body-ok.ps1 b/tools/ci/fixtures/bash-body-ok.ps1 index ff7062d9..c74f0069 100644 --- a/tools/ci/fixtures/bash-body-ok.ps1 +++ b/tools/ci/fixtures/bash-body-ok.ps1 @@ -36,7 +36,7 @@ $mk = 'id igneum >/dev/null 2>&1 || (useradd -m -s /bin/bash igneum && echo igne # 5. a concatenation in parentheses with a variable in the middle (the shard-test.ps1 shape) & wsl.exe -d $distro -u igneum -- bash -lc ("sudo -n true 2>/dev/null || echo 'sudo asks for a password'; cd " + $linuxDir + " && bash ./setup-wsl.sh") 2>&1 | ForEach-Object { "$_" } # 6. the Start-Process argument list, -c as its own quoted argument -$p = Start-Process -FilePath 'wsl.exe' -ArgumentList @('-d', $distro, '--', 'bash', '-c', 'pkill -f igneum-prove-host; pkill -f prove-shard.sh; true') -NoNewWindow -PassThru +$p = Start-Process -FilePath 'wsl.exe' -ArgumentList @('-d', $distro, '--', 'bash', '-c', 'pkill -f "[i]gneum-prove-host"; pkill -f "[p]rove-shard.sh"; true') -NoNewWindow -PassThru # 7. a single-quoted here-string piped to Set-Content, the file then run by a derived path $body = @' set -euo pipefail diff --git a/tools/ci/kill-by-name-check.sh b/tools/ci/kill-by-name-check.sh new file mode 100755 index 00000000..02f97e79 --- /dev/null +++ b/tools/ci/kill-by-name-check.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Kill or find a process by exact command line or pid file, never by a name (CLAUDE.md, 6 October 2026). +# +# Two incidents the same day: a `pgrep -f ""` whose literal sat in the calling shell's own command line matched +# itself, so the check always passed and no node ever started (twice, lunchtime and 17:1x UTC); and at 22:09 UK a Mac-side +# `pkill -f ` matched nothing (the name was a shell redirect, not part of any command line), the +# roll-everything script lived on and wiped a box it had been told to hold. +# +# Rule, as this check reads it, over every script in the tree (sh, bash, mjs, js, py, ps1; comments skipped): +# 1. `pgrep -f` or `pkill -f` with a plain literal pattern is flagged. Allowed: the bracket form (`[i]gneumd`, which never +# matches its own command line), `-x` on a binary name, `pkill -F ` / `kill $(cat )`, and a pattern +# that is a variable or starts with a slash (an anchored full path: "the binary's full path and first argument"). +# 2. Any pgrep/pkill pattern, bracketed or not, whose literal looks like a FILE NAME (an extension such as .log, .txt, +# .json, .jsonl, .out, .err, .pid, .csv, .md, .toml, .yml) is flagged: a file name is a redirect or an argument, and a +# redirect is never on a command line. +# 3. `ps ... | grep ` without the bracket form is flagged (the same self-match). +# +# tools/ci/kill-by-name-check.sh # exit 1 with file:line and the reason +# tools/ci/kill-by-name-check.sh --self-test # fires on each banned shape, passes each allowed one +set -euo pipefail + +check_line() { # -> prints the reason, returns 1, when the line carries a banned shape; returns 0 otherwise + local line="$1" code pat + code="${line%%#*}" # a comment is not code (a line that starts with // or * is a comment too) + [[ "$code" =~ ^[[:space:]]*(//|\*|/\*) ]] && return 0 + [[ "$line" =~ ^[[:space:]]*(//|\*|/\*) ]] && return 0 + # every pgrep/pkill -f on the line, not only the first (`pkill -f "[i]gneum-prove-host"; pkill -f prove-shard.sh` hid its second) + local rest="$code" m + while [[ "$rest" =~ (^|[^A-Za-z0-9_./-])(pgrep|pkill)([[:space:]]+-[A-Za-z0-9]+)*[[:space:]]+-[A-Za-z]*f[A-Za-z]*[[:space:]]+(\"[^\"]*\"|\'[^\']*\'|[^[:space:]|;\)]+) ]]; do + m="${BASH_REMATCH[0]}"; pat="${BASH_REMATCH[4]}"; pat="${pat#[\"\']}"; pat="${pat%[\"\']}" + rest="${rest#*"$m"}" + if [[ "$pat" =~ \.(log|txt|jsonl?|out|err|pid|csv|md|toml|ya?ml|lock)(\"|\'|$|[^A-Za-z0-9]) ]]; then echo "pkill/pgrep -f on a file name ($pat): a file name is a redirect or an argument, never the command line; use a pid file (tools/fleet/fleet-bg.sh) or the binary's full path"; return 1; fi + [[ "$pat" == *'$'* ]] && continue # a variable: the caller anchored it (reviewed by hand) + [[ "$pat" == /* ]] && continue # an anchored full path + [[ "$pat" =~ ^\[.\] ]] && continue # the bracket form never matches its own command line + echo "pgrep/pkill -f with a plain literal ($pat): it matches the calling shell's own command line; use the bracket form ([${pat:0:1}]${pat:1}), -x on the binary name, or a pid file"; return 1 + done + if [[ "$code" =~ (^|[^A-Za-z0-9_./-])(pgrep|pkill)[[:space:]]+(-[A-Za-z0-9]+[[:space:]]+)*(\"[^\"]*\"|\'[^\']*\'|[^[:space:]|;\)-][^[:space:]|;\)]*) ]]; then + pat="${BASH_REMATCH[4]}"; pat="${pat#[\"\']}"; pat="${pat%[\"\']}" + if [[ "$pat" =~ \.(log|txt|jsonl?|out|err|pid|csv|md|toml|ya?ml|lock)($|[^A-Za-z0-9]) ]] && [[ "$line" != *"-F "* ]]; then echo "pkill/pgrep on a file name ($pat): a file name is never a process name; use pkill -F or the binary's full path"; return 1; fi + fi + if [[ "$code" =~ (^|[^A-Za-z0-9_])ps[[:space:]][^|]*\|[[:space:]]*grep[[:space:]]+(-[A-Za-z]+[[:space:]]+)*(\"[^\"]*\"|\'[^\']*\'|[^[:space:]|;\)]+) ]]; then + pat="${BASH_REMATCH[3]}"; pat="${pat#[\"\']}"; pat="${pat%[\"\']}" + [[ "$pat" == *'$'* ]] && return 0 + [[ "$pat" =~ ^\[.\] ]] && return 0 + [[ "$pat" == grep ]] && return 0 # `grep -v grep` + echo "ps | grep with a plain literal ($pat): it matches the grep itself; use the bracket form ([${pat:0:1}]${pat:1}) or pgrep -x"; return 1 + fi + return 0 +} + +if [ "${1:-}" = "--self-test" ]; then + fails=0 + bad=( + 'pkill -f igneum-roll.log' + 'pkill -f "fleet-wave-3.log" || true' + 'pgrep -f igneumd >/dev/null && exit 0' + "pkill -f 'node tools/fleet/wave.mjs'" + 'if pgrep -f "igneum-miner --pool" >/dev/null; then echo up; fi' + 'pkill -9 -f run-shard.out' + 'ps aux | grep igneumd | grep -v grep' + 'ps -ef | grep "igneum-miner" | awk "{print \$2}"' + 'pkill node-1.pid' + "bash -c 'pkill -f \"[i]gneum-prove-host\"; pkill -f prove-shard.sh; true'" + "pgrep -fl 'igneumd --' | grep -v Wallet" + ) + good=( + 'pgrep -f "[i]gneumd" >/dev/null' + "pkill -f '[n]ode tools/fleet/wave.mjs'" + 'pgrep -x igneumd' + 'pkill -x igneum-miner' + 'pkill -F /srv/hands/node1.pid' + 'kill "$(cat "$PIDFILE")"' + 'pkill -f "$EXACT_CMD"' + 'pkill -f /opt/igneum/bin/igneumd' + 'pgrep -f "/srv/fleet/bin/igneum-miner --pool"' + '# pgrep -f igneumd is banned (a comment)' + '// pkill -f wave.log in a comment' + 'ps aux | grep "[i]gneumd"' + 'ps aux | grep -v grep | wc -l' + 'echo "the pgrep rule"' + "bash -c 'pkill -f \"[i]gneum-prove-host\"; pkill -f \"[p]rove-shard.sh\"; true'" + "pgrep -fl '[i]gneumd --' | grep -v Wallet" + ) + for l in "${bad[@]}"; do if check_line "$l" >/dev/null; then echo "self-test failed: accepted: $l"; fails=1; fi; done + for l in "${good[@]}"; do if ! out="$(check_line "$l")"; then echo "self-test failed: rejected: $l ($out)"; fails=1; fi; done + [ "$fails" = 0 ] && echo "self-test passed: ${#bad[@]} banned shapes fail (a log or out file name under pkill/pgrep, a plain literal under -f, the second pkill on a line, ps | grep with a literal); ${#good[@]} allowed shapes pass (bracket form, -x, -F pidfile, kill \$(cat pidfile), a variable, a full path, comments)" + exit $fails +fi + +cd "$(git rev-parse --show-toplevel)" +fail=0; n=0 +# only the lines that name the commands (git grep is a second over the tree; a bash loop over every line was a minute) +while IFS= read -r hit; do + f="${hit%%:*}"; rest="${hit#*:}"; ln="${rest%%:*}"; line="${rest#*:}"; n=$((n + 1)) + if ! why="$(check_line "$line")"; then echo "kill-by-name: $f:$ln: $why"; fail=1; fi +done < <(git grep -nE '(^|[^A-Za-z0-9_./-])(pgrep|pkill)([[:space:]]|$)|(^|[^A-Za-z0-9_])ps[[:space:]][^|]*\|[[:space:]]*grep' -- '*.sh' '*.bash' '*.mjs' '*.js' '*.py' '*.ps1' '*.bat' ':!vendor/**' ':!**/node_modules/**' ':!tools/ci/kill-by-name-check.sh' || true) +[ "$fail" = 0 ] && echo "kill-by-name: $n pgrep/pkill/ps-grep lines, none kills or finds a process by a plain name or a file name" +exit $fail diff --git a/tools/ci/pre-push.sh b/tools/ci/pre-push.sh index c6364234..5c6b335a 100755 --- a/tools/ci/pre-push.sh +++ b/tools/ci/pre-push.sh @@ -72,6 +72,7 @@ tree_checks() { run "commit-string gate self-test" bash tools/ci/commit-string-check.sh --self-test run "build server remote checkout self-test" bash infra/build-server/remote-run.sh --self-test run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh' + run "no script kills or finds a process by a plain name or a file name (pgrep/pkill -f literals, ps | grep)" bash -c 'bash tools/ci/kill-by-name-check.sh --self-test && bash tools/ci/kill-by-name-check.sh' run "the identity check's own self-test (excluded research path passes, exported leak fails)" bash tools/ci/identity-check.sh --self-test run "the Windows paths check's own self-test" bash tools/ci/windows-paths-check.sh --self-test run "the red watcher's own self-test (one line per run, posted once)" node tools/ci/red-watch.mjs --self-test diff --git a/tools/ci/prover-socket-check.sh b/tools/ci/prover-socket-check.sh index c67ad3e9..bcd2b18a 100755 --- a/tools/ci/prover-socket-check.sh +++ b/tools/ci/prover-socket-check.sh @@ -16,7 +16,8 @@ while IFS= read -r f; do [[ "$f" =~ $ALLOW ]] && continue # a playbook that only runs `--mode id` or `--mode verify` starts no GPU server; the prove modes do if grep -qE 'igneum-prove-host' "$f" && grep -qE -- '--mode (compressed|chain|aggregate|block|shard|all)\b' "$f" && grep -qE -- '-u root' "$f"; then - if ! grep -qE 'pkill -f sp1-gpu-server' "$f"; then echo "prover-socket: $f runs the prover host as root without killing sp1-gpu-server"; fail=1; fi + # -x on the binary name (kill-by-name rule, 6 October 2026: `pkill -x sp1-gpu-server` inside a `bash -c '...'` matched the calling bash's own command line) + if ! grep -qE 'pkill -(x|f) (\[s\]p1|sp1)-gpu-server' "$f"; then echo "prover-socket: $f runs the prover host as root without killing sp1-gpu-server"; fail=1; fi if ! grep -qE 'rm -f /tmp/sp1-cuda-' "$f"; then echo "prover-socket: $f runs the prover host as root without unlinking /tmp/sp1-cuda-*.sock"; fail=1; fi fi done < <(list_files "$@") diff --git a/tools/exec-attacks/lib/common.mjs b/tools/exec-attacks/lib/common.mjs index f3aadcb0..ca1f6f12 100644 --- a/tools/exec-attacks/lib/common.mjs +++ b/tools/exec-attacks/lib/common.mjs @@ -138,7 +138,7 @@ export function injectCmd(cmdArgs, grpc = GRPC1) { // Resident set size (KiB) of every igneumd process in the attack network, for the memory-bounded check. export function nodeRssKib() { try { - const out = execFileSync('bash', ['-c', "ps -axo rss,command | grep 'igneum-node-exec-attacks/target/release/igneumd --simnet' | grep -v grep | awk '{print $1}'"], { encoding: 'utf8' }); + const out = execFileSync('bash', ['-c', "ps -axo rss,command | grep '[i]gneum-node-exec-attacks/target/release/igneumd --simnet' | grep -v grep | awk '{print $1}'"], { encoding: 'utf8' }); return out.trim().split('\n').filter(Boolean).map(Number); } catch { return []; } } diff --git a/tools/exec-attacks/net.sh b/tools/exec-attacks/net.sh index 25337d3a..c50c4cad 100755 --- a/tools/exec-attacks/net.sh +++ b/tools/exec-attacks/net.sh @@ -87,8 +87,8 @@ case "${1:-}" in sleep 1 while read -r p; do [ -n "$p" ] && kill -9 "$p" 2>/dev/null; done < "$PIDS" fi - pkill -f "igneum-node-exec-attacks/target/release/igneumd --simnet" 2>/dev/null - pkill -f "igneum-miner mine grpc://127.0.0.1:276" 2>/dev/null + pkill -f "[i]gneum-node-exec-attacks/target/release/igneumd --simnet" 2>/dev/null + pkill -f "[i]gneum-miner mine grpc://127.0.0.1:276" 2>/dev/null echo "stopped" ;; grpc1) grpc1 ;; diff --git a/tools/lock/with-lock.sh b/tools/lock/with-lock.sh index 5e561574..2ee03e16 100755 --- a/tools/lock/with-lock.sh +++ b/tools/lock/with-lock.sh @@ -32,7 +32,7 @@ PY fi echo done - echo "waiting:"; ps -eo pid,etime,command | grep -E "with-lock.sh (build|measure|run) " | grep -v grep | awk '{printf " %s %s %s %s\n", $1, $2, $4, $5}' | head -20 + echo "waiting:"; ps -eo pid,etime,command | grep -E "[w]ith-lock.sh (build|measure|run) " | grep -v grep | awk '{printf " %s %s %s %s\n", $1, $2, $4, $5}' | head -20 exit 0 fi case "$kind" in diff --git a/tools/observer/autosync.sh b/tools/observer/autosync.sh index cfec7652..8b71b684 100755 --- a/tools/observer/autosync.sh +++ b/tools/observer/autosync.sh @@ -19,7 +19,7 @@ marker() { cat "$MARK" 2>/dev/null || echo none; } restart_due() { local t; t="$(observer_tree)"; [ -n "$t" ] && [ "$(marker)" != "$t" ]; } restart_observer() { # local pid - pid=$(pgrep -f "node tools/observer/observer.mjs" | head -1) + pid=$(pgrep -f "[n]ode tools/observer/observer.mjs" | head -1) if [ -n "$pid" ]; then kill "$pid" && echo "$(date -u +%FT%TZ) observer $pid restarted: $1" else echo "$(date -u +%FT%TZ) no observer process to restart ($1): is tools/observer/run.sh running?"; fi observer_tree > "$MARK" diff --git a/tools/prover-floor/pc2-floor-measure.ps1 b/tools/prover-floor/pc2-floor-measure.ps1 index 367012c6..ffac1f85 100644 --- a/tools/prover-floor/pc2-floor-measure.ps1 +++ b/tools/prover-floor/pc2-floor-measure.ps1 @@ -34,12 +34,12 @@ EMPTY='EMPTY_PLACEHOLDER'; V1="$FX/fees-v1-shards2.json"; FULL="$FX/block-338-sh [ -x "$H" ] || { echo "RESULT measure_failed no pv1 host at $H"; exit 2; } echo "RESULT patched_server sha256=$(sha256sum $SRV | cut -c1-64) version=$($SRV --version 2>/dev/null) host=$(sha256sum $H | cut -c1-16)" echo "RESULT live_server sha256=$(sha256sum /root/.sp1/bin/sp1-gpu-server | cut -c1-16) untouched" -pkill -f sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock echo "RESULT idle_mib $(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -1)" run() { # name fixture env... local name="$1" fx="$2"; shift 2 local tag="$name-$(basename $fx .json)" - pkill -f sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock + pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock local csv="$JOB/smi-$tag.csv" log="$JOB/log-$tag.txt" nvidia-smi --query-gpu=timestamp,memory.used,utilization.gpu --format=csv,noheader,nounits -l 1 > "$csv" 2>/dev/null & local SMI=$! @@ -47,7 +47,7 @@ run() { # name fixture env... env HOME=$FLOORHOME SP1_PROVER=cuda RUST_LOG=off SP1_GPU_FLOOR_LOG=1 "$@" $H "$fx" --mode compressed --shard 0 --out "$JOB/res-$tag.json" > "$log" 2>&1 local rc=$? local wall=$(( $(date +%s) - t0 )) - pkill -f sp1-gpu-server 2>/dev/null; sleep 1; kill $SMI 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock + pkill -x sp1-gpu-server 2>/dev/null; sleep 1; kill $SMI 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock local peak=$(awk -F', *' '{ if ($2+0 > m) m=$2+0 } END { print m+0 }' "$csv") local n=$(wc -l < "$csv") local line=$(grep -E "^RESULT compressed shard" "$log" | tail -1 | sed -E 's/.*prove ([0-9.]+) s, proof ([0-9]+) bytes, verify ([0-9.]+) s, ([A-Z ]+);.*/prove_s=\1 bytes=\2 verify_s=\3 \4/') @@ -57,7 +57,7 @@ run() { # name fixture env... grep -E "^FLOOR" "$log" | sed "s/^/RESULT floorline cfg=$name fixture=$(basename $fx .json) /" | head -40 } POINTS_PLACEHOLDER_BASH -pkill -f sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock echo "RESULT measure_end $(stamp)" '@ $bash = $bash.Replace('JOBW_PLACEHOLDER', $jobW).Replace('EMPTY_PLACEHOLDER', $emptyW).Replace('POINTS_PLACEHOLDER_BASH', ($points -replace ';', "`n")) diff --git a/tools/prover-floor/pc2-floor-sweep1.ps1 b/tools/prover-floor/pc2-floor-sweep1.ps1 index 7a38e1cb..4282a692 100644 --- a/tools/prover-floor/pc2-floor-sweep1.ps1 +++ b/tools/prover-floor/pc2-floor-sweep1.ps1 @@ -34,12 +34,12 @@ EMPTY='EMPTY_PLACEHOLDER'; V1="$FX/fees-v1-shards2.json"; FULL="$FX/block-338-sh [ -x "$H" ] || { echo "RESULT measure_failed no pv1 host at $H"; exit 2; } echo "RESULT patched_server sha256=$(sha256sum $SRV | cut -c1-64) version=$($SRV --version 2>/dev/null) host=$(sha256sum $H | cut -c1-16)" echo "RESULT live_server sha256=$(sha256sum /root/.sp1/bin/sp1-gpu-server | cut -c1-16) untouched" -pkill -f sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock echo "RESULT idle_mib $(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -1)" run() { # name fixture env... local name="$1" fx="$2"; shift 2 local tag="$name-$(basename $fx .json)" - pkill -f sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock + pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock local csv="$JOB/smi-$tag.csv" log="$JOB/log-$tag.txt" nvidia-smi --query-gpu=timestamp,memory.used,utilization.gpu --format=csv,noheader,nounits -l 1 > "$csv" 2>/dev/null & local SMI=$! @@ -47,7 +47,7 @@ run() { # name fixture env... env HOME=$FLOORHOME SP1_PROVER=cuda RUST_LOG=off SP1_GPU_FLOOR_LOG=1 "$@" $H "$fx" --mode compressed --shard 0 --out "$JOB/res-$tag.json" > "$log" 2>&1 local rc=$? local wall=$(( $(date +%s) - t0 )) - pkill -f sp1-gpu-server 2>/dev/null; sleep 1; kill $SMI 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock + pkill -x sp1-gpu-server 2>/dev/null; sleep 1; kill $SMI 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock local peak=$(awk -F', *' '{ if ($2+0 > m) m=$2+0 } END { print m+0 }' "$csv") local n=$(wc -l < "$csv") local line=$(grep -E "^RESULT compressed shard" "$log" | tail -1 | sed -E 's/.*prove ([0-9.]+) s, proof ([0-9]+) bytes, verify ([0-9.]+) s, ([A-Z ]+);.*/prove_s=\1 bytes=\2 verify_s=\3 \4/') @@ -57,7 +57,7 @@ run() { # name fixture env... grep -E "^FLOOR" "$log" | sed "s/^/RESULT floorline cfg=$name fixture=$(basename $fx .json) /" | head -40 } POINTS_PLACEHOLDER_BASH -pkill -f sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock echo "RESULT measure_end $(stamp)" '@ $bash = $bash.Replace('JOBW_PLACEHOLDER', $jobW).Replace('EMPTY_PLACEHOLDER', $emptyW).Replace('POINTS_PLACEHOLDER_BASH', ($points -replace ';', "`n")) diff --git a/tools/prover-floor/pc2-floor-sweep2.ps1 b/tools/prover-floor/pc2-floor-sweep2.ps1 index 5f9058f7..70d552c8 100644 --- a/tools/prover-floor/pc2-floor-sweep2.ps1 +++ b/tools/prover-floor/pc2-floor-sweep2.ps1 @@ -34,12 +34,12 @@ EMPTY='EMPTY_PLACEHOLDER'; V1="$FX/fees-v1-shards2.json"; FULL="$FX/block-338-sh [ -x "$H" ] || { echo "RESULT measure_failed no pv1 host at $H"; exit 2; } echo "RESULT patched_server sha256=$(sha256sum $SRV | cut -c1-64) version=$($SRV --version 2>/dev/null) host=$(sha256sum $H | cut -c1-16)" echo "RESULT live_server sha256=$(sha256sum /root/.sp1/bin/sp1-gpu-server | cut -c1-16) untouched" -pkill -f sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock echo "RESULT idle_mib $(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -1)" run() { # name fixture env... local name="$1" fx="$2"; shift 2 local tag="$name-$(basename $fx .json)" - pkill -f sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock + pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock local csv="$JOB/smi-$tag.csv" log="$JOB/log-$tag.txt" nvidia-smi --query-gpu=timestamp,memory.used,utilization.gpu --format=csv,noheader,nounits -l 1 > "$csv" 2>/dev/null & local SMI=$! @@ -47,7 +47,7 @@ run() { # name fixture env... env HOME=$FLOORHOME SP1_PROVER=cuda RUST_LOG=off SP1_GPU_FLOOR_LOG=1 "$@" $H "$fx" --mode compressed --shard 0 --out "$JOB/res-$tag.json" > "$log" 2>&1 local rc=$? local wall=$(( $(date +%s) - t0 )) - pkill -f sp1-gpu-server 2>/dev/null; sleep 1; kill $SMI 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock + pkill -x sp1-gpu-server 2>/dev/null; sleep 1; kill $SMI 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock local peak=$(awk -F', *' '{ if ($2+0 > m) m=$2+0 } END { print m+0 }' "$csv") local n=$(wc -l < "$csv") local line=$(grep -E "^RESULT compressed shard" "$log" | tail -1 | sed -E 's/.*prove ([0-9.]+) s, proof ([0-9]+) bytes, verify ([0-9.]+) s, ([A-Z ]+);.*/prove_s=\1 bytes=\2 verify_s=\3 \4/') @@ -57,7 +57,7 @@ run() { # name fixture env... grep -E "^FLOOR" "$log" | sed "s/^/RESULT floorline cfg=$name fixture=$(basename $fx .json) /" | head -40 } POINTS_PLACEHOLDER_BASH -pkill -f sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock echo "RESULT measure_end $(stamp)" '@ $bash = $bash.Replace('JOBW_PLACEHOLDER', $jobW).Replace('EMPTY_PLACEHOLDER', $emptyW).Replace('POINTS_PLACEHOLDER_BASH', ($points -replace ';', "`n")) diff --git a/tools/prover-floor/pc2-floor-sweep3-miner.ps1 b/tools/prover-floor/pc2-floor-sweep3-miner.ps1 index 6a29c5a1..5c89cc64 100644 --- a/tools/prover-floor/pc2-floor-sweep3-miner.ps1 +++ b/tools/prover-floor/pc2-floor-sweep3-miner.ps1 @@ -35,12 +35,12 @@ EMPTY='EMPTY_PLACEHOLDER'; V1="$FX/fees-v1-shards2.json"; FULL="$FX/block-338-sh [ -x "$H" ] || { echo "RESULT measure_failed no pv1 host at $H"; exit 2; } echo "RESULT patched_server sha256=$(sha256sum $SRV | cut -c1-64) version=$($SRV --version 2>/dev/null) host=$(sha256sum $H | cut -c1-16)" echo "RESULT live_server sha256=$(sha256sum /root/.sp1/bin/sp1-gpu-server | cut -c1-16) untouched" -pkill -f sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock echo "RESULT idle_mib $(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -1)" run() { # name fixture env... local name="$1" fx="$2"; shift 2 local tag="$name-$(basename $fx .json)" - pkill -f sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock + pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock local csv="$JOB/smi-$tag.csv" log="$JOB/log-$tag.txt" nvidia-smi --query-gpu=timestamp,memory.used,utilization.gpu --format=csv,noheader,nounits -l 1 > "$csv" 2>/dev/null & local SMI=$! @@ -48,7 +48,7 @@ run() { # name fixture env... env HOME=$FLOORHOME SP1_PROVER=cuda RUST_LOG=off SP1_GPU_FLOOR_LOG=1 "$@" $H "$fx" --mode compressed --shard 0 --out "$JOB/res-$tag.json" > "$log" 2>&1 local rc=$? local wall=$(( $(date +%s) - t0 )) - pkill -f sp1-gpu-server 2>/dev/null; sleep 1; kill $SMI 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock + pkill -x sp1-gpu-server 2>/dev/null; sleep 1; kill $SMI 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock local peak=$(awk -F', *' '{ if ($2+0 > m) m=$2+0 } END { print m+0 }' "$csv") local n=$(wc -l < "$csv") local line=$(grep -E "^RESULT compressed shard" "$log" | tail -1 | sed -E 's/.*prove ([0-9.]+) s, proof ([0-9]+) bytes, verify ([0-9.]+) s, ([A-Z ]+);.*/prove_s=\1 bytes=\2 verify_s=\3 \4/') @@ -58,7 +58,7 @@ run() { # name fixture env... grep -E "^FLOOR" "$log" | sed "s/^/RESULT floorline cfg=$name fixture=$(basename $fx .json) /" | head -40 } POINTS_PLACEHOLDER_BASH -pkill -f sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock echo "RESULT measure_end $(stamp)" '@ $bash = $bash.Replace('JOBW_PLACEHOLDER', $jobW).Replace('EMPTY_PLACEHOLDER', $emptyW).Replace('POINTS_PLACEHOLDER_BASH', ($points -replace ';', "`n")) diff --git a/tools/proving-v1/pc2-agg-cost.ps1 b/tools/proving-v1/pc2-agg-cost.ps1 index d7263fc4..c3007115 100644 --- a/tools/proving-v1/pc2-agg-cost.ps1 +++ b/tools/proving-v1/pc2-agg-cost.ps1 @@ -236,7 +236,7 @@ echo "RESULT fixtures $(stamp) $LIST" # 21:25Z fault of job 1): the server is killed and the socket removed here and again at the end (the rule of # tools/ci/prover-socket-check.sh) for i in $(seq 1 36); do pgrep -x sp1-gpu-server >/dev/null || break; sleep 5; done -pkill -f sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock # the root-socket class: the CI check (prover-socket-check.sh) wants pkill -f +pkill -x sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock # the root-socket class: the CI check (prover-socket-check.sh) wants pkill -f echo "RESULT gpu_server_before $(stamp) running=$(pgrep -x sp1-gpu-server | wc -l) socket=$(ls /tmp/sp1-cuda-0.sock 2>/dev/null || echo none)" '@ $phase = @' diff --git a/tools/proving-v1/pc2-chain.ps1 b/tools/proving-v1/pc2-chain.ps1 index 18c85634..70370690 100644 --- a/tools/proving-v1/pc2-chain.ps1 +++ b/tools/proving-v1/pc2-chain.ps1 @@ -82,7 +82,7 @@ free -m | awk '/Mem:/ {print "RESULT wsl_ram_now total_mb=" `$2 " used_mb=" `$3} STMT=`$(python3 -c "import json; print(json.load(open('`$JOB/chain-results.json'))['segment_statement'])") PROOF=`$(python3 -c "import json; print(json.load(open('`$JOB/chain-results.json'))['segment_proof_file'])") for i in 1 2 3; do `$H --mode verify-segment --proof "`$PROOF" --statement "`$STMT" 2>&1 | grep -E "^RESULT" | sed "s/^/verify-segment run `$i: /"; done -pkill -f sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock echo "RESULTS-JSON"; cat "`$JOB/chain-results.json"; echo; echo "END" "@ $bashFile = Join-Path $job 'chain.sh' diff --git a/tools/proving-v1/pc2-memory-miner-on.ps1 b/tools/proving-v1/pc2-memory-miner-on.ps1 index b560394c..b9dc66df 100644 --- a/tools/proving-v1/pc2-memory-miner-on.ps1 +++ b/tools/proving-v1/pc2-memory-miner-on.ps1 @@ -24,11 +24,11 @@ CUDA_DIR="`$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true) stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } JOB='$jobW'; H=/opt/igneum-pv1/igneum-prove-host; FX="`$HOME/igneum-prove-pv1/proving/fixtures" EMPTY='$emptyW'; FULL="`$FX/block-338-shard1.json" -pkill -f sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock echo "RESULT miner_resident_mib `$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -1) (the miner's working set before the prover starts)" run() { local name="`$1" fx="`$2"; shift 2 - pkill -f sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock + pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock local csv="`$JOB/smi-`$name-`$(basename `$fx .json).csv" nvidia-smi --query-gpu=timestamp,memory.used,utilization.gpu --format=csv,noheader,nounits -l 1 > "`$csv" 2>/dev/null & local SMI=`$! @@ -45,7 +45,7 @@ run() { run baseline "`$EMPTY" run baseline "`$FULL" run baseline "`$EMPTY" -pkill -f sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock S="`$HOME/.sp1/bin/sp1-gpu-server" echo "RESULT server_help `$(`$S --help 2>&1 | tr '\n' ' ' | cut -c1-900)" echo "RESULT server_env_names `$(strings `$S | grep -oE '^(SP1|MOONGATE|CUDA|GPU|NVIDIA|MEM|TRACE|SHARD|ELEMENT|HEIGHT|FULL)[A-Z0-9_]{3,}$' | sort -u | tr '\n' ' ' | cut -c1-1200)" diff --git a/tools/proving-v1/pc2-memory-sweep.ps1 b/tools/proving-v1/pc2-memory-sweep.ps1 index ab4741fc..88b75b58 100644 --- a/tools/proving-v1/pc2-memory-sweep.ps1 +++ b/tools/proving-v1/pc2-memory-sweep.ps1 @@ -27,7 +27,7 @@ CUDA_DIR="`$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true) stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } JOB='$jobW'; H=/opt/igneum-pv1/igneum-prove-host; X=/opt/igneum-pv1/igneum-prove-export; FX="`$HOME/igneum-prove-pv1/proving/fixtures" EMPTY='$emptyW'; FULL="`$FX/block-338-shard1.json" -pkill -f sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock echo "RESULT servers_before `$(pgrep -a sp1-gpu-server | tr '\n' ' ' || echo none)" # the S_p/2 and S_p/4 cuts of block 344 (27 M pgas: 8 and 16 shards), from the package's own export SEQ="`$HOME/igneum-prove-pv1/tools/prove-fixtures/seq.json" @@ -37,7 +37,7 @@ if [ -f "`$SEQ" ]; then fi run() { # name fixture env... local name="`$1" fx="`$2"; shift 2 - pkill -f sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock + pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock local csv="`$JOB/smi-`$name-`$(basename `$fx .json).csv" nvidia-smi --query-gpu=timestamp,memory.used,utilization.gpu --format=csv,noheader,nounits -l 1 > "`$csv" 2>/dev/null & local SMI=`$! @@ -69,7 +69,7 @@ run w1elem26 "`$EMPTY" ELEMENT_THRESHOLD=67108864 HEIGHT_THRESHOLD=2097152 `$W1 [ -f "`$JOB/block-344-half.json" ] && run w1elem26 "`$JOB/block-344-half.json" ELEMENT_THRESHOLD=67108864 HEIGHT_THRESHOLD=2097152 `$W1 [ -f "`$JOB/block-344-quarter.json" ] && run w1elem26 "`$JOB/block-344-quarter.json" ELEMENT_THRESHOLD=67108864 HEIGHT_THRESHOLD=2097152 `$W1 [ -f "`$JOB/block-344-quarter.json" ] && run baseline "`$JOB/block-344-quarter.json" -pkill -f sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock echo "RESULT sweep_end `$(stamp)" "@ $bashFile = Join-Path $job 'sweep.sh' diff --git a/tools/proving-v1/pc2-segments.ps1 b/tools/proving-v1/pc2-segments.ps1 index 58aa16cb..5011389a 100644 --- a/tools/proving-v1/pc2-segments.ps1 +++ b/tools/proving-v1/pc2-segments.ps1 @@ -93,7 +93,7 @@ echo "RESULT installed $(sha256sum $H | cut -c1-16) host, $(sha256sum $X | cut - $H --mode id | sed 's/^/RESULT segal-host /' echo "RESULT gpu_server_before $(stamp) running=$(pgrep -x sp1-gpu-server | wc -l) socket=$(ls -la /tmp/sp1-cuda-0.sock 2>/dev/null || echo none)" # the root-socket rule (tools/ci/prover-socket-check.sh): this run is root; the app's prover is off, so its server goes too -pkill -f sp1-gpu-server; sleep 1; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server; sleep 1; rm -f /tmp/sp1-cuda-*.sock echo "RESULT socket_start $(stamp) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)" '@ # bash seg.sh [prev file]: unwrap the export, cut the blocks, run the chain with the sampler @@ -337,6 +337,6 @@ $t1 = Epoch (Get-Date) $during = RateLines (Epoch $runStart) $t1 if ($during.Count) { $m = $during | Measure-Object -Average -Minimum -Maximum; "RESULT rate_during $(Stamp) n=$($during.Count) mean=$([math]::Round($m.Average,2)) min=$($m.Minimum) max=$($m.Maximum) MH/s (the app log's status lines over the run)" } else { "RESULT rate_during $(Stamp) no status lines found" } "RESULT summary $(Stamp) passes=$passes claimed=$claimed submitted=$submitted paid=$paidCount paid_wei=$paidTotal shards_accepted=$shardsAccepted shards_refused=$shardsRefused last_segment_s=$lastSegSecs run_min=$([math]::Round(((Get-Date) - $runStart).TotalMinutes,1))" -& wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -f sp1-gpu-server; sleep 1; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') } +& wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -x sp1-gpu-server; sleep 1; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') } "RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})" "RESULT end $(Stamp)" diff --git a/tools/proving-v1/pc2-socket-fix.ps1 b/tools/proving-v1/pc2-socket-fix.ps1 index 80eb0d28..f8b188d8 100644 --- a/tools/proving-v1/pc2-socket-fix.ps1 +++ b/tools/proving-v1/pc2-socket-fix.ps1 @@ -13,7 +13,7 @@ function Prove($on) { try { (Invoke-RestMethod -Method Post -Uri "$base/api/prov $bash = @' echo "RESULT sockets_before $(ls -la /tmp/sp1-cuda-*.sock 2>&1 | tr '\n' ' ')" echo "RESULT servers_before $(ps -eo user,pid,cmd | grep -E '[s]p1-gpu-server' | tr '\n' ' ' || echo none)" -pkill -f sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock echo "RESULT sockets_after $(ls -la /tmp/sp1-cuda-*.sock 2>&1 | tr '\n' ' ')" echo "RESULT servers_after $(ps -eo user,pid,cmd | grep -E '[s]p1-gpu-server' | tr '\n' ' ' || echo none)" '@ diff --git a/tools/proving-v1/pc2-sp-curve.ps1 b/tools/proving-v1/pc2-sp-curve.ps1 index 9c5643f1..41853c11 100644 --- a/tools/proving-v1/pc2-sp-curve.ps1 +++ b/tools/proving-v1/pc2-sp-curve.ps1 @@ -29,7 +29,7 @@ export PATH="`$HOME/.cargo/bin:`$HOME/.sp1/bin:`$PATH" CUDA_DIR="`$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true)"; [ -n "`$CUDA_DIR" ] && export PATH="`$CUDA_DIR/bin:`$PATH" && export LD_LIBRARY_PATH="`$CUDA_DIR/lib64:/usr/lib/wsl/lib:`${LD_LIBRARY_PATH:-}" stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } JOB='$jobW'; PKG='$pkgW'; DEST="`$HOME/igneum-prove-pv1"; LIVE_TARGET="`$HOME/igneum-prove/proving/igneum-prove/target" -pkill -f sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock # the pv1b host (--budget) from the fetched package, built against the warm target dir, into /opt/igneum-pv1 rsync -a --delete --exclude target "`$PKG/package/" "`$DEST/" find "`$DEST" -name target -prune -o -type f -exec touch {} + 2>/dev/null @@ -42,7 +42,7 @@ H=/opt/igneum-pv1/igneum-prove-host; FX="`$DEST/proving/fixtures" echo "RESULT miner_resident_mib `$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -1)" run() { # name fixture budget env... local name="`$1" fx="`$2" budget="`$3"; shift 3 - pkill -f sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock + pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock local tag="`$name-`$(basename `$fx .json)" local csv="`$JOB/smi-`$tag.csv" nvidia-smi --query-gpu=timestamp,memory.used,utilization.gpu --format=csv,noheader,nounits -l 1 > "`$csv" 2>/dev/null & @@ -71,7 +71,7 @@ run base "`$FX/block-338-shard1.json" 0 run e25 "`$FX/fees-v1-shards2.json" 0 `$E25 run e25 "`$FX/block-344-shards4.json" 2249264 `$E25 run e25 "`$FX/block-338-shard1.json" 0 `$E25 -pkill -f sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock +pkill -x sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock echo "RESULT curve_end `$(stamp)" "@ $bashFile = Join-Path $job 'curve.sh'