From 342fd50aac1ea273289b2e3579b162bc6f6fd8d3 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:10:51 +0000 Subject: [PATCH] CUDA worker serve-bench: keep the coproc pid past its exit, IGNEUM_PID_DIR for the fleet pid directory Co-Authored-By: Claude Fable 5.1 --- proto-cuda/nvrtc/serve-bench.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/proto-cuda/nvrtc/serve-bench.sh b/proto-cuda/nvrtc/serve-bench.sh index bc2807f1d..62ada452e 100755 --- a/proto-cuda/nvrtc/serve-bench.sh +++ b/proto-cuda/nvrtc/serve-bench.sh @@ -8,13 +8,14 @@ # rail); the worker's own CPU seconds (utime + stime from /proc) stand in for the host-side cost of the read-back and # the scan. The kernel timer is --bench on the same pack and batch, printed beside the rows. # serve-bench.sh [jobs 32] [batch-log2 24] [device 0] -# Pid files: every process this script starts writes /pids/..pid (the fleet's rule: a job is +# Pid files: every process this script starts writes /pids/..pid (IGNEUM_PID_DIR overrides the directory) (the fleet's rule: a job is # stopped through its pid file, never by a name pattern). Output: /.row (one line, key=value) plus the # worker log, the stub's timeline and the power samples. set -euo pipefail W="$1"; PACK="$2"; OUT="$3"; RUN="$4"; MODE="$5"; JOBS="${6:-32}"; B="${7:-24}"; DEV="${8:-0}" [ "$MODE" = full ] || [ "$MODE" = select ] || { echo "readback must be full or select" >&2; exit 2; } -mkdir -p "$OUT/pids" +PIDDIR="${IGNEUM_PID_DIR:-$OUT/pids}" # the fleet keeps pid files under one directory per pod +mkdir -p "$PIDDIR" LOG="$OUT/$RUN.worker.log"; TL="$OUT/$RUN.timeline"; SMI="$OUT/$RUN.smi"; ROW="$OUT/$RUN.row" : > "$TL" now() { date +%s.%N; } @@ -27,11 +28,12 @@ jobline() { printf 'job %d %064x %s %d %d %s %s\n' "$1" "$1" "$target" $(( $1 * # power sampler first (its own pid file) nvidia-smi -i "$DEV" --query-gpu=timestamp,power.draw,clocks.sm,temperature.gpu --format=csv,noheader,nounits -lms 250 > "$SMI" 2>/dev/null & -echo $! > "$OUT/pids/$RUN.smi.pid" +echo $! > "$PIDDIR/$RUN.smi.pid" sleep 1 t0=$(now) coproc WK { IGNEUM_READBACK="$MODE" "$W" --serve --pack "$PACK" --device "$DEV" --batch-log2 "$B" --race off --readback "$MODE" 2>&1; } -echo "$WK_PID" > "$OUT/pids/$RUN.worker.pid" +wkpid=$WK_PID # bash clears WK_PID when the coproc ends; the copy outlives it +echo "$wkpid" > "$PIDDIR/$RUN.worker.pid" sent=0; done_n=0; found=0; tready=""; tlast=""; cpu="" : > "$LOG" while IFS= read -r line <&"${WK[0]}"; do @@ -43,16 +45,16 @@ while IFS= read -r line <&"${WK[0]}"; do done*) done_n=$((done_n + 1)); tlast=$t; echo "$t $line" >> "$TL" if [ "$sent" -lt "$JOBS" ]; then jobline $sent >&"${WK[1]}"; sent=$((sent + 1)); fi if [ "$done_n" -ge "$JOBS" ]; then - cpu=$(awk '{ printf "%.2f", ($14 + $15) / 100.0 }' "/proc/$WK_PID/stat" 2>/dev/null || echo "") + cpu=$(awk '{ printf "%.2f", ($14 + $15) / 100.0 }' "/proc/$wkpid/stat" 2>/dev/null || echo "") echo "quit" >&"${WK[1]}" fi ;; error*) echo "$t $line" >> "$TL" ;; esac done -wait "$WK_PID" 2>/dev/null || true +wait "$wkpid" 2>/dev/null || true t1=$(now) sleep 1 -kill "$(cat "$OUT/pids/$RUN.smi.pid")" 2>/dev/null || true +kill "$(cat "$PIDDIR/$RUN.smi.pid")" 2>/dev/null || true [ -n "$tready" ] && [ -n "$tlast" ] || { echo "FAIL: no ready or no done line; see $LOG"; tail -5 "$LOG"; exit 1; } [ "$done_n" = "$JOBS" ] || { echo "FAIL: $done_n of $JOBS jobs done; see $LOG"; exit 1; }