igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

CUDA worker serve-bench: keep the coproc pid past its exit, IGNEUM_PID_DIR for the fleet pid directory

Browse source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs 2026-10-08 19:10:51 +00:00
parent a1802ac31c
commit 342fd50aac
1 changed files with 9 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
proto-cuda/nvrtc/serve-bench.sh
View file

@ -8,13 +8,14 @@
# rail); the worker's own CPU seconds (utime + stime from /proc) stand in for the host-side cost of the read-back and
# the scan. The kernel timer is --bench on the same pack and batch, printed beside the rows.
# serve-bench.sh <worker> <pack dir> <out dir> <run name> <readback: full|select> [jobs 32] [batch-log2 24] [device 0]
# Pid files: every process this script starts writes <out dir>/pids/<run name>.<what>.pid (the fleet's rule: a job is
# Pid files: every process this script starts writes <out dir>/pids/<run name>.<what>.pid (IGNEUM_PID_DIR overrides the directory) (the fleet's rule: a job is
# stopped through its pid file, never by a name pattern). Output: <out dir>/<run name>.row (one line, key=value) plus the
# worker log, the stub's timeline and the power samples.
set -euo pipefail
W="$1"; PACK="$2"; OUT="$3"; RUN="$4"; MODE="$5"; JOBS="${6:-32}"; B="${7:-24}"; DEV="${8:-0}"
[ "$MODE" = full ] || [ "$MODE" = select ] || { echo "readback must be full or select" >&2; exit 2; }
mkdir -p "$OUT/pids"
PIDDIR="${IGNEUM_PID_DIR:-$OUT/pids}" # the fleet keeps pid files under one directory per pod
mkdir -p "$PIDDIR"
LOG="$OUT/$RUN.worker.log"; TL="$OUT/$RUN.timeline"; SMI="$OUT/$RUN.smi"; ROW="$OUT/$RUN.row"
: > "$TL"
now() { date +%s.%N; }
@ -27,11 +28,12 @@ jobline() { printf 'job %d %064x %s %d %d %s %s\n' "$1" "$1" "$target" $(( $1 *
# power sampler first (its own pid file)
nvidia-smi -i "$DEV" --query-gpu=timestamp,power.draw,clocks.sm,temperature.gpu --format=csv,noheader,nounits -lms 250 > "$SMI" 2>/dev/null &
echo $! > "$OUT/pids/$RUN.smi.pid"
echo $! > "$PIDDIR/$RUN.smi.pid"
sleep 1
t0=$(now)
coproc WK { IGNEUM_READBACK="$MODE" "$W" --serve --pack "$PACK" --device "$DEV" --batch-log2 "$B" --race off --readback "$MODE" 2>&1; }
echo "$WK_PID" > "$OUT/pids/$RUN.worker.pid"
wkpid=$WK_PID # bash clears WK_PID when the coproc ends; the copy outlives it
echo "$wkpid" > "$PIDDIR/$RUN.worker.pid"
sent=0; done_n=0; found=0; tready=""; tlast=""; cpu=""
: > "$LOG"
while IFS= read -r line <&"${WK[0]}"; do
@ -43,16 +45,16 @@ while IFS= read -r line <&"${WK[0]}"; do
done*) done_n=$((done_n + 1)); tlast=$t; echo "$t $line" >> "$TL"
if [ "$sent" -lt "$JOBS" ]; then jobline $sent >&"${WK[1]}"; sent=$((sent + 1)); fi
if [ "$done_n" -ge "$JOBS" ]; then
cpu=$(awk '{ printf "%.2f", ($14 + $15) / 100.0 }' "/proc/$WK_PID/stat" 2>/dev/null || echo "")
cpu=$(awk '{ printf "%.2f", ($14 + $15) / 100.0 }' "/proc/$wkpid/stat" 2>/dev/null || echo "")
echo "quit" >&"${WK[1]}"
fi ;;
error*) echo "$t $line" >> "$TL" ;;
esac
done
wait "$WK_PID" 2>/dev/null || true
wait "$wkpid" 2>/dev/null || true
t1=$(now)
sleep 1
kill "$(cat "$OUT/pids/$RUN.smi.pid")" 2>/dev/null || true
kill "$(cat "$PIDDIR/$RUN.smi.pid")" 2>/dev/null || true
[ -n "$tready" ] && [ -n "$tlast" ] || { echo "FAIL: no ready or no done line; see $LOG"; tail -5 "$LOG"; exit 1; }
[ "$done_n" = "$JOBS" ] || { echo "FAIL: $done_n of $JOBS jobs done; see $LOG"; exit 1; }