86 lines
6.5 KiB
Bash
Executable file
86 lines
6.5 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Ember Tune's two-knob ladder on a rented NVIDIA card (docs/plans/ember-tune.md, branch ember-tune): the miner runs
|
|
# throughout; the power ladder 100, 90, 80, 70, 60, 50% of the default limit at the unlocked clock (clamped at the
|
|
# card's reported minimum), then the clock ladder 90, 80, 70, 60% of the maximum graphics clock at the power the
|
|
# first ladder chose; 15 s settle and 60 s hold per step; the choice is the best MH/W among the steps whose rate is
|
|
# within 1% of the fastest. Linux root: nvidia-smi -pl and -lgc, no prompt. Every step prints a RESULT line; the
|
|
# ladder and the choice go to /root/fleet/out/ember.json as a TUNE-shaped record (relay/lib/ember.mjs parseRecords).
|
|
set -uo pipefail
|
|
F=/root/fleet; OUT=$F/out; LOG=$OUT/ember.log; B=/opt/igneum/pkg/bin
|
|
LABEL="${LABEL:-box}"; WALLET="${WALLET:-0x1919191919191919191919191919191919191919}"
|
|
SETTLE="${SETTLE:-15}"; HOLD="${HOLD:-60}"
|
|
mkdir -p $OUT $F/mine/packs
|
|
exec > >(tee -a $LOG) 2>&1
|
|
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
|
say() { echo "$(stamp) $*"; }
|
|
q() { nvidia-smi --query-gpu="$1" --format=csv,noheader,nounits -i 0 2>/dev/null | head -1 | tr -d ' '; }
|
|
NAME="$(q name)"; DRV="$(q driver_version)"; PDEF="$(q power.default_limit)"; PMIN="$(q power.min_limit)"; PMAX="$(q power.max_limit)"; CMAX="$(q clocks.max.graphics)"
|
|
pkill -x sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock
|
|
echo "RESULT start $(stamp) card=$NAME driver=$DRV power_default_w=$PDEF min_w=$PMIN max_w=$PMAX clock_max_mhz=$CMAX"
|
|
# can we set anything?
|
|
nvidia-smi -i 0 -pl "$PDEF" >/dev/null 2>&1 && PL_OK=1 || PL_OK=0
|
|
nvidia-smi -i 0 -lgc 0,"$CMAX" >/dev/null 2>&1 && LGC_OK=1 || LGC_OK=0
|
|
nvidia-smi -i 0 -rgc >/dev/null 2>&1
|
|
echo "RESULT knobs power_limit_settable=$PL_OK clock_cap_settable=$LGC_OK"
|
|
cd $F/mine
|
|
rm -rf packs/devnet; $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet > $OUT/ember-export.log 2>&1
|
|
nohup $B/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 "$LABEL" --worker $B/igneum-worker-cuda --worker-args "--device 0 --pack packs/devnet" \
|
|
--prepare-packs packs/prepare --exit-on-seed-change --evm-address "$WALLET" --payout-label "$LABEL" --status-secs 10 > $OUT/ember-miner.log 2>&1 &
|
|
MPID=$!; cd $F
|
|
cleanup() { nvidia-smi -i 0 -rgc >/dev/null 2>&1; [ "$PL_OK" = 1 ] && nvidia-smi -i 0 -pl "$PDEF" >/dev/null 2>&1; kill $MPID 2>/dev/null; pkill -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda' 2>/dev/null; }
|
|
trap cleanup EXIT
|
|
say "miner warming 90 s"; sleep 90
|
|
STEPS=$OUT/ember-steps.jsonl; : > $STEPS
|
|
step() { # <label> <power_pct> <limit_w> <clock_cap_mhz or 0>
|
|
local lab="$1" pct="$2" lim="$3" cap="$4"
|
|
if [ "$PL_OK" = 1 ]; then nvidia-smi -i 0 -pl "$lim" >/dev/null 2>&1 || say "could not set -pl $lim"; fi
|
|
if [ "$cap" != 0 ]; then nvidia-smi -i 0 -lgc 0,"$cap" >/dev/null 2>&1 || say "could not set -lgc $cap"; else nvidia-smi -i 0 -rgc >/dev/null 2>&1; fi
|
|
sleep "$SETTLE"
|
|
local n0; n0="$(grep -c 'STATUS' $OUT/ember-miner.log)"
|
|
( while :; do nvidia-smi --query-gpu=power.draw,clocks.sm,clocks.mem,temperature.gpu --format=csv,noheader,nounits -i 0 2>/dev/null; sleep 1; done ) > $OUT/ember-samp-$lab.csv & local sp=$!
|
|
sleep "$HOLD"; pkill -P $sp 2>/dev/null; kill $sp 2>/dev/null; sleep 1; kill -9 $sp 2>/dev/null
|
|
local n1; n1="$(grep -c 'STATUS' $OUT/ember-miner.log)"
|
|
local mhs; mhs="$(grep STATUS $OUT/ember-miner.log | grep -o ' now=[0-9.]*' | tail -n $((n1 - n0 > 0 ? n1 - n0 : 1)) | cut -d= -f2 | awk '{s+=$1; n++} END {if (n) printf "%.2f", s/n; else print 0}')"
|
|
local w gclk mclk tmax; read -r w gclk mclk tmax <<< "$(awk -F', *' '{w+=$1; g+=$2; m+=$3; if ($4+0 > t) t=$4+0; n++} END {if (n) printf "%.1f %.0f %.0f %d", w/n, g/n, m/n, t; else print "0 0 0 0"}' $OUT/ember-samp-$lab.csv)"
|
|
local eff; eff="$(awk -v a="$mhs" -v b="$w" 'BEGIN {if (b > 0) printf "%.4f", a / b; else print 0}')"
|
|
echo "RESULT step $lab power_pct=$pct limit_w=$lim clock_cap_mhz=$cap watts=$w mhs=$mhs mhw=$eff gclk=$gclk mclk=$mclk tmax=$tmax"
|
|
printf '%s\n' "{\"label\":\"$lab\",\"clock_mhz\":$cap,\"power_pct\":$pct,\"limit_w\":$lim,\"watts\":$w,\"mhs\":$mhs,\"eff\":$eff,\"gclk\":$gclk,\"mclk\":$mclk,\"tmax\":$tmax,\"faults\":0,\"mark\":\"ok\"}" >> $STEPS
|
|
}
|
|
# the power ladder
|
|
last=""
|
|
for pct in 100 90 80 70 60 50; do
|
|
lim="$(awk -v d="$PDEF" -v p="$pct" -v mn="$PMIN" 'BEGIN {l = d * p / 100; if (l < mn) l = mn; printf "%d", l}')"
|
|
[ "$lim" = "$last" ] && { say "power $pct% clamps to the same $lim W; skipped"; continue; }
|
|
last="$lim"; step "p$pct" "$pct" "$lim" 0
|
|
[ "$PL_OK" = 0 ] && { say "power limit not settable here; one baseline step only"; break; }
|
|
done
|
|
# choose the power point: best eff within 1% of the top rate
|
|
read -r BEST_PCT BEST_LIM <<< "$(python3 - $STEPS <<'PY'
|
|
import json, sys
|
|
s = [json.loads(l) for l in open(sys.argv[1]) if l.strip()]
|
|
top = max(x["mhs"] for x in s); ok = [x for x in s if x["mhs"] >= top * 0.99]
|
|
b = max(ok, key=lambda x: x["eff"]); print(b["power_pct"], b["limit_w"])
|
|
PY
|
|
)"
|
|
echo "RESULT power_choice pct=$BEST_PCT limit_w=$BEST_LIM"
|
|
if [ "$LGC_OK" = 1 ]; then
|
|
for cpct in 90 80 70 60; do
|
|
cap="$(awk -v c="$CMAX" -v p="$cpct" 'BEGIN {printf "%d", int(c * p / 100 / 10) * 10}')"
|
|
step "c$cpct" "$BEST_PCT" "$BEST_LIM" "$cap"
|
|
done
|
|
fi
|
|
python3 - $STEPS $OUT/ember.json "$NAME" "$DRV" "$LABEL" "$PDEF" "$CMAX" "$PL_OK" "$LGC_OK" <<'PY'
|
|
import json, sys, datetime
|
|
steps = [json.loads(l) for l in open(sys.argv[1]) if l.strip()]
|
|
name, drv, label, pdef, cmax, pl, lgc = sys.argv[3:]
|
|
top = max(x["mhs"] for x in steps); ok = [x for x in steps if x["mhs"] >= top * 0.99]
|
|
chosen = max(ok, key=lambda x: x["eff"]); base = steps[0]
|
|
rec = {"ts": datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"), "machine": "fleet-" + label, "app": "fleet-ladder", "os": "linux", "card": name, "vendor": "nvidia",
|
|
"driver": drv, "driver_major": drv.split(".")[0], "class": "v3", "key": f"nvidia|{name}|{drv.split('.')[0]}|v3",
|
|
"plan": "full" if pl == "1" else "baseline", "steps": steps, "chosen": chosen, "before": base, "eff": chosen["eff"], "mhs": chosen["mhs"], "watts": chosen["watts"],
|
|
"power_default_w": float(pdef or 0), "clock_max_mhz": float(cmax or 0), "knobs": {"power": pl == "1", "clock": lgc == "1"}}
|
|
json.dump(rec, open(sys.argv[2], "w"), indent=1)
|
|
print("TUNE " + json.dumps(rec))
|
|
print(f"RESULT choice clock_cap_mhz={chosen['clock_mhz']} power_pct={chosen['power_pct']} limit_w={chosen['limit_w']} watts={chosen['watts']} mhs={chosen['mhs']} mhw={chosen['eff']} baseline_mhs={base['mhs']} baseline_watts={base['watts']} baseline_mhw={base['eff']} steps={len(steps)}")
|
|
PY
|
|
echo "RESULT ember_done $(stamp)"
|