115 lines
9 KiB
Bash
Executable file
115 lines
9 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Igneum relay agent for the Mac (or Linux/WSL): the bash twin of igneum-agent.ps1 for `run` tasks whose body is a bash script.
|
|
# agent.sh register this machine and poll every 20 s; run each `run` task, post a result, mark it done
|
|
# agent.sh once one pass (used by the tests)
|
|
# Before anything runs (X23) the task's HMAC tag must verify with this machine's secret (machine-secret.txt next to this
|
|
# file, or RELAY_MACHINE_SECRET) over the text the Mac signed, and the nonce must be new; else exit 77 and a result.
|
|
# The token, key and secret go to curl through a header file (-K), never on the command line or in the URL (X24, X29).
|
|
# Env: RELAY_MACHINE overrides the name (default: what the relay returns for this hostname, else `hostname -s`);
|
|
# RELAY_DL_BASE reaches every task (the downloads base the agent holds, never written into a body: X26).
|
|
# State: ~/.local/state/igneum-relay (state.json, nonces.txt, headers.cfg, tasks/, logs/). Needs curl, python3 (jq optional).
|
|
set -uo pipefail
|
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
RELAY_URL='__RELAY_URL__'; RELAY_KEY='__RELAY_KEY__'; RELAY_TOKEN='__RELAY_TOKEN__'
|
|
DL_BASE_BAKED='__DL_BASE__'
|
|
export RELAY_DL_BASE="${RELAY_DL_BASE:-$DL_BASE_BAKED}"
|
|
API="$RELAY_URL/api/relay?fn="
|
|
STATE="${XDG_STATE_HOME:-$HOME/.local/state}/igneum-relay"; mkdir -p "$STATE/tasks" "$STATE/logs"; chmod 700 "$STATE"
|
|
POLL="${RELAY_POLL:-20}"; TAIL=65536
|
|
SECRET="${RELAY_MACHINE_SECRET:-}"; [ -n "$SECRET" ] || { [ -f "$HERE/machine-secret.txt" ] && SECRET="$(tr -d '[:space:]' < "$HERE/machine-secret.txt")" || SECRET=""; }
|
|
HDR="$STATE/headers.cfg"
|
|
( umask 077; { printf 'header = "x-relay-token: %s"\nheader = "x-igneum-key: %s"\n' "$RELAY_TOKEN" "$RELAY_KEY"; [ -n "$SECRET" ] && printf 'header = "x-machine-secret: %s"\n' "$SECRET"; } > "$HDR" )
|
|
log() { echo "[$(date +%H:%M:%S)] $*"; }
|
|
get() { curl -sS --max-time 60 -K "$HDR" "$API${1/\?/&}"; }
|
|
post() { curl -sS --max-time 120 -K "$HDR" -X POST "$API$1" -H 'Content-Type: application/json' --data-binary "$2"; }
|
|
py() { python3 -c "$@"; }
|
|
register() {
|
|
local info
|
|
# no username and no folder in the registration (X28)
|
|
info="$(py 'import json,platform,shutil,subprocess
|
|
gpus=[]
|
|
try:
|
|
out=subprocess.run(["system_profiler","SPDisplaysDataType"],capture_output=True,text=True,timeout=20).stdout
|
|
gpus=[l.split(":",1)[1].strip() for l in out.splitlines() if "Chipset Model" in l]
|
|
except Exception: pass
|
|
if not gpus and shutil.which("nvidia-smi"):
|
|
try: gpus=[l.strip() for l in subprocess.run(["nvidia-smi","--query-gpu=name","--format=csv,noheader"],capture_output=True,text=True,timeout=10).stdout.splitlines() if l.strip()]
|
|
except Exception: pass
|
|
print(json.dumps({"hostname":platform.node().split(".")[0],"info":{"os":platform.platform(),"gpus":gpus,"nvcc":bool(shutil.which("nvcc")),"agent":"agent.sh v2"}}))')"
|
|
local r; r="$(post register "$info")" || { log "register failed"; return 1; }
|
|
MACHINE="${RELAY_MACHINE:-$(printf '%s' "$r" | py 'import json,sys; print(json.load(sys.stdin)["name"])')}" || { log "register refused: $r"; return 1; }
|
|
ROLE="$(printf '%s' "$r" | py 'import json,sys; print(json.load(sys.stdin).get("role",""))')"
|
|
printf '%s\n' "$MACHINE" > "$STATE/machine.txt"
|
|
log "registered as $MACHINE (role ${ROLE:-unset}, bound $(printf '%s' "$r" | py 'import json,sys; print(json.load(sys.stdin).get("bound",False))'))"
|
|
[ -n "$SECRET" ] || log "no machine-secret.txt next to agent.sh: run tasks are refused until one is here"
|
|
}
|
|
check_task() { # json-of-one-item -> prints why it may not run, or nothing
|
|
RELAY_MACHINE_SECRET="$SECRET" NONCES="$STATE/nonces.txt" py 'import sys,json,hashlib,hmac,os,re
|
|
it=json.load(sys.stdin); f=it.get("flags") or {}
|
|
s=os.environ.get("RELAY_MACHINE_SECRET","")
|
|
if not s: print("this machine has no machine secret; nothing runs until machine-secret.txt is next to agent.sh"); sys.exit()
|
|
n=str(f.get("nonce","")); m=str(f.get("mac",""))
|
|
if not re.fullmatch(r"[0-9a-f]{32}", n): print("no nonce on the task"); sys.exit()
|
|
if not re.fullmatch(r"[0-9a-f]{64}", m): print("no machine tag (flags.mac) on the task"); sys.exit()
|
|
p=os.environ["NONCES"]
|
|
if os.path.exists(p) and n in open(p).read().split(): print("nonce already executed on this machine"); sys.exit()
|
|
fl=lambda v: "1" if v is True or str(v) in ("1","true") else "0"
|
|
canon="igneum-relay-run/1\nto=%s\nnonce=%s\nelevated=%s\nreboot_continue=%s\nreboot=%s\nbody_sha256=%s\n" % (it.get("to",""), n, fl(f.get("elevated")), fl(f.get("reboot_continue")), fl(f.get("reboot")), hashlib.sha256(str(it.get("body","")).encode()).hexdigest())
|
|
want=hmac.new(s.encode(), canon.encode(), hashlib.sha256).hexdigest()
|
|
if not hmac.compare_digest(want, m): print("the machine tag does not verify: not signed for this machine, or changed after signing")' <<< "$1"
|
|
}
|
|
post_result() { # id title code log note
|
|
local id="$1" title="$2" code="$3" logf="$4" note="${5:-}" body
|
|
body="$(tail -c "$TAIL" "$logf" 2>/dev/null | py 'import sys,json; print(json.dumps(sys.stdin.read()))')"
|
|
local t; t="$(printf '%s' "$title: exit $code${note:+ ($note)}" | py 'import sys,json; print(json.dumps(sys.stdin.read()))')"
|
|
post drop "{\"kind\":\"result\",\"from\":$(printf '%s' "$MACHINE" | py 'import sys,json; print(json.dumps(sys.stdin.read()))'),\"to\":\"all\",\"task_id\":$id,\"title\":$t,\"body\":$body,\"flags\":{\"exit_code\":$code,\"pass\":${RELAY_PASS:-1}}}" >/dev/null && log "result posted for #$id"
|
|
if [ "$(stat -f%z "$logf" 2>/dev/null || stat -c%s "$logf")" -gt "$TAIL" ]; then RELAY_MACHINE="$MACHINE" RELAY_TITLE="$title full log" bash "$HERE/send.sh" "$logf" >/dev/null || true; fi
|
|
}
|
|
run_task() { # json-of-one-item pass
|
|
local it="$1" pass="${2:-1}" id title body elevated rc
|
|
id="$(printf '%s' "$it" | py 'import json,sys; print(json.load(sys.stdin)["id"])')"
|
|
title="$(printf '%s' "$it" | py 'import json,sys; print(json.load(sys.stdin)["title"])')"
|
|
elevated="$(printf '%s' "$it" | py 'import json,sys; print("1" if json.load(sys.stdin)["flags"].get("elevated") else "")')"
|
|
local rebootc rebootok; rebootc="$(printf '%s' "$it" | py 'import json,sys; print("1" if json.load(sys.stdin)["flags"].get("reboot_continue") else "")')"
|
|
rebootok="$(printf '%s' "$it" | py 'import json,sys; f=json.load(sys.stdin)["flags"]; print("1" if f.get("reboot") or f.get("reboot_continue") else "")')"
|
|
local logf="$STATE/logs/task-$id-pass$pass-$(date +%Y%m%d-%H%M%S).log"
|
|
log "task #$id '$title' pass $pass${elevated:+ elevated}${rebootc:+ reboot_continue}"
|
|
local why; why="$(check_task "$it")"
|
|
if [ -n "$why" ]; then
|
|
log "task #$id REFUSED: $why"; echo "REFUSED: $why" >> "$logf"
|
|
post_result "$id" "$title" 77 "$logf" "refused: $why"; post done "{\"id\":$id,\"exit_code\":77}" >/dev/null; return 0
|
|
fi
|
|
printf '%s' "$it" | py 'import json,sys; print(json.load(sys.stdin)["flags"]["nonce"])' >> "$STATE/nonces.txt"
|
|
printf '%s' "$it" | py 'import json,sys; sys.stdout.write(json.load(sys.stdin)["body"])' > "$STATE/tasks/task-$id.sh"
|
|
if [ -n "$elevated" ]; then
|
|
RELAY_PASS="$pass" RELAY_TASK_ID="$id" RELAY_MACHINE="$MACHINE" RELAY_SEND="$HERE/send.sh" sudo -n -E bash "$STATE/tasks/task-$id.sh" > >(tee -a "$logf") 2>&1; rc=$?
|
|
else
|
|
RELAY_PASS="$pass" RELAY_TASK_ID="$id" RELAY_MACHINE="$MACHINE" RELAY_SEND="$HERE/send.sh" bash "$STATE/tasks/task-$id.sh" > >(tee -a "$logf") 2>&1; rc=$?
|
|
fi
|
|
wait 2>/dev/null; sleep 0.2
|
|
echo "__RELAY_EXIT__=$rc" >> "$logf"
|
|
# the marker on a line of its own (X28), and only for a task queued with --reboot or --reboot-continue
|
|
if grep -qx 'RELAY-REBOOT' "$logf" && [ -n "$rebootok" ]; then
|
|
if [ -n "$rebootc" ]; then
|
|
post_result "$id" "$title" "$rc" "$logf" "rebooting, resumes as pass $((pass+1))"
|
|
printf '{"pending":%s,"pass":%s}\n' "$id" "$((pass+1))" > "$STATE/state.json"
|
|
log "task asked for a reboot; re-run agent.sh after the restart to resume (no auto-restart on the Mac)"; return 0
|
|
fi
|
|
fi
|
|
post_result "$id" "$title" "$rc" "$logf"
|
|
post done "{\"id\":$id,\"exit_code\":$rc}" >/dev/null
|
|
}
|
|
one_pass() {
|
|
if [ -f "$STATE/state.json" ]; then
|
|
local pend pass; pend="$(py 'import json; d=json.load(open("'"$STATE/state.json"'")); print(d["pending"])')"; pass="$(py 'import json; d=json.load(open("'"$STATE/state.json"'")); print(d["pass"])')"
|
|
rm -f "$STATE/state.json"; run_task "$(get "item?id=$pend" | py 'import json,sys; print(json.dumps(json.load(sys.stdin)["item"]))')" "$pass"
|
|
fi
|
|
local j; j="$(post inbox "{\"machine\":$(printf '%s' "$MACHINE" | py 'import sys,json; print(json.dumps(sys.stdin.read()))'),\"kind\":\"run\",\"ack\":true}")" || { log "poll failed"; return 1; }
|
|
local n; n="$(printf '%s' "$j" | py 'import json,sys; print(len(json.load(sys.stdin)["items"]))')" || { log "poll refused: $j"; return 1; }
|
|
local i=0; while [ "$i" -lt "$n" ]; do run_task "$(printf '%s' "$j" | py 'import json,sys; print(json.dumps(json.load(sys.stdin)["items"]['"$i"']))')" 1; i=$((i+1)); done
|
|
[ "$n" = 0 ] && printf '\r[%s] idle as %s ' "$(date +%H:%M:%S)" "$MACHINE"
|
|
return 0
|
|
}
|
|
register || exit 1
|
|
if [ "${1:-}" = "once" ]; then one_pass; exit $?; fi
|
|
while true; do one_pass; sleep "$POLL"; done
|