diff --git a/infra/build-server/discord-hooks/install.sh b/infra/build-server/discord-hooks/install.sh index 25ae612b4..f8d5586db 100755 --- a/infra/build-server/discord-hooks/install.sh +++ b/infra/build-server/discord-hooks/install.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Install or refresh the Discord webhooks scheduler on igneum-build-1 from this Mac. -# IGNEUM_SECRET_ON_BOX_OK=1 infra/build-server/discord-hooks/install.sh +# IGNEUM_SECRET_ON_BOX_OK=1 infra/build-server/discord-hooks/install.sh (re-run whenever the credentials file changes) # Copies tools/community/discord-hooks.mjs to /srv/discord-hooks/bin, the webhook file ~/.config/igneum/discord to # /srv/discord-hooks/env (mode 600, owner build; never into the repository), the two units, and enables the timer. # Needs root over ssh (root@ with ~/.ssh/igneum_ed25519); the host ip comes from ~/.config/igneum/build-server (build@). @@ -16,9 +16,14 @@ HOST_LINE="$(head -1 "${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-serve IP="${HOST_LINE#*@}"; [ -n "$IP" ] || { echo "no build server in ~/.config/igneum/build-server" >&2; exit 1; } [ "${IGNEUM_SECRET_ON_BOX_OK:-}" = "1" ] || { echo "refusing: the box holds no secret by rule; set IGNEUM_SECRET_ON_BOX_OK=1 once the coordinator has ruled the exception" >&2; exit 1; } [ -f "$CRED" ] || { echo "no credentials file at $CRED (DISCORD_WEBHOOK_NUMBERS, _ANNOUNCEMENTS, _INCIDENTS)" >&2; exit 1; } +# a partial file is accepted (main's ruling, 6 October 2026, 20:1x UK): at least one key, the missing ones named here and +# by every tick's log line; re-run this script when the other webhooks are created to copy the fuller file +present=0; missing="" for k in DISCORD_WEBHOOK_NUMBERS DISCORD_WEBHOOK_ANNOUNCEMENTS DISCORD_WEBHOOK_INCIDENTS; do - grep -q "^$k=" "$CRED" || { echo "credentials file lacks $k" >&2; exit 1; } + if grep -q "^$k=." "$CRED"; then present=$((present + 1)); else missing="$missing $k"; fi done +[ "$present" -ge 1 ] || { echo "credentials file has none of the three webhook keys" >&2; exit 1; } +[ -z "$missing" ] || echo "note: missing${missing}; posts to those channels are logged, not sent, until the file is re-copied" >&2 MODE="$(stat -f %Lp "$CRED" 2>/dev/null || stat -c %a "$CRED")" [ "$MODE" = "600" ] || { echo "credentials file must be mode 600 (is $MODE)" >&2; exit 1; } @@ -30,5 +35,5 @@ scp -q -i "$KEY" "$ROOT/tools/community/discord-hooks.mjs" "build@$IP:/srv/disco scp -q -i "$KEY" "$HERE/igneum-discord-hooks.service" "$HERE/igneum-discord-hooks.timer" "root@$IP:/etc/systemd/system/" "${SSH[@]}" "root@$IP" 'systemctl daemon-reload && systemctl enable --now igneum-discord-hooks.timer >/dev/null 2>&1; systemctl is-active igneum-discord-hooks.timer; systemctl list-timers igneum-discord-hooks.timer --no-pager | sed -n 2p' # one check pass as the unit's user: names only, never values -"${SSH[@]}" "build@$IP" 'IGNEUM_DISCORD_ENV=/srv/discord-hooks/env /usr/local/bin/node /srv/discord-hooks/bin/discord-hooks.mjs check' +"${SSH[@]}" "build@$IP" 'IGNEUM_DISCORD_ENV=/srv/discord-hooks/env /usr/local/bin/node /srv/discord-hooks/bin/discord-hooks.mjs check' || true # exit 1 on a partial file is the note above, not a failure echo "installed; the first tick runs within a minute: journalctl -u igneum-discord-hooks -n 20" diff --git a/tools/community/discord-hooks.mjs b/tools/community/discord-hooks.mjs index 1c67936be..dbb9bae58 100755 --- a/tools/community/discord-hooks.mjs +++ b/tools/community/discord-hooks.mjs @@ -560,6 +560,9 @@ export class Poster { this.state.posts = this.state.posts || {}; this.state.pulses = this.state.pulses || []; this.state.incidents = this.state.incidents || {}; this.state.watch = this.state.watch || {}; } already(key) { return !this.force && !!this.state.posts[key]; } + // the webhook keys the credentials file lacks (names only); empty in dry run + missingKeys() { return this.live ? Object.values(CHANNEL_KEY).filter(k => !this.creds[k]) : []; } + has(channel) { return !this.live || !!this.creds[CHANNEL_KEY[channel]]; } // Posts one payload under an idempotency key; returns {posted, skipped, id}. Writes the dry-run JSON and preview in dry-run mode. async post(channel, key, payload) { const total = guardPayload(payload); @@ -710,6 +713,12 @@ export async function resolveIncident(poster, { id, at, cause, fix }) { export async function runWatch(poster, data) { const actions = watchPass(data, poster.state); const results = []; + if (actions.length && !poster.has('incidents')) { + // the state still advances (no backlog flood when the key lands); the action is logged, not sent + for (const a of actions) poster.log(`watch: ${a.kind} ${a.id} not posted, no ${CHANNEL_KEY.incidents} in the credentials file`); + poster.save(); + return []; + } for (const a of actions) { if (a.kind === 'open') results.push(await openIncident(poster, { id: a.id, at: a.at, what: a.what, affected: a.affected, doing: a.doing, auto: true })); else results.push(await resolveIncident(poster, { id: a.id, at: a.at, cause: a.cause, fix: a.fix })); @@ -777,7 +786,8 @@ export async function main(argv = process.argv.slice(2)) { } const wdata = data && !data.fetchFailed ? data : await fetchForWatch(); const r = await runWatch(poster, wdata); - poster.log(`tick ${ukStamp(now)}: ${due.length} due, watch ${r.length} action(s)`); + const missing = poster.missingKeys(); + poster.log(`tick ${ukStamp(now)}: ${due.length} due, watch ${r.length} action(s)${missing.length ? `, missing ${missing.join(', ')}` : ''}`); return 0; } throw new Error(`unknown command ${cmd}`); diff --git a/tools/community/discord-hooks.test.mjs b/tools/community/discord-hooks.test.mjs index 0be48114c..9d43d4ac7 100644 --- a/tools/community/discord-hooks.test.mjs +++ b/tools/community/discord-hooks.test.mjs @@ -304,6 +304,24 @@ test('poster: live mode posts once, stores the message id, skips the rerun; the assert.equal(calls.length, 1); await assert.rejects(() => p.post('incidents', 'inc:x', payload), /no DISCORD_WEBHOOK_INCIDENTS/); }); +test('poster: a partial credentials file names its missing keys; the watcher advances its state and posts nothing without the incidents webhook', async () => { + const dir = tmpDir(); + const cred = path.join(dir, 'discord'); fs.writeFileSync(cred, 'DISCORD_WEBHOOK_NUMBERS=https://discord.com/api/webhooks/1/secret\n', { mode: 0o600 }); + const calls = []; + const fetchImpl = async (url, init) => { calls.push(url); return { ok: true, status: 200, headers: new Map(), json: async () => ({ id: '1' }) }; }; + const logs = []; + const p = new Poster({ live: true, credFile: cred, stateFile: path.join(dir, 'state.json'), outDir: dir, log: m => logs.push(m), fetchImpl }); + assert.deepEqual(p.missingKeys(), ['DISCORD_WEBHOOK_ANNOUNCEMENTS', 'DISCORD_WEBHOOK_INCIDENTS']); + assert.equal(p.has('numbers'), true); assert.equal(p.has('incidents'), false); + const { runWatch } = await import('./discord-hooks.mjs'); + watchPass(data(), p.state, at(NOW, -60)); + const lag = data(); lag.live.proving.median_proof_lag_s = 1000; + const r = await runWatch(p, lag); + assert.deepEqual(r, []); + assert.equal(calls.length, 0, 'nothing sent'); + assert.match(logs.join('\n'), /open auto-proof_lag-\S+ not posted, no DISCORD_WEBHOOK_INCIDENTS/); + assert.ok(p.state.watch.proof_lag.open, 'the state still records the open, so the key landing later does not flood'); +}); test('backoff: a 429 waits retry_after then doubles; success returns the id; six failures give up', async () => { const sleeps = []; let n = 0;