igneum/tools/observer/hash-origin.mjs

#!/usr/bin/env node
// The launch-week hash-origin report (mission item 10, docs/analysis/mission/mission.md 2.10; past.md verdict row 2:
// "secret hardware or fleets show up as an unexplained hashrate step in the first weeks", Nervos 2020, Iron Fish 2023,
// Kaspa 2023). Once a day, from the observer's tables, who found the blocks: how many keys, how much of the hash sits
// under pools, how much under the project's own fleet, the ten largest keys' share, and whether the network estimate
// stepped since yesterday. Daily for the first 90 days from the go; on Devnet 2 for the seven days before it.
//
//   node tools/observer/hash-origin.mjs --dry [--prefix dn2_] [--go 2026-10-20]      read the tables, print the report, write nothing
//   node tools/observer/hash-origin.mjs --write [--post [--live]] [--prefix P] [--go D] persist the day (hash_origin_days, hash_origin_reports,
//                                                                                      live_state.hash_origin) and post to #numbers
//   node tools/observer/hash-origin.mjs --fixture tools/observer/fixtures/hash-origin-day.json   run on fabricated rows, no database
//   node --test tools/observer/hash-origin.test.mjs                                    one known-finished day, one known-failed day
//
// Reads: <prefix>live_blocks (the 24 h window: vote_key_hash, evm_miner = the coinbase's payout address, color), <prefix>live_state
// (the node's hash-rate estimate), miner_logs (the vote keys the project's own intake-reporting workers logged: "identity N
// '...' vote_key_hash=<64 hex>", the same lines tools/observer/vendor-share.mjs reads), and its own two tables. Writes only
// with --write: <prefix>hash_origin_days (one row per key per day: the 30-day counts for the dust and independence rules),
// <prefix>hash_origin_reports (the day's report as it was posted), and the jsonb column hash_origin on live_state.
// DATABASE_URL from ~/.config/igneum/env (never in the repo). The project's pool payout addresses come from
// IGNEUM_PROJECT_POOLS (comma-separated IGNA addresses) so the report can mark pool-0 as the project's.
//
// Definitions (ledger-decisions.md section 3, X5; CLAUDE.md finality rule: dust = 100 blocks in the 30-day window):
//   a key "with a block" found at least one blue block in the window; "above dust" found 100 or more blue blocks in the
//   last 30 days (its own table); a "shared payout" is a payout address that two or more keys mined to in the window, which
//   is a pool OR one machine running several identities to one address (the devnet default until decision 13 of
//   ledger-decisions.md lands), so a shared payout counts as a POOL only when it is attested: the project's own pools
//   (IGNEUM_PROJECT_POOLS) or an outside pool whose operator published the signed key list X5 asks for
//   (IGNEUM_KNOWN_POOLS); "fleet" is a key one of the project's intake-reporting workers logged (miner_logs) or one listed
//   in the fleet registry's key file (IGNEUM_FLEET_KEYS_FILE, one 64-hex key per line: the rented boxes do not upload
//   identity lines, so without the file the devnet reads its own fleet as outside); "outside" is everything else.
//   The independent-key count (X5: autonomous system, machine fingerprint, pool attestation) needs two observer columns
//   that do not exist yet; the report prints the keys above dust as the upper bound and says the columns are owed.
import { readFileSync } from 'node:fs';
import { homedir } from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

export const WINDOW_H = 24;
export const DUST_BLOCKS_30D = 100;
export const REPORT_DAYS = 90;
export const OUTSIDE_POOL_SHARE = 0.10;   // reinvent.md 4.1 G-C4: a pool not run by the project with 10 percent of blocks for 7 days
export const OUTSIDE_POOL_DAYS = 7;
export const STEP_FLAG = 2;               // the network estimate doubled (or halved) since yesterday: say so

const pct = x => (x === null || x === undefined || Number.isNaN(x) ? 'n/a' : `${(x * 100).toFixed(1)}%`);
const n = x => Number(x).toLocaleString('en-GB');
const id8 = k => String(k || '').slice(0, 8);

/**
 * The report from rows already read.
 *   date: 'YYYY-MM-DD' (the day the window ends, UTC)
 *   blocks: [{ vote_key_hash, evm_miner, blue, total }] for the window
 *   fleetKeys: iterable of 64-hex vote key hashes the project's workers logged
 *   projectPools: iterable of payout addresses the project runs
 *   history: [{ day, vote_key_hash, blue }] for the last 30 days (today's rows included or not; today is taken from blocks)
 *   networkHps: the node's estimate (H/s) or null
 *   prevReports: [report] for earlier days, newest first (for the step and the outside-pool streak)
 *   goDate: 'YYYY-MM-DD' of the testnet go, or null
 */
export function compute({ date, blocks = [], fleetKeys = [], projectPools = [], knownPools = [], history = [], networkHps = null, prevReports = [], goDate = null }) {
  const fleet = new Set([...fleetKeys].map(k => String(k).toLowerCase()));
  const pools = new Set([...projectPools].map(a => String(a).toLowerCase()));
  const attested = new Set([...knownPools].map(a => String(a).toLowerCase()));
  const rows = blocks.filter(b => b.vote_key_hash && Number(b.blue) > 0).map(b => ({ key: String(b.vote_key_hash).toLowerCase(), payout: b.evm_miner ? String(b.evm_miner).toLowerCase() : null, blue: Number(b.blue), total: Number(b.total) || Number(b.blue) }));
  const total = rows.reduce((s, r) => s + r.blue, 0);

  // keys
  const byKey = new Map();
  for (const r of rows) byKey.set(r.key, (byKey.get(r.key) || 0) + r.blue);
  const keys = [...byKey.entries()].sort((a, b) => b[1] - a[1]);
  const top10 = keys.slice(0, 10).reduce((s, [, b]) => s + b, 0);
  const seenBefore = new Set(history.filter(h => h.day < date).map(h => String(h.vote_key_hash).toLowerCase()));
  const newKeys = keys.filter(([k]) => !seenBefore.has(k)).length;
  const thirty = new Map();
  for (const h of history) if (h.day < date) thirty.set(String(h.vote_key_hash).toLowerCase(), (thirty.get(String(h.vote_key_hash).toLowerCase()) || 0) + Number(h.blue));
  for (const [k, b] of keys) thirty.set(k, (thirty.get(k) || 0) + b);
  const aboveDust = [...thirty.values()].filter(b => b >= DUST_BLOCKS_30D).length;

  // shared payouts: a payout address two or more keys mined to, or a listed pool; kind = project | attested | shared
  const byPayout = new Map();
  for (const r of rows) {
    if (!r.payout) continue;
    const p = byPayout.get(r.payout) || { payout: r.payout, keys: new Set(), blue: 0 };
    p.keys.add(r.key); p.blue += r.blue; byPayout.set(r.payout, p);
  }
  const kindOf = a => (pools.has(a) ? 'project' : attested.has(a) ? 'attested' : 'shared');
  const poolRows = [...byPayout.values()].filter(p => p.keys.size >= 2 || pools.has(p.payout) || attested.has(p.payout))
    .map(p => ({ payout: p.payout, keys: p.keys.size, blue_blocks: p.blue, share: total ? p.blue / total : 0, kind: kindOf(p.payout), project: pools.has(p.payout) }))
    .sort((a, b) => b.blue_blocks - a.blue_blocks);
  const poolShare = poolRows.reduce((s, p) => s + p.share, 0);
  const attestedRows = poolRows.filter(p => p.kind !== 'shared');
  const attestedShare = attestedRows.reduce((s, p) => s + p.share, 0);

  // fleet and outside
  let fleetBlue = 0; const fleetSeen = new Set();
  for (const [k, b] of keys) if (fleet.has(k)) { fleetBlue += b; fleetSeen.add(k); }
  const outsideKeys = keys.length - fleetSeen.size;
  const outsideBlue = total - fleetBlue;

  // the step since yesterday
  const prev = prevReports[0] || null;
  const prevHps = prev && prev.network && prev.network.hps ? Number(prev.network.hps) : null;
  const hps = networkHps === null || networkHps === undefined || !(Number(networkHps) > 0) ? null : Number(networkHps);
  const ratio = hps !== null && prevHps !== null && prevHps > 0 ? hps / prevHps : null;
  const stepFlag = ratio !== null && (ratio >= STEP_FLAG || ratio <= 1 / STEP_FLAG);

  // the outside-pool streak: days in a row (today included) with an ATTESTED non-project pool at or over 10 percent
  const outsidePoolToday = poolRows.some(p => p.kind === 'attested' && p.share >= OUTSIDE_POOL_SHARE);
  let streak = outsidePoolToday ? 1 : 0;
  if (outsidePoolToday) for (const r of prevReports) { if (r.gates && r.gates.outside_pool_today) streak += 1; else break; }

  const dayOfGo = goDate ? Math.floor((Date.parse(`${date}T00:00:00Z`) - Date.parse(`${goDate}T00:00:00Z`)) / 86400e3) + 1 : null;

  const report = {
    date, window_h: WINDOW_H, day_of_go: dayOfGo, report_days: REPORT_DAYS,
    blue_blocks: total,
    keys: { with_block: keys.length, new_today: newKeys, above_dust_30d: aboveDust, dust_rule: `${DUST_BLOCKS_30D} blue blocks in 30 days`, top10_share: total ? top10 / total : null,
      largest: keys.slice(0, 10).map(([k, b]) => ({ id: id8(k), blue_blocks: b, share: total ? b / total : 0, fleet: fleet.has(k) })) },
    pools: { attested: attestedRows.length, attested_share: attestedShare, shared_payouts: poolRows.length, shared_share: poolShare,
      rows: poolRows.map(p => ({ ...p, payout: `${p.payout.slice(0, 10)}…` })),
      rule: 'a shared payout is an address two or more keys mined to in the window (a pool, or one machine with several identities); it counts as a pool when the project runs it or its operator published the signed key list' },
    fleet: { keys: fleetSeen.size, blue_blocks: fleetBlue, share: total ? fleetBlue / total : null, logged_keys: fleet.size },
    outside: { keys: outsideKeys, blue_blocks: outsideBlue, share: total ? outsideBlue / total : null },
    network: { hps, prev_hps: prevHps, ratio, step_flag: stepFlag },
    independent: { upper_bound: aboveDust, asn_and_fingerprint: 'owed: the observer does not store the autonomous system per announcing address or the machine fingerprint per key (ledger-decisions.md section 3); until it does the count above dust is an upper bound on N_ind' },
    gates: {
      first_100_keys: keys.length >= 100,
      first_block_outside_fleet: outsideBlue > 0,
      outside_pool_today: outsidePoolToday,
      outside_pool_days: streak,
      first_outside_pool: streak >= OUTSIDE_POOL_DAYS,
      first_1000_independent: false, first_1000_note: 'needs the two owed columns; never true from this report alone',
    },
  };
  report.text = renderText(report);
  return report;
}

export function renderText(r) {
  const L = [];
  L.push(`Hash origin, ${r.date}${r.day_of_go !== null ? ` (day ${r.day_of_go} of ${r.report_days})` : ''}: ${n(r.blue_blocks)} blue blocks in ${r.window_h} h.`);
  L.push(`Keys: ${n(r.keys.with_block)} found a block (${n(r.keys.new_today)} new today), ${n(r.keys.above_dust_30d)} above dust (${r.keys.dust_rule}). Ten largest keys: ${pct(r.keys.top10_share)} of blocks.`);
  L.push(`Pools: ${n(r.pools.attested)} attested, ${pct(r.pools.attested_share)} of blocks; ${n(r.pools.shared_payouts)} shared payout addresses in all (pools or multi-key machines), ${pct(r.pools.shared_share)}${r.pools.rows.length ? ' (' + r.pools.rows.slice(0, 5).map(p => `${p.kind === 'project' ? 'project pool' : p.kind === 'attested' ? 'pool' : 'shared'} ${p.payout}: ${p.keys} keys, ${pct(p.share)}`).join('; ') + ')' : ''}.`);
  L.push(`Project fleet: ${n(r.fleet.keys)} keys, ${pct(r.fleet.share)} of blocks. Outside the project: ${n(r.outside.keys)} keys, ${pct(r.outside.share)}.`);
  if (r.network.hps !== null) L.push(`Network estimate: ${(r.network.hps / 1e9).toFixed(2)} GH/s${r.network.ratio !== null ? `, ${r.network.ratio.toFixed(2)}x yesterday` : ''}${r.network.step_flag ? '. A STEP: the estimate moved by 2x or more in a day; the keys that carry it are in the list above' : ''}.`);
  else L.push('Network estimate: n/a.');
  L.push(`Independent keys (X5): at most ${n(r.independent.upper_bound)}; the autonomous-system and fingerprint columns are owed, so this is an upper bound.`);
  const g = r.gates;
  L.push(`Gates: first 100 keys ${g.first_100_keys ? 'PASSED' : 'not yet'}; first block from a card the project does not own ${g.first_block_outside_fleet ? 'PASSED' : 'not yet'}; first attested outside pool at 10 percent for 7 days ${g.first_outside_pool ? 'PASSED' : `${g.outside_pool_days} of ${OUTSIDE_POOL_DAYS} days`}.`);
  return L.join('\n');
}

// ---- the queries (read-only; the writes are in persist()) ----
const BLOCKS_SQL = T => `
  SELECT vote_key_hash, evm_miner, count(*) FILTER (WHERE color = 'blue')::int AS blue, count(*)::int AS total
  FROM ${T}live_blocks WHERE received_at > now() - interval '${WINDOW_H} hours' AND vote_key_hash IS NOT NULL GROUP BY 1, 2`;
const STATE_SQL = T => `SELECT hashes_per_second_estimate FROM ${T}live_state WHERE id = 1`;
const FLEET_KEYS_SQL = `
  SELECT DISTINCT m[1] AS key FROM miner_logs, regexp_matches(lines, 'vote_key_hash=([0-9a-f]{64})', 'g') m
  WHERE label LIKE 'miner-%' AND received_at > now() - interval '30 days'`;
const HISTORY_SQL = T => `SELECT day::text AS day, vote_key_hash, blue FROM ${T}hash_origin_days WHERE day > current_date - 30`;
const PREV_SQL = T => `SELECT report FROM ${T}hash_origin_reports WHERE day < $1::date ORDER BY day DESC LIMIT ${OUTSIDE_POOL_DAYS}`;

export async function ensureTables(sql, T = '') {
  await sql(`CREATE TABLE IF NOT EXISTS ${T}hash_origin_days (day date NOT NULL, vote_key_hash text NOT NULL, evm_miner text, blue int NOT NULL, PRIMARY KEY (day, vote_key_hash))`);
  await sql(`CREATE TABLE IF NOT EXISTS ${T}hash_origin_reports (day date PRIMARY KEY, report jsonb NOT NULL, written_at timestamptz NOT NULL DEFAULT now())`);
  await sql(`ALTER TABLE ${T}live_state ADD COLUMN IF NOT EXISTS hash_origin jsonb`);
}

/** Reads the day. DDL (the two own tables, the live_state column) runs only when `write` is set: a --dry run writes nothing, and its own tables may not exist yet (then the 30-day history and the streak are empty, and the report says so by its numbers). */
export function readKeyFile(file) {
  if (!file) return [];
  try { return readFileSync(file, 'utf8').split(/\r?\n/).map(l => l.trim().toLowerCase()).filter(l => /^[0-9a-f]{64}$/.test(l)); } catch { return []; }
}

export async function readDay(sql, { T = '', date, goDate = null, projectPools = [], knownPools = [], extraFleetKeys = [], write = false }) {
  if (write) await ensureTables(sql, T);
  const own = q => sql(q[0], q[1]).catch(e => { if (/does not exist/i.test(String(e.message || e))) return []; throw e; });
  const [blocks, state, fleet, history, prev] = await Promise.all([sql(BLOCKS_SQL(T)), sql(STATE_SQL(T)), sql(FLEET_KEYS_SQL).catch(() => []), own([HISTORY_SQL(T), []]), own([PREV_SQL(T), [date]])]);
  return compute({ date, blocks, fleetKeys: [...fleet.map(r => r.key), ...extraFleetKeys], projectPools, knownPools, history, networkHps: state[0] ? state[0].hashes_per_second_estimate : null, prevReports: prev.map(r => (typeof r.report === 'string' ? JSON.parse(r.report) : r.report)), goDate });
}

export async function persist(sql, T, report, blocks) {
  for (const b of blocks) if (b.vote_key_hash && Number(b.blue) > 0) {
    await sql(`INSERT INTO ${T}hash_origin_days (day, vote_key_hash, evm_miner, blue) VALUES ($1::date, $2, $3, $4)
               ON CONFLICT (day, vote_key_hash) DO UPDATE SET blue = EXCLUDED.blue, evm_miner = EXCLUDED.evm_miner`, [report.date, String(b.vote_key_hash).toLowerCase(), b.evm_miner || null, Number(b.blue)]);
  }
  await sql(`INSERT INTO ${T}hash_origin_reports (day, report) VALUES ($1::date, $2::jsonb) ON CONFLICT (day) DO UPDATE SET report = EXCLUDED.report, written_at = now()`, [report.date, JSON.stringify(report)]);
  await sql(`UPDATE ${T}live_state SET hash_origin = $1::jsonb WHERE id = 1`, [JSON.stringify(report)]);
}

function readDatabaseUrl() {
  if (process.env.DATABASE_URL) return process.env.DATABASE_URL;
  try {
    const env = readFileSync(path.join(homedir(), '.config', 'igneum', 'env'), 'utf8');
    const m = /^DATABASE_URL=(.+)$/m.exec(env);
    if (m) return m[1].trim().replace(/^["']|["']$/g, '');
  } catch { /* no file */ }
  throw new Error('DATABASE_URL is not set and ~/.config/igneum/env has none');
}

function makeSql(url) {
  const host = new URL(url).hostname.replace('-pooler', '');
  return async (query, params = []) => {
    const r = await fetch(`https://${host}/sql`, { method: 'POST', headers: { 'Neon-Connection-String': url, 'Content-Type': 'application/json' }, body: JSON.stringify({ query, params }) });
    const j = await r.json();
    if (!r.ok) throw new Error(j.message || JSON.stringify(j));
    return j.rows || [];
  };
}

async function postReport(report, { live = false }) {
  const hooks = await import('../community/discord-hooks.mjs');
  hooks.guardText(report.text, 'hash-origin');
  const poster = new hooks.Poster({ live });
  const payload = hooks.embed({ title: `Hash origin, ${report.date}`, url: `${hooks.SITE}/live`, description: report.text, footerNote: 'who found the blocks: keys, pools, the project fleet; daily for the first 90 days', now: new Date() });
  return poster.post('numbers', `hash-origin:${report.date}`, payload);
}

async function main() {
  const a = process.argv.slice(2);
  const flag = k => a.includes(k);
  const val = k => { const i = a.indexOf(k); return i >= 0 ? a[i + 1] : null; };
  const date = val('--date') || new Date().toISOString().slice(0, 10);
  const goDate = val('--go');
  const T = (val('--prefix') || process.env.LIVE_TABLE_PREFIX || '').replace(/[^a-z0-9_]/gi, '');
  const projectPools = (process.env.IGNEUM_PROJECT_POOLS || '').split(',').map(s => s.trim()).filter(Boolean);
  const knownPools = (process.env.IGNEUM_KNOWN_POOLS || '').split(',').map(s => s.trim()).filter(Boolean);
  const extraFleetKeys = readKeyFile(val('--fleet-keys') || process.env.IGNEUM_FLEET_KEYS_FILE);
  if (flag('--fixture')) {
    const fx = JSON.parse(readFileSync(val('--fixture'), 'utf8'));
    const r = compute({ ...fx, date: fx.date || date, goDate: fx.goDate || goDate });
    process.stdout.write(`${r.text}\n`);
    if (flag('--json')) process.stdout.write(JSON.stringify(r, null, 1) + '\n');
    return 0;
  }
  const sql = makeSql(readDatabaseUrl());
  const blocks = await sql(BLOCKS_SQL(T));
  const report = await readDay(sql, { T, date, goDate, projectPools, knownPools, extraFleetKeys, write: flag('--write') });
  process.stdout.write(`${report.text}\n`);
  if (flag('--json')) process.stdout.write(JSON.stringify(report, null, 1) + '\n');
  if (flag('--write')) {
    await persist(sql, T, report, blocks);
    process.stderr.write(`wrote ${T}hash_origin_days (${blocks.length} rows), ${T}hash_origin_reports and live_state.hash_origin for ${date}\n`);
    if (flag('--post')) { const r = await postReport(report, { live: flag('--live') }); process.stderr.write(`post: ${JSON.stringify(r)}\n`); }
  } else if (flag('--post')) { process.stderr.write('--post needs --write (a posted day is a persisted day)\n'); return 2; }
  return 0;
}

if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
  main().then(code => process.exit(code), e => { process.stderr.write(`${String(e.message || e)}\n`); process.exit(1); });
}