igneum/tools/observer/lib/concentration.mjs

// Concentration and independence (ledger X5 and X14, O-X.1): pure functions shared by the observer's nightly table
// and by tools/finality-attacks/x14-concentration.mjs. No RPC, no database, no network in this file.
//
// 1. decodeFinalitySection(payload): the finality items a block's coinbase carries (spec 03 C2, C3, 3.6; the fork's
//    consensus/core/src/finality.rs: `items || len_le32 || "IGNF"` at the end of the coinbase extra data; tag 1 vote
//    280 B, tag 2 certificate, tag 3 evidence 560 B) and the key reveal (`IGNK` plus 288 hex) before it.
// 2. voterTableAt(checkpoint, blues, params): the canonical voter list at a checkpoint, rebuilt the way the node
//    builds it (consensus/src/processes/finality.rs compute_weights and refresh_voters), so a certificate's bitmap
//    can be read as key hashes.
// 3. topShares(counts): top-1, top-3, top-10 shares of a Map key -> weight.
// 4. independenceClasses(keys, attributes): N_ind, the PROPOSED definition of ledger X5 (decision item 3 for the project lead):
//    distinct (ASN, machine fingerprint, pool) classes among keys above dust.
// 5. parsePoolStatement / verifyPoolStatement: the pool attestation format, a signed JSON {pool, keys[], signed_at}.
// 6. asnOf(address, table): the autonomous system of a peer address from an offline prefix table.

import { createPublicKey, verify as edVerify } from 'node:crypto';

const VOTE_LEN = 8 + 32 + 48 + 96 + 96;   // 280
const EVIDENCE_LEN = 2 * VOTE_LEN;        // 560
const hex = (u8) => Buffer.from(u8).toString('hex');

export function payloadBytes(p) {
  if (p instanceof Uint8Array) return p;
  if (Array.isArray(p)) return Uint8Array.from(p);
  if (typeof p === 'string') return Uint8Array.from(Buffer.from(p, 'hex'));
  return new Uint8Array();
}

/** The extra data of a coinbase payload: `blue_score u64 || subsidy u64 || script version u16 || script len u8 || script || extra`. */
export function coinbaseExtra(payload) {
  const p = payloadBytes(payload);
  if (p.length < 19) return new Uint8Array();
  return p.subarray(19 + p[18]);
}

function readVote(b, o) {
  if (b.length < o + VOTE_LEN) return null;
  const dv = new DataView(b.buffer, b.byteOffset, b.byteLength);
  return {
    index: Number(dv.getBigUint64(o, true)),
    checkpoint: hex(b.subarray(o + 8, o + 40)),
    pubkey: hex(b.subarray(o + 40, o + 88)),
    signature: hex(b.subarray(o + 88, o + 184)),
    sortition: hex(b.subarray(o + 184, o + 280)),
  };
}

function readCertificate(b, s) {
  if (b.length < s + 48) return null;
  const dv = new DataView(b.buffer, b.byteOffset, b.byteLength);
  const voterCount = dv.getUint32(s + 40, true);
  const bl = dv.getUint32(s + 44, true);
  const end = s + 48 + bl + 96 + 32 + 96;
  if (bl > 1 << 20 || b.length < end) return null;
  const bitmap = b.subarray(s + 48, s + 48 + bl);
  const signerPositions = [];
  for (let i = 0; i < bitmap.length; i++) for (let bit = 0; bit < 8; bit++) if (bitmap[i] & (1 << bit)) { const pos = i * 8 + bit; if (pos < voterCount) signerPositions.push(pos); }
  return {
    index: Number(dv.getBigUint64(s, true)), checkpoint: hex(b.subarray(s + 8, s + 40)), voterCount,
    bitmap: hex(bitmap), signerPositions, signature: hex(b.subarray(s + 48 + bl, s + 144 + bl)),
    aggregator: hex(b.subarray(s + 144 + bl, s + 176 + bl)), aggregatorProof: hex(b.subarray(s + 176 + bl, end)),
    bytes: hex(b.subarray(s, end)), length: end - s,
  };
}

/**
 * Every finality item of a coinbase payload (or of its extra data: the section is a trailer, so either works) plus
 * the key reveal before it. A malformed section yields the items decoded before the fault, as the node does.
 */
export function decodeFinalitySection(payload, { raw = false } = {}) {
  const p = payloadBytes(payload);
  const extra = raw ? p : coinbaseExtra(p);
  const out = { votes: [], certificates: [], evidence: [], reveal: null, sectionBytes: 0, malformed: false };
  const n = extra.length;
  let before = extra;
  if (n >= 8 && String.fromCharCode(...extra.subarray(n - 4)) === 'IGNF') {
    const len = new DataView(extra.buffer, extra.byteOffset, n).getUint32(n - 8, true);
    if (len + 8 <= n) {
      const body = extra.subarray(n - 8 - len, n - 8);
      before = extra.subarray(0, n - 8 - len);
      out.sectionBytes = len + 8;
      let o = 0;
      while (o < body.length) {
        const tag = body[o];
        if (tag === 1) { const v = readVote(body, o + 1); if (!v) { out.malformed = true; break; } out.votes.push(v); o += 1 + VOTE_LEN; }
        else if (tag === 2) { const c = readCertificate(body, o + 1); if (!c) { out.malformed = true; break; } out.certificates.push(c); o += 1 + c.length; }
        else if (tag === 3) { const a = readVote(body, o + 1), b2 = readVote(body, o + 1 + VOTE_LEN); if (!a || !b2) { out.malformed = true; break; } out.evidence.push({ first: a, second: b2 }); o += 1 + EVIDENCE_LEN; }
        else { out.malformed = true; break; }
      }
    }
  }
  // the reveal: "IGNK" then 288 lowercase hex characters (48-byte key, 96-byte proof of possession)
  const text = Buffer.from(before).toString('latin1');
  const at = text.indexOf('IGNK');
  if (at >= 0 && text.length >= at + 4 + 288) {
    const h = text.slice(at + 4, at + 4 + 288);
    if (/^[0-9a-f]{288}$/.test(h)) out.reveal = { pubkey: h.slice(0, 96), pop: h.slice(96) };
  }
  return out;
}

/**
 * The canonical voter list at a checkpoint, as the node computes it.
 * `checkpoint`: { hash, daaScore }. `blues`: every blue block that may count, as { hash, daaScore, voteKeyHash,
 * mergerDaa } where mergerDaa is the DAA score of the chain block whose mergeset holds it (a chain block is held by
 * its child chain block; the checkpoint itself is counted whatever its merger). `params`: { weightWindow, dust }.
 * `bans`: optional Map keyHash -> DAA score the ban ends at (a stripped key leaves the list while daa < until).
 * Returns { voters: [keyHash] sorted as the node sorts (byte order of the hash), perKey: Map keyHash -> blocks,
 * total: the voters' blocks, keys: every key seen in the window }.
 */
export function voterTableAt(checkpoint, blues, params, bans = new Map()) {
  const daaC = Number(checkpoint.daaScore);
  const start = daaC - Number(params.weightWindow);
  const perKey = new Map();
  for (const b of blues) {
    const d = Number(b.daaScore);
    if (!(d > start && d <= daaC)) continue;
    if (b.hash !== checkpoint.hash && Number(b.mergerDaa) > daaC) continue;
    perKey.set(b.voteKeyHash, (perKey.get(b.voteKeyHash) || 0) + 1);
  }
  const dust = Number(params.dust);
  const voters = [...perKey.entries()].filter(([k, n]) => n >= dust && !(bans.has(k) && daaC < bans.get(k))).map(([k]) => k).sort();
  return { voters, perKey, total: voters.reduce((s, k) => s + perKey.get(k), 0), keys: perKey.size };
}

/** Top-1, top-3 and top-10 shares of a Map key -> weight (numbers), with the count of keys and the total. */
export function topShares(counts) {
  const vals = [...counts.values()].map(Number).sort((a, b) => b - a);
  const total = vals.reduce((s, v) => s + v, 0);
  const top = (n) => vals.slice(0, n).reduce((s, v) => s + v, 0) / (total || 1);
  return { keys: vals.length, total, top1: top(1), top3: top(3), top10: top(10) };
}

/**
 * Signed weight per key over a set of certificates. `certs`: [{ index, checkpoint, voterCount, signerPositions }];
 * `tables`: Map index -> the voter table at that index's checkpoint ({ voters, perKey, hash }). A certificate whose
 * checkpoint hash is not the table's, or whose voter_count is not the table's length, is counted in `unmapped` with
 * its reason and adds nothing. Returns { weight: Map key -> signed weight, count: Map key -> certificates signed,
 * mapped, unmapped: [{index, reason}] }.
 */
export function signingShares(certs, tables) {
  const weight = new Map(), count = new Map(), unmapped = [];
  let mapped = 0;
  for (const c of certs) {
    const t = tables.get(c.index);
    if (!t) { unmapped.push({ index: c.index, reason: 'no table for the index' }); continue; }
    if (t.hash && t.hash !== c.checkpoint) { unmapped.push({ index: c.index, reason: 'certificate names another block' }); continue; }
    if (t.voters.length !== c.voterCount) { unmapped.push({ index: c.index, reason: `voter_count ${c.voterCount} against a rebuilt list of ${t.voters.length}` }); continue; }
    mapped++;
    for (const p of c.signerPositions) {
      const k = t.voters[p];
      weight.set(k, (weight.get(k) || 0) + t.perKey.get(k));
      count.set(k, (count.get(k) || 0) + 1);
    }
  }
  return { weight, count, mapped, unmapped };
}

// ---------------------------------------------------------------------------------------------
// Independence (ledger X5, decision item 3): PROPOSED definition, not adopted

/**
 * N_ind under the proposed definition: the number of distinct (ASN, machine fingerprint, pool) classes among the
 * keys above dust. `keys`: [{ keyHash, blocks }]; `attributes`: Map keyHash -> { asn, fingerprint, pool } with any
 * field null when unknown. The decision request's addition: a key with no fingerprint (a miner that sends no logs)
 * is its own class only when its ASN is used by no other key; otherwise it joins the class of that ASN with no
 * fingerprint and no pool. Returns { nInd, classes: Map classKey -> [keyHash], eligible, unattributed }.
 */
export function independenceClasses(keys, attributes, dust) {
  const eligible = keys.filter(k => Number(k.blocks) >= Number(dust));
  const asnUsers = new Map();
  for (const k of eligible) { const a = attributes.get(k.keyHash); if (a && a.asn) asnUsers.set(a.asn, (asnUsers.get(a.asn) || 0) + 1); }
  const classes = new Map();
  let unattributed = 0;
  for (const k of eligible) {
    const a = attributes.get(k.keyHash) || {};
    let cls;
    if (!a.asn && !a.fingerprint && !a.pool) { unattributed++; cls = `unattributed:${k.keyHash}`; }
    else if (!a.fingerprint) cls = asnUsers.get(a.asn) > 1 ? `asn:${a.asn}|fp:-|pool:${a.pool || '-'}` : `asn:${a.asn}|solo:${k.keyHash}`;
    else cls = `asn:${a.asn || '-'}|fp:${a.fingerprint}|pool:${a.pool || '-'}`;
    if (!classes.has(cls)) classes.set(cls, []);
    classes.get(cls).push(k.keyHash);
  }
  return { nInd: classes.size, classes, eligible: eligible.length, unattributed };
}

// ---------------------------------------------------------------------------------------------
// Pool statement: {pool, keys[], signed_at} signed by the pool's Ed25519 key

export const POOL_STATEMENT_FORMAT = 'igneum-pool-statement-1';

/**
 * The bytes a pool signs: the canonical JSON of {format, pool, keys (sorted, lowercase hex), signed_at} with the
 * keys in that order and no whitespace, so two encoders agree byte for byte.
 */
export function poolStatementBytes({ pool, keys, signed_at }) {
  const body = { format: POOL_STATEMENT_FORMAT, pool, keys: [...keys].map(k => String(k).toLowerCase()).sort(), signed_at };
  return Buffer.from(JSON.stringify(body), 'utf8');
}

/**
 * Parses a pool statement envelope: JSON text of {format, pool, keys[], signed_at, pubkey, sig}. Returns
 * { ok, statement, error }. Shape only; verifyPoolStatement checks the signature.
 */
export function parsePoolStatement(text) {
  let j;
  try { j = typeof text === 'string' ? JSON.parse(text) : text; } catch (e) { return { ok: false, error: `not JSON: ${e.message}` }; }
  if (!j || typeof j !== 'object') return { ok: false, error: 'not an object' };
  if (j.format !== POOL_STATEMENT_FORMAT) return { ok: false, error: `format is ${JSON.stringify(j.format)}, want ${POOL_STATEMENT_FORMAT}` };
  if (typeof j.pool !== 'string' || !/^[a-z0-9][a-z0-9.-]{0,62}$/.test(j.pool)) return { ok: false, error: 'pool must be a lowercase label (letters, digits, dot, dash, at most 63 characters)' };
  if (!Array.isArray(j.keys) || j.keys.length === 0 || j.keys.length > 100000) return { ok: false, error: 'keys must be a non-empty list (at most 100,000)' };
  for (const k of j.keys) if (typeof k !== 'string' || !/^[0-9a-fA-F]{64}$/.test(k)) return { ok: false, error: `key ${JSON.stringify(k)} is not a 64-hex vote key hash` };
  if (new Set(j.keys.map(k => k.toLowerCase())).size !== j.keys.length) return { ok: false, error: 'keys repeat' };
  if (typeof j.signed_at !== 'string' || Number.isNaN(Date.parse(j.signed_at)) || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z$/.test(j.signed_at)) return { ok: false, error: 'signed_at must be an ISO 8601 UTC instant' };
  if (typeof j.pubkey !== 'string' || !/^[0-9a-f]{64}$/.test(j.pubkey)) return { ok: false, error: 'pubkey must be a 32-byte Ed25519 key in hex' };
  if (typeof j.sig !== 'string' || !/^[0-9a-f]{128}$/.test(j.sig)) return { ok: false, error: 'sig must be a 64-byte Ed25519 signature in hex' };
  return { ok: true, statement: { pool: j.pool, keys: j.keys.map(k => k.toLowerCase()).sort(), signed_at: j.signed_at, pubkey: j.pubkey, sig: j.sig } };
}

/**
 * Verifies a parsed statement's Ed25519 signature against the pool's registered key (hex). `registry` maps the pool
 * label to its key; a statement signed by a key that is not the pool's is refused even when the signature verifies.
 * `maxAgeMs` refuses a statement older than that (default 35 days: one weight window plus slack).
 */
export function verifyPoolStatement(statement, registry, { now = Date.now(), maxAgeMs = 35 * 86400e3 } = {}) {
  const want = registry instanceof Map ? registry.get(statement.pool) : registry?.[statement.pool];
  if (!want) return { ok: false, error: `pool ${statement.pool} has no registered key` };
  if (want.toLowerCase() !== statement.pubkey) return { ok: false, error: `signed by ${statement.pubkey.slice(0, 8)}, the pool's key is ${want.slice(0, 8)}` };
  const age = now - Date.parse(statement.signed_at);
  if (age < -300e3) return { ok: false, error: 'signed_at is in the future' };
  if (age > maxAgeMs) return { ok: false, error: `signed ${Math.round(age / 86400e3)} days ago, over the ${Math.round(maxAgeMs / 86400e3)}-day limit` };
  let key;
  try { key = createPublicKey({ key: Buffer.concat([Buffer.from('302a300506032b6570032100', 'hex'), Buffer.from(statement.pubkey, 'hex')]), format: 'der', type: 'spki' }); }
  catch (e) { return { ok: false, error: `bad public key: ${e.message}` }; }
  const ok = edVerify(null, poolStatementBytes(statement), key, Buffer.from(statement.sig, 'hex'));
  return ok ? { ok: true } : { ok: false, error: 'signature does not verify' };
}

// ---------------------------------------------------------------------------------------------
// Autonomous system of a peer address: an offline prefix table, no third-party lookups

/** Parses a table of `prefix asn [name]` lines (IPv4 CIDR), # comments allowed. Returns [{ net, bits, asn, name }]. */
export function parseAsnTable(text) {
  const rows = [];
  for (const raw of String(text).split('\n')) {
    const line = raw.replace(/#.*/, '').trim();
    if (!line) continue;
    const m = /^(\d+\.\d+\.\d+\.\d+)\/(\d+)\s+(\S+)(?:\s+(.*))?$/.exec(line);
    if (!m) continue;
    const bits = Number(m[2]);
    if (bits < 0 || bits > 32) continue;
    rows.push({ net: ipv4(m[1]), bits, asn: m[3], name: (m[4] || '').trim() });
  }
  return rows.sort((a, b) => b.bits - a.bits);
}

function ipv4(s) {
  const p = s.split('.').map(Number);
  if (p.length !== 4 || p.some(x => !(x >= 0 && x <= 255))) return null;
  return ((p[0] << 24) | (p[1] << 16) | (p[2] << 8) | p[3]) >>> 0;
}

/**
 * The autonomous system of an address (`ip`, `ip:port` or `[v6]:port`) from the parsed table, longest prefix first.
 * Private and loopback ranges answer `local`; an IPv6 address or a miss answers null (the stub: the table is the
 * offline lookup; a live lookup against a BGP source is named in the README and not called from here).
 */
export function asnOf(address, table) {
  const s = String(address || '').trim();
  if (s.startsWith('[')) return null;
  const host = s.includes(':') ? s.split(':')[0] : s;
  const n = ipv4(host);
  if (n === null) return null;
  const a = n >>> 24, b = (n >>> 16) & 255;
  if (a === 10 || a === 127 || (a === 192 && b === 168) || (a === 172 && b >= 16 && b <= 31) || (a === 169 && b === 254)) return { asn: 'local', name: 'private or loopback' };
  for (const r of table) if (r.net !== null && (r.bits === 0 || ((n ^ r.net) >>> (32 - r.bits)) === 0)) return { asn: r.asn, name: r.name };
  return null;
}