diff --git a/tools/attack/adv-cache-2/Cargo.lock b/tools/attack/adv-cache-2/Cargo.lock new file mode 100644 index 000000000..d81676b05 --- /dev/null +++ b/tools/attack/adv-cache-2/Cargo.lock @@ -0,0 +1,14 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "attack-adv-cache-2" +version = "0.1.0" +dependencies = [ + "igneum-pow", +] + +[[package]] +name = "igneum-pow" +version = "0.2.0" diff --git a/tools/attack/adv-cache-2/Cargo.toml b/tools/attack/adv-cache-2/Cargo.toml new file mode 100644 index 000000000..4db63cf6d --- /dev/null +++ b/tools/attack/adv-cache-2/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "attack-adv-cache-2" +version = "0.1.0" +edition = "2021" +description = "Adversarial lane adv-cache-2: partial-state and hot-set censuses of the chained cache and the item derivation of class v4 sub-version 3, on the real igneum-pow derivation and interpreter (internal adversarial pass, not an independent review)" +license = "MIT" +publish = false + +[[bin]] +name = "adv-cache-2" +path = "src/main.rs" + +[dependencies] +igneum-pow = { path = "../../../igneum-pow" } + +[workspace] + +[profile.release] +opt-level = 3 +lto = true +codegen-units = 1 diff --git a/tools/attack/adv-cache-2/drain.sh b/tools/attack/adv-cache-2/drain.sh new file mode 100755 index 000000000..aaa207dd9 --- /dev/null +++ b/tools/attack/adv-cache-2/drain.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# adv-cache-2 queue drain, run ON a build box with nohup: claims and runs every unclaimed file of +# /srv/builds/_adv/cache/queue/ in name order, one at a time, no gap between them (the brief's queue rule). A file is +# claimed by `mkdir claims/` (atomic; exists = taken, by this lane or a sibling). Files whose name does not carry +# `adv-cache-2` belong to a sibling lane; they are run too when nothing of ours is left, and listed in drain.log. +# Stop the drain by its pid file; a running item is stopped by its own pid file, never by name. +set -uo pipefail +Q=/srv/builds/_adv/cache/queue; C=/srv/builds/_adv/cache/claims; R=/srv/builds/_adv-adv-cache-2 +mkdir -p "$Q" "$C" "$R"; echo $$ > "$R/drain.pid" +say() { echo "[$(date -u +%FT%TZ)] $*" >> "$R/drain.log"; } +ran=0 +while :; do + picked="" + # ours first, then any sibling file, name order + for pass in own any; do + for f in $(ls "$Q" 2>/dev/null | sort); do + case "$pass" in own) case "$f" in *adv-cache-2*) ;; *) continue ;; esac ;; esac + [ -x "$Q/$f" ] || continue + if mkdir "$C/$f" 2>/dev/null; then picked="$f"; break 2; fi + done + done + [ -n "$picked" ] || { say "queue empty after $ran items; exiting"; break; } + owner=$(echo "$picked" | sed -E 's/^[0-9]+-(adv-[a-z0-9-]*?)-.*/\1/' | sed -E 's/^(adv-[a-z]+(-[0-9]+)?).*/\1/') + say "start $picked (owner $owner)" + t0=$(date +%s) + setsid "$Q/$picked"; rc=$? + say "end $picked rc $rc after $(( $(date +%s) - t0 )) s" + ran=$((ran+1)) +done diff --git a/tools/attack/adv-cache-2/queue/10-adv-cache-2-lines-2e28.sh b/tools/attack/adv-cache-2/queue/10-adv-cache-2-lines-2e28.sh new file mode 100755 index 000000000..1801ea07a --- /dev/null +++ b/tools/attack/adv-cache-2/queue/10-adv-cache-2-lines-2e28.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# adv-cache-2 queue item (self-contained). Owner: lane adv-cache-2. Internal adversarial pass, not an independent review. +R=/srv/builds/_adv-adv-cache-2; mkdir -p $R; echo $$ > $R/lines-2e28.pid +exec nice -n 10 taskset -c 8-95 $R/bin/adv-cache-2 lines --day0 20730 --days 16 --items-log2 24 --threads 80 --validate sample --out $R > $R/lines-2e28.log 2>&1 diff --git a/tools/attack/adv-cache-2/queue/11-adv-cache-2-steer-2e20.sh b/tools/attack/adv-cache-2/queue/11-adv-cache-2-steer-2e20.sh new file mode 100755 index 000000000..280341b51 --- /dev/null +++ b/tools/attack/adv-cache-2/queue/11-adv-cache-2-steer-2e20.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# adv-cache-2 queue item (self-contained). Owner: lane adv-cache-2. Internal adversarial pass, not an independent review. +R=/srv/builds/_adv-adv-cache-2; mkdir -p $R; echo $$ > $R/steer-2e20.pid +exec nice -n 10 taskset -c 8-95 $R/bin/adv-cache-2 steer --day 20730 --items-log2 20 --threads 80 --out $R > $R/steer-2e20.log 2>&1 diff --git a/tools/attack/adv-cache-2/queue/12-adv-cache-2-steer-2e20-d20733.sh b/tools/attack/adv-cache-2/queue/12-adv-cache-2-steer-2e20-d20733.sh new file mode 100755 index 000000000..427008280 --- /dev/null +++ b/tools/attack/adv-cache-2/queue/12-adv-cache-2-steer-2e20-d20733.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# adv-cache-2 queue item (self-contained). Owner: lane adv-cache-2. Internal adversarial pass, not an independent review. +R=/srv/builds/_adv-adv-cache-2; mkdir -p $R; echo $$ > $R/steer-2e20-d20733.pid +exec nice -n 10 taskset -c 8-95 $R/bin/adv-cache-2 steer --day 20733 --items-log2 20 --threads 80 --out $R > $R/steer-2e20-d20733.log 2>&1 diff --git a/tools/attack/adv-cache-2/queue/13-adv-cache-2-windows-fixed-4096.sh b/tools/attack/adv-cache-2/queue/13-adv-cache-2-windows-fixed-4096.sh new file mode 100755 index 000000000..24404d8f7 --- /dev/null +++ b/tools/attack/adv-cache-2/queue/13-adv-cache-2-windows-fixed-4096.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# adv-cache-2 queue item (self-contained). Owner: lane adv-cache-2. Internal adversarial pass, not an independent review. +R=/srv/builds/_adv-adv-cache-2; mkdir -p $R; echo $$ > $R/windows-fixed-4096.pid +exec nice -n 10 taskset -c 8-95 $R/bin/adv-cache-2 windows --programs era-fixed:1..4096 --threads 80 --out $R > $R/windows-fixed-4096.log 2>&1 diff --git a/tools/attack/adv-cache-2/queue/14-adv-cache-2-windows-drawn-4096.sh b/tools/attack/adv-cache-2/queue/14-adv-cache-2-windows-drawn-4096.sh new file mode 100755 index 000000000..5826d472d --- /dev/null +++ b/tools/attack/adv-cache-2/queue/14-adv-cache-2-windows-drawn-4096.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# adv-cache-2 queue item (self-contained). Owner: lane adv-cache-2. Internal adversarial pass, not an independent review. +R=/srv/builds/_adv-adv-cache-2; mkdir -p $R; echo $$ > $R/windows-drawn-4096.pid +exec nice -n 10 taskset -c 8-95 $R/bin/adv-cache-2 windows --programs era-drawn:1..4096 --threads 80 --out $R > $R/windows-drawn-4096.log 2>&1 diff --git a/tools/attack/adv-cache-2/queue/15-adv-cache-2-days-16384.sh b/tools/attack/adv-cache-2/queue/15-adv-cache-2-days-16384.sh new file mode 100755 index 000000000..524dd8638 --- /dev/null +++ b/tools/attack/adv-cache-2/queue/15-adv-cache-2-days-16384.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# adv-cache-2 queue item (self-contained). Owner: lane adv-cache-2. Internal adversarial pass, not an independent review. +R=/srv/builds/_adv-adv-cache-2; mkdir -p $R; echo $$ > $R/days-16384.pid +exec nice -n 10 taskset -c 8-95 $R/bin/adv-cache-2 days --day0 20730 --days 16384 --items-log2 16 --threads 48 --out $R > $R/days-16384.log 2>&1 diff --git a/tools/attack/adv-cache-2/queue/16-adv-cache-2-lines-2e32.sh b/tools/attack/adv-cache-2/queue/16-adv-cache-2-lines-2e32.sh new file mode 100755 index 000000000..a202b9734 --- /dev/null +++ b/tools/attack/adv-cache-2/queue/16-adv-cache-2-lines-2e32.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# adv-cache-2 queue item (self-contained). Owner: lane adv-cache-2. Internal adversarial pass, not an independent review. +R=/srv/builds/_adv-adv-cache-2; mkdir -p $R; echo $$ > $R/lines-2e32.pid +exec nice -n 10 taskset -c 8-95 $R/bin/adv-cache-2 lines --day0 20746 --days 256 --items-log2 24 --threads 80 --validate sample --out $R > $R/lines-2e32.log 2>&1 diff --git a/tools/attack/adv-cache-2/queue/20-adv-cache-2-warps-devnet3-2e24-fp.sh b/tools/attack/adv-cache-2/queue/20-adv-cache-2-warps-devnet3-2e24-fp.sh new file mode 100755 index 000000000..d169ec587 --- /dev/null +++ b/tools/attack/adv-cache-2/queue/20-adv-cache-2-warps-devnet3-2e24-fp.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# adv-cache-2 queue item (self-contained). Owner: lane adv-cache-2. Internal adversarial pass, not an independent review. +R=/srv/builds/_adv-adv-cache-2; mkdir -p $R; echo $$ > $R/warps-devnet3-2e24-fp.pid +exec nice -n 10 taskset -c 8-95 $R/bin/adv-cache-2 warps --programs devnet3 --day 20733 --nonces 16777216 --threads 80 --validate sample --fingerprint 1 --out $R > $R/warps-devnet3-2e24-fp.log 2>&1 diff --git a/tools/attack/adv-cache-2/queue/21-adv-cache-2-warps-devnet-2e26.sh b/tools/attack/adv-cache-2/queue/21-adv-cache-2-warps-devnet-2e26.sh new file mode 100755 index 000000000..fbdb661bd --- /dev/null +++ b/tools/attack/adv-cache-2/queue/21-adv-cache-2-warps-devnet-2e26.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# adv-cache-2 queue item (self-contained). Owner: lane adv-cache-2. Internal adversarial pass, not an independent review. +R=/srv/builds/_adv-adv-cache-2; mkdir -p $R; echo $$ > $R/warps-devnet-2e26.pid +exec nice -n 10 taskset -c 8-95 $R/bin/adv-cache-2 warps --programs devnet --day 20730 --nonces 67108864 --threads 80 --validate sample --out $R > $R/warps-devnet-2e26.log 2>&1 diff --git a/tools/attack/adv-cache-2/queue/22-adv-cache-2-warps-devnet3-2e26.sh b/tools/attack/adv-cache-2/queue/22-adv-cache-2-warps-devnet3-2e26.sh new file mode 100755 index 000000000..bf1f60359 --- /dev/null +++ b/tools/attack/adv-cache-2/queue/22-adv-cache-2-warps-devnet3-2e26.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# adv-cache-2 queue item (self-contained). Owner: lane adv-cache-2. Internal adversarial pass, not an independent review. +R=/srv/builds/_adv-adv-cache-2; mkdir -p $R; echo $$ > $R/warps-devnet3-2e26.pid +exec nice -n 10 taskset -c 8-95 $R/bin/adv-cache-2 warps --programs devnet3 --day 20733 --nonces 67108864 --threads 80 --validate sample --out $R > $R/warps-devnet3-2e26.log 2>&1 diff --git a/tools/attack/adv-cache-2/queue/23-adv-cache-2-warps-era-fixed-32-2e24.sh b/tools/attack/adv-cache-2/queue/23-adv-cache-2-warps-era-fixed-32-2e24.sh new file mode 100755 index 000000000..c980bc626 --- /dev/null +++ b/tools/attack/adv-cache-2/queue/23-adv-cache-2-warps-era-fixed-32-2e24.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# adv-cache-2 queue item (self-contained). Owner: lane adv-cache-2. Internal adversarial pass, not an independent review. +R=/srv/builds/_adv-adv-cache-2; mkdir -p $R; echo $$ > $R/warps-era-fixed-32-2e24.pid +exec nice -n 10 taskset -c 8-95 $R/bin/adv-cache-2 warps --programs era-fixed:1..32 --day 20730 --nonces 16777216 --threads 80 --validate sample --out $R > $R/warps-era-fixed-32-2e24.log 2>&1 diff --git a/tools/attack/adv-cache-2/queue/24-adv-cache-2-warps-era-drawn-32-2e24.sh b/tools/attack/adv-cache-2/queue/24-adv-cache-2-warps-era-drawn-32-2e24.sh new file mode 100755 index 000000000..fd9763e29 --- /dev/null +++ b/tools/attack/adv-cache-2/queue/24-adv-cache-2-warps-era-drawn-32-2e24.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# adv-cache-2 queue item (self-contained). Owner: lane adv-cache-2. Internal adversarial pass, not an independent review. +R=/srv/builds/_adv-adv-cache-2; mkdir -p $R; echo $$ > $R/warps-era-drawn-32-2e24.pid +exec nice -n 10 taskset -c 8-95 $R/bin/adv-cache-2 warps --programs era-drawn:1..32 --day 20730 --nonces 16777216 --threads 80 --validate sample --out $R > $R/warps-era-drawn-32-2e24.log 2>&1 diff --git a/tools/attack/adv-cache-2/run-box.sh b/tools/attack/adv-cache-2/run-box.sh new file mode 100755 index 000000000..16d7ec7c3 --- /dev/null +++ b/tools/attack/adv-cache-2/run-box.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# adv-cache-2 sweep runner, run ON a build box (internal adversarial pass, not an independent review). +# run-box.sh +# Starts `nice -n 10 taskset -c 8-95 ` detached in its own session, log and pid file under +# /srv/builds/_adv-adv-cache-2/ (outside the worktree mirror, which build-remote.sh's sync would wipe), result files +# in the same directory. Cores 0 to 7 are never used (reserved for release builds, the seed and the observer). +# Rule history: the plan's SIGSTOP yield-to-builds loop was dropped by the coordinator's order at 19:3x BST on +# 7 October 2026 (a build slot is held nearly all the time on both boxes, so a yielding sweep never ran); the sweep +# runs at nice 10 and the scheduler arbitrates. Kill by pid file only: `kill -- -$(cat )`. +set -uo pipefail +NAME="$1"; shift +BIN="$1"; shift +ROOT=/srv/builds/_adv-adv-cache-2 +mkdir -p "$ROOT" +LOG="$ROOT/$NAME.log" +PID="$ROOT/$NAME.pid" +if [ -f "$PID" ] && kill -0 "$(cat "$PID")" 2>/dev/null; then echo "already running: $NAME pid $(cat "$PID")"; exit 1; fi +setsid nohup nice -n 10 taskset -c 8-95 "$BIN" "$@" --out "$ROOT" > "$LOG" 2>&1 & +P=$! +echo "$P" > "$PID" +echo "started $NAME pid $P log $LOG" diff --git a/tools/attack/adv-cache-2/src/main.rs b/tools/attack/adv-cache-2/src/main.rs new file mode 100644 index 000000000..6866ddd7b --- /dev/null +++ b/tools/attack/adv-cache-2/src/main.rs @@ -0,0 +1,1473 @@ +//! adv-cache-2: partial-state and hot-set censuses of the chained cache (internal adversarial pass, not an +//! independent review). Everything runs on the real `igneum-pow` derivation and interpreter; the two traced +//! mirrors (the item derivation with its 8 line indices recorded, the warp interpreter with every load's item +//! index recorded) are checked word for word against `derive_items` and `Epoch::hash_warp` on every run. +//! +//! Commands (see `usage`): +//! selftest the pack vectors: cache FNV of days 20730 and 20733, the two real program ids, warp 0 lanes 0..4 +//! lines Q1: the line index over 2^22 lines, per round and pooled, segments, depth, distinct per item +//! days Q3(2): the weak-day scan, one cache per day, segment and depth histograms per day +//! warps Q2: the hot set of items and lines across the hashes of one or more programs, per site +//! steer Q3(1): bit sensitivity of the line index to the item index t +//! windows Q3(3): the window layer's quarter and half shares over drawn programs +//! +//! Plant hooks (the known-failed shapes): `--plant quarter-lines`, `--plant half-lines` (lines, days), +//! `--plant const-item` (warps), `--plant t-low` (steer), `--plant all-quarter` (windows). Under a plant the +//! library comparisons are skipped and the log says so. + +use igneum_pow::bind::{day_bytes, hex, unhex}; +use igneum_pow::generator::{EraParams, Instr, Op, Program, ProgramClass, ITERATIONS, LANES}; +use igneum_pow::memhard::{mixer, round_key_mult, Cache, Layout, MixParams, ITEM_ROUNDS}; +use igneum_pow::seed::{seed_words_from_bytes, SplitMix64}; +use igneum_pow::verify::{load_index, splitmix32, DatasetSource, Epoch}; +use std::io::Write; +use std::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +/// The shared devnet's genesis hash: epoch seed and era seed of its epoch 0 (pack v4-devnet-epoch0, program.json). +const DEVNET_HEX: &str = "edc4fa844da9dc98d37e965176f6558a31560e40502ab3ae5491b21aaaabfb07"; +/// Devnet 3's epoch-0 seed, epoch and era (pack v4-devnet3-epoch0, program.json). +const DEVNET3_HEX: &str = "4020cb4382e3fe4b281c817c02582e147d8f851f566ae9172b28912b8e68b925"; +const DEVNET_DAY: u64 = 20730; +const DEVNET3_DAY: u64 = 20733; +const DATASET_LOG2: u32 = 28; +const LINES_N: usize = 1 << 22; +const SEGS_N: usize = 1 << 16; +const ITEMS_N: usize = 1 << (DATASET_LOG2 - 4); +const POS: usize = 128; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Plant { + None, + QuarterLines, + HalfLines, + ConstItem, + TLow, + AllQuarter, +} + +impl Plant { + fn parse(s: &str) -> Plant { + match s { + "none" => Plant::None, + "quarter-lines" => Plant::QuarterLines, + "half-lines" => Plant::HalfLines, + "const-item" => Plant::ConstItem, + "t-low" => Plant::TLow, + "all-quarter" => Plant::AllQuarter, + _ => panic!("unknown plant {s}"), + } + } + fn line_mask(self) -> u32 { + match self { + Plant::QuarterLines => (1u32 << 20) - 1, + Plant::HalfLines => (1u32 << 21) - 1, + _ => u32::MAX, + } + } + fn name(self) -> &'static str { + match self { + Plant::None => "none", + Plant::QuarterLines => "quarter-lines", + Plant::HalfLines => "half-lines", + Plant::ConstItem => "const-item", + Plant::TLow => "t-low", + Plant::AllQuarter => "all-quarter", + } + } +} + +fn utc_now() -> String { + let s = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + let (d, t) = (s / 86400, s % 86400); + let z = d as i64 + 719468; + let era = z.div_euclid(146097); + let doe = z.rem_euclid(146097); + let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let dd = doy - (153 * mp + 2) / 5 + 1; + let mm = if mp < 10 { mp + 3 } else { mp - 9 }; + let yy = if mm <= 2 { y + 1 } else { y }; + format!("{yy:04}-{mm:02}-{dd:02}T{:02}:{:02}:{:02}Z", t / 3600, (t / 60) % 60, t % 60) +} + +macro_rules! log { + ($($arg:tt)*) => {{ + println!("[{}] {}", utc_now(), format!($($arg)*)); + std::io::stdout().flush().ok(); + }}; +} + +// -------------------------------------------------------------------------------------------------------------- +// The traced item derivation: `memhard::derive_items_mask` step for step, the line index of every round recorded. +// -------------------------------------------------------------------------------------------------------------- + +fn derive_traced(ts: &[u32], mp: &MixParams, cache: &Cache, out: &mut [[u32; 16]], lines: &mut [[u32; 8]], plant: Plant) { + let n = ts.len(); + let m = mp.shape.mixer_mult as usize; + assert_eq!(mp.shape.derive_len, 0, "class v4 has the fixed mixer"); + let line_mask = cache.line_mask(); + let plant_mask = plant.line_mask(); + for k in 0..n { + let s = &mut out[k]; + let t = ts[k]; + s[..8].copy_from_slice(&mp.key); + for i in 0..8 { + s[8 + i] = t.wrapping_mul(mp.mul[i]).wrapping_add(mp.rc[i]); + } + } + for r in 0..ITEM_ROUNDS { + for j in 0..m { + let rk = round_key_mult(r, j, m); + for s in out[..n].iter_mut() { + mixer(s, rk, mp); + } + } + for k in 0..n { + let s = &mut out[k]; + let mut a = s[0] & line_mask & plant_mask; + if plant == Plant::TLow && r == 0 { + a = ts[k] & line_mask; + } + lines[k][r] = a; + let line = cache.line(a); + for i in 0..16 { + s[i] ^= line[i]; + } + } + } + for j in 0..m { + let rk = round_key_mult(ITEM_ROUNDS, j, m); + for s in out[..n].iter_mut() { + mixer(s, rk, mp); + } + } +} + +// -------------------------------------------------------------------------------------------------------------- +// Histogram statistics +// -------------------------------------------------------------------------------------------------------------- + +struct Stats { + bins: usize, + total: u64, + mean: f64, + sigma: f64, + max: u64, + argmax: usize, + min: u64, + z_max: f64, + z_min: f64, + chi2_per_dof: f64, + chi2_z: f64, + /// (f, top-f share) for f = 0.1%, 0.5%, 1% + top: [(f64, f64); 3], +} + +fn stats_of(counts: impl Iterator + Clone, bins: usize) -> Stats { + let total: u64 = counts.clone().sum(); + let mean = total as f64 / bins as f64; + let sigma = mean.sqrt(); + let mut max = 0u64; + let mut argmax = 0usize; + let mut min = u64::MAX; + let mut chi2 = 0f64; + for (i, c) in counts.clone().enumerate() { + if c > max { + max = c; + argmax = i; + } + if c < min { + min = c; + } + let d = c as f64 - mean; + chi2 += d * d; + } + chi2 /= mean.max(1e-9); + let dof = (bins - 1) as f64; + let cap = 1usize << 16; + let mut coc = vec![0u64; cap]; + let mut big: Vec = Vec::new(); + for c in counts { + if (c as usize) < cap { + coc[c as usize] += 1; + } else { + big.push(c); + } + } + big.sort_unstable_by(|a, b| b.cmp(a)); + let mut top = [(0f64, 0f64); 3]; + for (k, f) in [1.0 / 1000.0, 1.0 / 200.0, 1.0 / 100.0].into_iter().enumerate() { + let want = (f * bins as f64).round() as u64; + let mut left = want; + let mut reads = 0u64; + for &c in &big { + if left == 0 { + break; + } + reads += c; + left -= 1; + } + let mut v = cap - 1; + while left > 0 { + let n = coc[v].min(left); + reads += n * v as u64; + left -= n; + if v == 0 { + break; + } + v -= 1; + } + top[k] = (f, reads as f64 / total.max(1) as f64); + } + Stats { bins, total, mean, sigma, max, argmax, min, z_max: (max as f64 - mean) / sigma, z_min: (min as f64 - mean) / sigma, chi2_per_dof: chi2 / dof, chi2_z: (chi2 - dof) / (2.0 * dof).sqrt(), top } +} + +impl Stats { + fn line(&self, label: &str) -> String { + format!( + "{label}: bins {} reads {} mean {:.3} sigma {:.3} max {} (bin {}) z_max {:+.2} min {} z_min {:+.2} chi2/dof {:.5} chi2_z {:+.2} top0.1% {:.5}% top0.5% {:.5}% top1% {:.5}%", + self.bins, self.total, self.mean, self.sigma, self.max, self.argmax, self.z_max, self.min, self.z_min, self.chi2_per_dof, self.chi2_z, + self.top[0].1 * 100.0, self.top[1].1 * 100.0, self.top[2].1 * 100.0 + ) + } +} + +fn snapshot(counts: &[AtomicU32]) -> Vec { + counts.iter().map(|x| x.load(Ordering::Relaxed) as u64).collect() +} + +fn zero(counts: &[AtomicU32]) { + for c in counts { + c.store(0, Ordering::Relaxed); + } +} + +fn atomic_vec(n: usize) -> Vec { + (0..n).map(|_| AtomicU32::new(0)).collect() +} + +/// A uniform control: `n` SplitMix64 indices into `bins` (a power of two) bins. +fn control(bins: usize, n: u64, seed: u64, threads: usize, hist: &[AtomicU32]) { + let mask = (bins - 1) as u64; + assert!(bins.is_power_of_two()); + std::thread::scope(|sc| { + for th in 0..threads { + let hist = &hist; + sc.spawn(move || { + let mut s = SplitMix64::new(seed ^ (th as u64).wrapping_mul(0x9E3779B97F4A7C15)); + let per = n / threads as u64 + if (th as u64) < n % threads as u64 { 1 } else { 0 }; + for _ in 0..per { + hist[(s.next() & mask) as usize].fetch_add(1, Ordering::Relaxed); + } + }); + } + }); +} + +/// The hot-set test: for the top-f bins by count, the share S_f of all reads against the same share E_f of a +/// uniform control of the same size; excess X_f = S_f - E_f. HOT when X_f >= f (the top f of bins win at least one +/// extra proportional share above chance, which is the least a chip's store of f of them must win to matter). +fn hot_set_test(label: &str, real: &Stats, ctrl: &Stats) -> bool { + let mut flagged = false; + for k in 0..3 { + let (f, s) = real.top[k]; + let e = ctrl.top[k].1; + let x = s - e; + let fire = x >= f; + flagged |= fire; + log!("hot-set {label}: f {:.1}% S_f {:.5}% E_f(control) {:.5}% X_f {:+.5}% X_f/f {:+.4} ratio S/E {:.4}x -> {}", f * 100.0, s * 100.0, e * 100.0, x * 100.0, x / f, s / e.max(1e-12), if fire { "HOT SET" } else { "no hot set" }); + } + log!("hot-set {label}: verdict {}", if flagged { "FLAGGED" } else { "clear" }); + flagged +} + +/// The 6-sigma test: the largest and smallest bucket within 6 sigma of uniform. Gaussian sigma means nothing for a +/// Poisson bucket of mean under 50 (a bucket of 7 at mean 0.5 is ordinary), so such a histogram is reported and +/// not flagged; its control line carries the same statistic for comparison. +fn six_sigma_test(label: &str, s: &Stats) -> bool { + if s.mean < 50.0 { + log!("6-sigma {label}: mean {:.3} under 50, Poisson regime: largest bucket {} ({:+.2} sigma), smallest {}; not a test here (compare the CONTROL line)", s.mean, s.max, s.z_max, s.min); + return false; + } + let fire = s.z_max > 6.0 || s.z_min < -6.0; + log!("6-sigma {label}: largest bucket {} at {:+.2} sigma, smallest {} at {:+.2} sigma -> {}", s.max, s.z_max, s.min, s.z_min, if fire { "FLAGGED (beyond 6 sigma)" } else { "within 6 sigma" }); + fire +} + +fn day_dataset(d: u64) -> DatasetSource { + let ds = Epoch::chain_dataset_day(&day_bytes(d), ProgramClass::V4, 0, DATASET_LOG2); + let mh = ds.memhard().expect("memory-hard"); + assert_eq!(mh.params.shape.mixer_mult, 8); + assert_eq!(mh.params.shape.cache_log2_words, 26); + assert_eq!(mh.cache.line_mask(), (LINES_N - 1) as u32); + ds +} + +// -------------------------------------------------------------------------------------------------------------- +// Q1: the line index over all items of `days` day keys +// -------------------------------------------------------------------------------------------------------------- + +fn census_lines(day0: u64, days: u64, items_log2: u32, threads: usize, plant: Plant, validate: &str, out_dir: &str) { + log!("lines: day0 {day0} days {days} items 2^{items_log2} per day threads {threads} plant {} validate {validate}", plant.name()); + let per_round: Vec> = (0..ITEM_ROUNDS).map(|_| atomic_vec(LINES_N)).collect(); + let pooled: Vec> = (0..ITEM_ROUNDS).map(|_| atomic_vec(LINES_N)).collect(); + // distinct lines and distinct segments per item (9 bins each: 0..8), pooled over days + let distinct_lines = atomic_vec(9); + let distinct_segs = atomic_vec(9); + let items = 1u64 << items_log2; + let batch = 64u64; + let mut any_fire = false; + let mut mism_total = 0u64; + let mut derivations = 0u64; + let t_all = Instant::now(); + for d in day0..day0 + days { + let t0 = Instant::now(); + let ds = day_dataset(d); + let mh = ds.memhard().unwrap(); + log!("day {d}: day bytes {} key {} rot {:?} cache fnv {:016x} filled in {:.2} s", hex(&day_bytes(d)), mh.params.key.iter().map(|w| format!("{w:08x}")).collect::>().join(""), mh.params.rot, mh.cache.fnv1a64(), t0.elapsed().as_secs_f64()); + for h in &per_round { + zero(h); + } + let next = AtomicUsize::new(0); + let mism = AtomicUsize::new(0); + let t1 = Instant::now(); + std::thread::scope(|sc| { + for _ in 0..threads { + let (next, mism, per_round, distinct_lines, distinct_segs, mp, cache) = (&next, &mism, &per_round, &distinct_lines, &distinct_segs, &mh.params, &mh.cache); + sc.spawn(move || { + let mut ts = [0u32; 64]; + let mut out = [[0u32; 16]; 64]; + let mut lines = [[0u32; 8]; 64]; + let mut lib = [[0u32; 16]; 64]; + let mut dl = [0u32; 9]; + let mut dsg = [0u32; 9]; + loop { + let b = next.fetch_add(1, Ordering::Relaxed) as u64; + let start = b * batch; + if start >= items { + break; + } + for k in 0..64 { + ts[k] = (start + k as u64) as u32; + } + derive_traced(&ts, mp, cache, &mut out, &mut lines, plant); + for k in 0..64 { + let mut l = lines[k]; + for r in 0..ITEM_ROUNDS { + per_round[r][l[r] as usize].fetch_add(1, Ordering::Relaxed); + } + l.sort_unstable(); + let mut nd = 1; + for r in 1..8 { + if l[r] != l[r - 1] { + nd += 1; + } + } + dl[nd] += 1; + let mut sg = [0u32; 8]; + for r in 0..8 { + sg[r] = l[r] >> 6; + } + sg.sort_unstable(); + let mut ns = 1; + for r in 1..8 { + if sg[r] != sg[r - 1] { + ns += 1; + } + } + dsg[ns] += 1; + } + let check = plant == Plant::None && (validate == "all" || (validate == "sample" && b % 64 == 0)); + if check { + igneum_pow::memhard::derive_items(&ts, mp, cache, &mut lib); + for k in 0..64 { + if lib[k] != out[k] { + mism.fetch_add(1, Ordering::Relaxed); + } + } + } + } + for i in 0..9 { + distinct_lines[i].fetch_add(dl[i], Ordering::Relaxed); + distinct_segs[i].fetch_add(dsg[i], Ordering::Relaxed); + } + }); + } + }); + let el = t1.elapsed().as_secs_f64(); + derivations += items; + let mm = mism.load(Ordering::Relaxed) as u64; + mism_total += mm; + log!("day {d}: {items} items ({} line reads) derived in {el:.1} s ({:.3} M items/s); library comparison: {} ({} mismatches)", items * 8, items as f64 / el / 1e6, if plant == Plant::None { validate } else { "skipped under the plant" }, mm); + if mm > 0 { + log!("day {d}: THE TRACED MIRROR DISAGREES WITH THE LIBRARY: {mm} items; nothing below is trusted"); + } + let mut total = vec![0u64; LINES_N]; + for r in 0..ITEM_ROUNDS { + let snap = snapshot(&per_round[r]); + let s = stats_of(snap.iter().copied(), LINES_N); + log!("{}", s.line(&format!("day {d} round {r} lines"))); + let seg: Vec = snap.chunks(64).map(|c| c.iter().sum()).collect(); + let ss = stats_of(seg.iter().copied(), SEGS_N); + log!("{}", ss.line(&format!("day {d} round {r} segments"))); + let mut depth = vec![0u64; 64]; + for (i, c) in snap.iter().enumerate() { + depth[i & 63] += c; + total[i] += c; + pooled[r][i].fetch_add(*c as u32, Ordering::Relaxed); + } + let sd = stats_of(depth.iter().copied(), 64); + log!("{}", sd.line(&format!("day {d} round {r} depth(j)"))); + any_fire |= six_sigma_test(&format!("day {d} round {r} segments"), &ss); + any_fire |= six_sigma_test(&format!("day {d} round {r} depth"), &sd); + } + let s_full = stats_of(total.iter().copied(), LINES_N); + log!("{}", s_full.line(&format!("day {d} all rounds lines"))); + let seg: Vec = total.chunks(64).map(|c| c.iter().sum()).collect(); + let s_seg = stats_of(seg.iter().copied(), SEGS_N); + log!("{}", s_seg.line(&format!("day {d} all rounds segments"))); + any_fire |= six_sigma_test(&format!("day {d} all rounds segments"), &s_seg); + any_fire |= six_sigma_test(&format!("day {d} all rounds lines"), &s_full); + let ctrl = atomic_vec(LINES_N); + control(LINES_N, items * 8, 0xADC2_0000 + d, threads, &ctrl); + let cs = snapshot(&ctrl); + let c_full = stats_of(cs.iter().copied(), LINES_N); + log!("{}", c_full.line(&format!("day {d} CONTROL lines"))); + let cseg: Vec = cs.chunks(64).map(|c| c.iter().sum()).collect(); + log!("{}", stats_of(cseg.iter().copied(), SEGS_N).line(&format!("day {d} CONTROL segments"))); + any_fire |= hot_set_test(&format!("day {d} lines"), &s_full, &c_full); + } + // pooled over the days + let mut total = vec![0u64; LINES_N]; + for r in 0..ITEM_ROUNDS { + let snap = snapshot(&pooled[r]); + log!("{}", stats_of(snap.iter().copied(), LINES_N).line(&format!("POOLED {days} days round {r} lines"))); + let seg: Vec = snap.chunks(64).map(|c| c.iter().sum()).collect(); + let ss = stats_of(seg.iter().copied(), SEGS_N); + log!("{}", ss.line(&format!("POOLED {days} days round {r} segments"))); + any_fire |= six_sigma_test(&format!("POOLED round {r} segments"), &ss); + let mut depth = vec![0u64; 64]; + for (i, c) in snap.iter().enumerate() { + depth[i & 63] += c; + total[i] += c; + } + let sd = stats_of(depth.iter().copied(), 64); + log!("{}", sd.line(&format!("POOLED {days} days round {r} depth(j)"))); + any_fire |= six_sigma_test(&format!("POOLED round {r} depth"), &sd); + } + let s_full = stats_of(total.iter().copied(), LINES_N); + log!("{}", s_full.line(&format!("POOLED {days} days all rounds lines"))); + let seg: Vec = total.chunks(64).map(|c| c.iter().sum()).collect(); + let s_seg = stats_of(seg.iter().copied(), SEGS_N); + log!("{}", s_seg.line(&format!("POOLED {days} days all rounds segments"))); + any_fire |= six_sigma_test(&format!("POOLED {days} days segments"), &s_seg); + any_fire |= six_sigma_test(&format!("POOLED {days} days full 2^22 lines"), &s_full); + let mut depth = vec![0u64; 64]; + for (i, c) in total.iter().enumerate() { + depth[i & 63] += c; + } + let sd = stats_of(depth.iter().copied(), 64); + log!("{}", sd.line(&format!("POOLED {days} days all rounds depth(j)"))); + any_fire |= six_sigma_test(&format!("POOLED {days} days depth"), &sd); + log!("POOLED depth histogram (j: count): {}", depth.iter().enumerate().map(|(j, c)| format!("{j}:{c}")).collect::>().join(" ")); + let ctrl = atomic_vec(LINES_N); + control(LINES_N, derivations * 8, 0xADC2_1111, threads, &ctrl); + let cs = snapshot(&ctrl); + let c_full = stats_of(cs.iter().copied(), LINES_N); + log!("{}", c_full.line("POOLED CONTROL lines")); + let cseg: Vec = cs.chunks(64).map(|c| c.iter().sum()).collect(); + let c_seg = stats_of(cseg.iter().copied(), SEGS_N); + log!("{}", c_seg.line("POOLED CONTROL segments")); + any_fire |= hot_set_test(&format!("POOLED {days} days lines"), &s_full, &c_full); + any_fire |= hot_set_test(&format!("POOLED {days} days segments"), &s_seg, &c_seg); + // the stride, prefix and hottest-lines stores at the measured distribution: hit rate of a store of fraction f + // (what Q4 prices). Hottest-lines: the top f of lines by pooled count; stride k: lines with j % k == 0; prefix + // L: lines with j < L. Hit rate = share of reads on held lines. + { + let mut sorted: Vec = total.clone(); + sorted.sort_unstable_by(|a, b| b.cmp(a)); + let tot = s_full.total as f64; + let mut acc = vec![0u64; sorted.len() + 1]; + for (i, c) in sorted.iter().enumerate() { + acc[i + 1] = acc[i] + c; + } + for &k in &[2usize, 4, 8, 16, 32, 64] { + let f = 1.0 / k as f64; + let held = (LINES_N as f64 * f).round() as usize; + let hottest = acc[held] as f64 / tot; + let mut stride = 0u64; + let mut prefix = 0u64; + let l = 64 / k; + for (i, c) in total.iter().enumerate() { + let j = i & 63; + if j % k == 0 { + stride += c; + } + if j < l { + prefix += c; + } + } + log!("STORE f=1/{k}: hit rate hottest-lines {:.6} stride {:.6} prefix {:.6} (uniform {:.6})", hottest, stride as f64 / tot, prefix as f64 / tot, f); + } + } + let dl = snapshot(&distinct_lines); + let dsg = snapshot(&distinct_segs); + let n_items = derivations as f64; + // uniform expectation of 8 distinct among 8 draws from N bins: prod (1 - i/N) + let exp_distinct = |n: f64| -> f64 { (0..8).map(|i| 1.0 - i as f64 / n).product::() }; + log!("DISTINCT lines per item: {} ; P(8 distinct) {:.8} uniform {:.8}; segments per item: {} ; P(8 distinct) {:.6} uniform {:.6}", + dl.iter().enumerate().map(|(i, c)| format!("{i}:{c}")).collect::>().join(" "), dl[8] as f64 / n_items, exp_distinct(LINES_N as f64), + dsg.iter().enumerate().map(|(i, c)| format!("{i}:{c}")).collect::>().join(" "), dsg[8] as f64 / n_items, exp_distinct(SEGS_N as f64)); + let tag = format!("lines-d{day0}-n{days}-i{items_log2}-{}", plant.name()); + { + let mut f = std::fs::File::create(format!("{out_dir}/{tag}-segments.txt")).unwrap(); + writeln!(f, "# segment count ; pooled over {days} days from {day0}, items 2^{items_log2} per day, plant {}", plant.name()).unwrap(); + for (i, c) in seg.iter().enumerate() { + writeln!(f, "{i} {c}").unwrap(); + } + let mut g = std::fs::File::create(format!("{out_dir}/{tag}-full.u32le")).unwrap(); + let mut bytes = Vec::with_capacity(LINES_N * 4); + for c in &total { + bytes.extend_from_slice(&(*c as u32).to_le_bytes()); + } + g.write_all(&bytes).unwrap(); + } + log!("lines DONE: {derivations} item derivations ({} line reads) over {days} days in {:.1} s; mirror mismatches {mism_total}; verdict {}", derivations * 8, t_all.elapsed().as_secs_f64(), if any_fire { "FLAGGED" } else { "PASS (no test fired)" }); +} + +// -------------------------------------------------------------------------------------------------------------- +// Q3(2): the weak-day scan, one cache per day in parallel over days +// -------------------------------------------------------------------------------------------------------------- + +fn census_days(day0: u64, days: u64, items_log2: u32, threads: usize, plant: Plant, plant_day: u64, out_dir: &str) { + log!("days: day0 {day0} days {days} items 2^{items_log2} per day threads {threads} plant {} on day {plant_day}", plant.name()); + let items = 1usize << items_log2; + let next = AtomicU64::new(day0); + let t0 = Instant::now(); + let rows: Vec> = std::thread::scope(|sc| { + let mut hs = Vec::new(); + for _ in 0..threads { + let next = &next; + hs.push(sc.spawn(move || { + let mut rows = Vec::new(); + loop { + let d = next.fetch_add(1, Ordering::Relaxed); + if d >= day0 + days { + break; + } + let p = if d == plant_day { plant } else { Plant::None }; + let ds = day_dataset(d); + let mh = ds.memhard().unwrap(); + let mut seg = vec![0u64; SEGS_N]; + let mut depth = vec![0u64; 64]; + let mut ts = [0u32; 64]; + let mut out = [[0u32; 16]; 64]; + let mut lines = [[0u32; 8]; 64]; + let mut lib = [[0u32; 16]; 64]; + let mut mism = 0u64; + for b in 0..items / 64 { + for k in 0..64 { + ts[k] = (b * 64 + k) as u32; + } + derive_traced(&ts, &mh.params, &mh.cache, &mut out, &mut lines, p); + for k in 0..64 { + for r in 0..8 { + seg[(lines[k][r] >> 6) as usize] += 1; + depth[(lines[k][r] & 63) as usize] += 1; + } + } + if p == Plant::None && b % 256 == 0 { + igneum_pow::memhard::derive_items(&ts, &mh.params, &mh.cache, &mut lib); + for k in 0..64 { + if lib[k] != out[k] { + mism += 1; + } + } + } + } + let ss = stats_of(seg.iter().copied(), SEGS_N); + let sd = stats_of(depth.iter().copied(), 64); + // a control of the same size for this day + let mut rng = SplitMix64::new(0xADC2_2222 ^ d); + let mut cseg = vec![0u64; SEGS_N]; + for _ in 0..items * 8 { + cseg[(rng.next() & (SEGS_N as u64 - 1)) as usize] += 1; + } + let cs = stats_of(cseg.iter().copied(), SEGS_N); + rows.push((d, ss.chi2_z, ss.z_max, ss.max, sd.chi2_z, sd.z_max, sd.max, cs.chi2_z, cs.z_max, cs.max, mism)); + } + rows + })); + } + hs.into_iter().map(|h| h.join().unwrap()).collect() + }); + let mut all: Vec<_> = rows.into_iter().flatten().collect(); + all.sort_unstable_by_key(|r| r.0); + let mut f = std::fs::File::create(format!("{out_dir}/days-d{day0}-n{days}-i{items_log2}-{}.txt", plant.name())).unwrap(); + writeln!(f, "# day seg_chi2_z seg_z_max seg_max depth_chi2_z depth_z_max depth_max ctrl_seg_chi2_z ctrl_seg_z_max ctrl_seg_max mirror_mismatches").unwrap(); + let mut flagged = Vec::new(); + let mut worst_real = (0u64, f64::MIN, 0u64); + let mut worst_ctrl = (0u64, f64::MIN, 0u64); + let mut worst_chi = (0u64, f64::MIN); + let mut worst_depth = (0u64, f64::MIN); + let mut mism_total = 0u64; + let mut real_max_hist = std::collections::BTreeMap::new(); + let mut ctrl_max_hist = std::collections::BTreeMap::new(); + for r in &all { + writeln!(f, "{} {:.3} {:.3} {} {:.3} {:.3} {} {:.3} {:.3} {} {}", r.0, r.1, r.2, r.3, r.4, r.5, r.6, r.7, r.8, r.9, r.10).unwrap(); + mism_total += r.10; + *real_max_hist.entry(r.3).or_insert(0u64) += 1; + *ctrl_max_hist.entry(r.9).or_insert(0u64) += 1; + if r.2 > worst_real.1 { + worst_real = (r.0, r.2, r.3); + } + if r.8 > worst_ctrl.1 { + worst_ctrl = (r.0, r.8, r.9); + } + if r.1 > worst_chi.1 { + worst_chi = (r.0, r.1); + } + if r.5 > worst_depth.1 { + worst_depth = (r.0, r.5); + } + if r.1 > 6.0 || r.4 > 6.0 || r.5 > 6.0 { + flagged.push(format!("day {} seg_chi2_z {:+.2} depth_chi2_z {:+.2} depth_z_max {:+.2}", r.0, r.1, r.4, r.5)); + } + } + log!("days: {} days scanned in {:.1} s, {} item derivations; mirror mismatches {mism_total}", all.len(), t0.elapsed().as_secs_f64(), all.len() * items); + log!("days: worst segment chi2_z {:+.3} on day {}; worst segment max bucket {} ({:+.2} sigma) on day {}; CONTROL worst max bucket {} ({:+.2} sigma) on day {}; worst depth z_max {:+.3} on day {}", worst_chi.1, worst_chi.0, worst_real.2, worst_real.1, worst_real.0, worst_ctrl.2, worst_ctrl.1, worst_ctrl.0, worst_depth.1, worst_depth.0); + log!("days: per-day max segment count histogram REAL {} ; CONTROL {}", real_max_hist.iter().map(|(k, v)| format!("{k}:{v}")).collect::>().join(" "), ctrl_max_hist.iter().map(|(k, v)| format!("{k}:{v}")).collect::>().join(" ")); + log!("days: flagged (segment chi2_z > 6 or depth beyond 6 sigma): {} days{}{}", flagged.len(), if flagged.is_empty() { "" } else { ": " }, flagged.join("; ")); + log!("days DONE: verdict {}", if flagged.is_empty() { "PASS (no day flagged)" } else { "FLAGGED" }); +} + +// -------------------------------------------------------------------------------------------------------------- +// Q2: the warp interpreter mirrored, every load's item index recorded +// -------------------------------------------------------------------------------------------------------------- + +struct Table { + items: Vec<[u32; 16]>, + lines: Vec<[u32; 8]>, +} + +fn build_table(mp: &MixParams, cache: &Cache, threads: usize, validate: &str) -> (Table, u64) { + let n = ITEMS_N; + let t0 = Instant::now(); + let mut items = vec![[0u32; 16]; n]; + let mut lines = vec![[0u32; 8]; n]; + let mism = AtomicUsize::new(0); + std::thread::scope(|sc| { + let per = n / threads; + for (th, (ic, lc)) in items.chunks_mut(per).zip(lines.chunks_mut(per)).enumerate() { + let mism = &mism; + sc.spawn(move || { + let mut ts = [0u32; 64]; + let mut lib = [[0u32; 16]; 64]; + let base = th * per; + for (bi, (ib, lb)) in ic.chunks_mut(64).zip(lc.chunks_mut(64)).enumerate() { + let start = base + bi * 64; + for k in 0..ib.len() { + ts[k] = (start + k) as u32; + } + derive_traced(&ts[..ib.len()], mp, cache, ib, lb, Plant::None); + if validate == "all" || (validate == "sample" && bi % 64 == 0) { + igneum_pow::memhard::derive_items(&ts[..ib.len()], mp, cache, &mut lib); + for k in 0..ib.len() { + if lib[k] != ib[k] { + mism.fetch_add(1, Ordering::Relaxed); + } + } + } + } + }); + } + }); + let mm = mism.load(Ordering::Relaxed) as u64; + log!("table: {n} items derived with their 8 lines in {:.1} s; library comparison {validate} ({mm} mismatches)", t0.elapsed().as_secs_f64()); + (Table { items, lines }, mm) +} + +struct ProgramSpec { + label: String, + epoch_seed: Vec, + era: Vec, +} + +fn words_bytes(s: &str) -> Vec { + seed_words_from_bytes(s.as_bytes()).iter().flat_map(|x| x.to_le_bytes()).collect() +} + +/// `devnet`, `devnet3`, `era-fixed:k` (the devnet era, a drawn epoch seed), `era-drawn:k` (both drawn). +fn program_spec(name: &str) -> ProgramSpec { + let genesis = unhex(DEVNET_HEX).unwrap(); + if name == "devnet" { + return ProgramSpec { label: "devnet-epoch0".into(), epoch_seed: genesis.clone(), era: genesis }; + } + if name == "devnet3" { + let s = unhex(DEVNET3_HEX).unwrap(); + return ProgramSpec { label: "devnet3-epoch0".into(), epoch_seed: s.clone(), era: s }; + } + if let Some(k) = name.strip_prefix("era-fixed:") { + return ProgramSpec { label: format!("era-fixed-{k}"), epoch_seed: words_bytes(&format!("igneum-adv-cache-2/epoch/{k}")), era: genesis }; + } + if let Some(k) = name.strip_prefix("era-drawn:") { + return ProgramSpec { label: format!("era-drawn-{k}"), epoch_seed: words_bytes(&format!("igneum-adv-cache-2/epoch/{k}")), era: words_bytes(&format!("igneum-adv-cache-2/era/{k}")) }; + } + panic!("unknown program spec {name}"); +} + +fn expand_programs(spec: &str) -> Vec { + let mut v = Vec::new(); + for part in spec.split(',') { + if let Some((pre, range)) = part.rsplit_once(':') { + if let Some((a, b)) = range.split_once("..") { + for k in a.parse::().unwrap()..=b.parse::().unwrap() { + v.push(format!("{pre}:{k}")); + } + continue; + } + } + v.push(part.to_string()); + } + v +} + +struct Mirror<'a> { + program: &'a Program, + era: Option<&'a EraParams>, + layout: Layout, + mask: u32, + log2: u32, + table: &'a Table, + plant_site: Option, +} + +struct Sink { + items: Vec<[u32; LANES]>, +} + +impl<'a> Mirror<'a> { + #[inline(always)] + fn step(&self, ins: &Instr, r: &mut [[u32; LANES]; 8], sel: &[u32; LANES], sink: &mut Sink, site: &mut usize) { + let d = ins.dst as usize; + let a = ins.src as usize; + match ins.op { + Op::Add => { + let (imm, imm2, bit) = (ins.imm, ins.imm2, ins.bit as u32); + let src = r[a]; + for lane in 0..LANES { + let c = if (sel[lane] >> bit) & 1 != 0 { imm2 } else { imm }; + r[d][lane] = r[d][lane].wrapping_add(src[lane]).wrapping_add(c); + } + } + Op::Sub => { + let src = r[a]; + for lane in 0..LANES { + r[d][lane] = r[d][lane].wrapping_sub(src[lane]); + } + } + Op::Mul => { + let src = r[a]; + for lane in 0..LANES { + r[d][lane] = r[d][lane].wrapping_mul(src[lane]); + } + } + Op::MulHi => { + let src = r[a]; + for lane in 0..LANES { + r[d][lane] = ((r[d][lane] as u64 * src[lane] as u64) >> 32) as u32; + } + } + Op::Xor => { + let src = r[a]; + for lane in 0..LANES { + r[d][lane] ^= src[lane]; + } + } + Op::Or => { + let src = r[a]; + for lane in 0..LANES { + r[d][lane] |= src[lane]; + } + } + Op::Rotl => { + let n = ins.rot; + for lane in 0..LANES { + r[d][lane] = r[d][lane].rotate_left(n); + } + } + Op::Rotr => { + let src = r[a]; + for lane in 0..LANES { + r[d][lane] = r[d][lane].rotate_right(src[lane] & 31); + } + } + Op::Mad => { + let src = r[a]; + let src2 = r[ins.src2 as usize]; + for lane in 0..LANES { + r[d][lane] = src[lane].wrapping_mul(src2[lane]).wrapping_add(r[d][lane]); + } + } + Op::Shfl => { + let src = r[a]; + let m = ins.mask as usize; + for lane in 0..LANES { + r[d][lane] ^= src[lane ^ m]; + } + } + Op::Load => { + assert_eq!(ins.width, 1, "class v4 loads one word"); + let mut ts = [0u32; LANES]; + for lane in 0..LANES { + let idx = load_index(self.era, ins, r[a][lane], self.mask, self.log2); + let (t, j) = self.layout.split(idx); + let t = if self.plant_site == Some(*site) { 0x00_1234 } else { t }; + ts[lane] = t; + r[d][lane] ^= self.table.items[t as usize][j as usize]; + } + sink.items.push(ts); + *site += 1; + } + Op::WLoad | Op::Scratch | Op::Hot => panic!("op {:?} is not a class v4 op", ins.op), + } + } + + fn warp(&self, base_nonce: u32, sink: &mut Sink) -> [u64; LANES] { + let seed = &self.program.seed; + let mut r = [[0u32; LANES]; 8]; + for lane in 0..LANES { + let nonce = base_nonce.wrapping_add(lane as u32); + for i in 0..8 { + let mut x = nonce ^ seed[i]; + x = x.wrapping_add(0x9e3779b9u32.wrapping_mul(i as u32 + 1)); + x = splitmix32(x); + r[i][lane] = x ^ seed[(i + 1) & 7]; + } + } + sink.items.clear(); + for _ in 0..ITERATIONS { + let sel = r[0]; + let mut site = 0usize; + for ins in &self.program.instrs { + self.step(ins, &mut r, &sel, sink, &mut site); + } + for _ in 0..self.program.shadow_reps() { + for ins in &self.program.shadow { + self.step(ins, &mut r, &sel, sink, &mut site); + } + } + } + let mut hashes = [0u64; LANES]; + for lane in 0..LANES { + let lo = r[0][lane] ^ r[1][lane].rotate_left(7) ^ r[2][lane].rotate_left(14) ^ r[3][lane].rotate_left(21); + let hi = r[4][lane] ^ r[5][lane].rotate_left(9) ^ r[6][lane].rotate_left(18) ^ r[7][lane].rotate_left(27); + hashes[lane] = ((hi as u64) << 32) | lo as u64; + } + hashes + } +} + +/// Every load site's window as an item range `(first item, items)`. +fn site_item_windows(program: &Program, mask: u32, log2: u32) -> Vec<(u32, u32)> { + program + .instrs + .iter() + .filter(|i| i.op == Op::Load) + .map(|i| { + let (wm, off) = igneum_pow::verify::window(i, mask, log2); + (off >> 4, (wm >> 4) + 1) + }) + .collect() +} + +/// Expected share of reads per quarter of the item space under the window layer alone (every site uniform on its +/// own window, one sixteenth of the reads each). +fn quarter_shares(windows: &[(u32, u32)]) -> [f64; 4] { + let q = ITEMS_N as u32 / 4; + let mut d = [0f64; 4]; + for &(first, n) in windows { + for (k, dq) in d.iter_mut().enumerate() { + let qs = k as u32 * q; + if qs >= first && qs < first + n { + *dq += (q as f64 / n as f64) / windows.len() as f64; + } + } + } + d +} + +/// The windowed control: `n` reads, site `i mod 16`, uniform on the site's window. +fn windowed_control(windows: &[(u32, u32)], n: u64, seed: u64, threads: usize, hist: &[AtomicU32]) { + std::thread::scope(|sc| { + for th in 0..threads { + let hist = &hist; + sc.spawn(move || { + let mut s = SplitMix64::new(seed ^ (th as u64).wrapping_mul(0x9E3779B97F4A7C15)); + let per = n / threads as u64 + if (th as u64) < n % threads as u64 { 1 } else { 0 }; + for i in 0..per { + let (first, cnt) = windows[(i % 16) as usize]; + hist[(first + (s.next() as u32 & (cnt - 1))) as usize].fetch_add(1, Ordering::Relaxed); + } + }); + } + }); +} + +/// Lines weighted by item reads: line_hist[l] = sum over items t holding l of item_hist[t]; one pass, no atomics. +fn lines_from_items(item_hist: &[u64], table: &Table) -> Vec { + let mut lh = vec![0u64; LINES_N]; + for (t, &c) in item_hist.iter().enumerate() { + if c != 0 { + for &l in &table.lines[t] { + lh[l as usize] += c; + } + } + } + lh +} + +struct Summary { + label: String, + id: u64, + attempt: u32, + nonces: u64, + s01: f64, + e01_w: f64, + e01_flat: f64, + s10: f64, + e10_w: f64, + e10_flat: f64, + hot: bool, + z_w: f64, + top_quarter: f64, + top_half: f64, + exp_quarter: f64, + exp_half: f64, + line_s01: f64, + line_e01: f64, + line_s10: f64, + line_e10: f64, + worst_site_ratio: f64, + worst_site: usize, + hash_mism: u64, +} + +#[allow(clippy::too_many_arguments)] +fn run_program(spec: &ProgramSpec, ds: DatasetSource, table: &Table, day: u64, nonces: u64, threads: usize, plant: Plant, check_every: u64, fingerprint: bool, expected_warp0: Option<&[u64]>, out_dir: &str) -> (Summary, DatasetSource) { + let program = Epoch::chain_program(&spec.epoch_seed, Some(&spec.era), ProgramClass::V4, &spec.label); + assert_eq!(program.generator, 4); + assert_eq!(program.class.mixer_mult, 8); + assert_eq!(program.class.shadow.map(|s| (s.instrs, s.reps)), Some((256, 27))); + let era = program.class.era.expect("class v4 draws the era"); + let sites: Vec = program.instrs.iter().enumerate().filter(|(_, i)| i.op == Op::Load).map(|(k, _)| k).collect(); + assert_eq!(sites.len(), 16); + log!("program {}: epoch seed {} era seed {} id {:016x} attempt {} class {} op mix {}; era stride mul {:#010x} rot {} interleave {:?}; sites instr:win:off {}", spec.label, hex(&spec.epoch_seed), hex(&spec.era), program.program_id(), program.attempt, program.class.name(), program.op_mix(), era.stride_mul, era.stride_rot, era.pos, program.instrs.iter().enumerate().filter(|(_, i)| i.op == Op::Load).map(|(k, i)| format!("{k}:{}:{}", i.win, i.off)).collect::>().join(" ")); + let epoch = Epoch { program, dataset: ds }; + let mirror = Mirror { program: &epoch.program, era: epoch.program.class.era.as_ref(), layout: epoch.program.class.layout(), mask: epoch.dataset.mask, log2: epoch.dataset.log2_words, table, plant_site: if plant == Plant::ConstItem { Some(0) } else { None } }; + let windows = site_item_windows(&epoch.program, epoch.dataset.mask, epoch.dataset.log2_words); + let qs = quarter_shares(&windows); + let mut qsorted = qs; + qsorted.sort_by(|a, b| b.partial_cmp(a).unwrap()); + let exp_half = (qs[0] + qs[1]).max(qs[2] + qs[3]); + log!("window layer {}: site windows (first, items) {}; expected share per quarter {:.4} {:.4} {:.4} {:.4}; top quarter {:.4} (uniform 0.25), top aligned half {:.4} (uniform 0.5)", spec.label, windows.iter().map(|(f, n)| format!("({:#x},2^{})", f, n.trailing_zeros())).collect::>().join(" "), qs[0], qs[1], qs[2], qs[3], qsorted[0], exp_half); + let item_hist = atomic_vec(ITEMS_N); + let site_hist: Vec> = (0..16).map(|_| atomic_vec(ITEMS_N)).collect(); + let warps = nonces / LANES as u64; + let next = AtomicUsize::new(0); + let checked = AtomicUsize::new(0); + let hash_mism = AtomicUsize::new(0); + let warp0_ok = AtomicUsize::new(usize::MAX); + let all_hashes: Vec = if fingerprint { (0..nonces as usize).map(|_| AtomicU64::new(0)).collect() } else { Vec::new() }; + let t1 = Instant::now(); + std::thread::scope(|sc| { + for _ in 0..threads { + let (next, checked, hash_mism, item_hist, site_hist, mirror, epoch, warp0_ok, all_hashes) = (&next, &checked, &hash_mism, &item_hist, &site_hist, &mirror, &epoch, &warp0_ok, &all_hashes); + sc.spawn(move || { + let mut sink = Sink { items: Vec::with_capacity(POS) }; + loop { + let w = next.fetch_add(1, Ordering::Relaxed) as u64; + if w >= warps { + break; + } + let base = (w * LANES as u64) as u32; + let hashes = mirror.warp(base, &mut sink); + assert_eq!(sink.items.len(), POS); + if plant == Plant::None && (w < 64 || w % check_every == 0) { + let lib = epoch.hash_warp(base); + checked.fetch_add(1, Ordering::Relaxed); + if lib != hashes { + hash_mism.fetch_add(1, Ordering::Relaxed); + } + } + if w == 0 { + if let Some(exp) = expected_warp0 { + let ok = exp.iter().enumerate().all(|(l, &h)| hashes[l] == h); + warp0_ok.store(ok as usize, Ordering::Relaxed); + } + } + if fingerprint { + for lane in 0..LANES { + all_hashes[base as usize + lane].store(hashes[lane], Ordering::Relaxed); + } + } + for (p, load) in sink.items.iter().enumerate() { + let sh = &site_hist[p % 16]; + for lane in 0..LANES { + let t = load[lane] as usize; + item_hist[t].fetch_add(1, Ordering::Relaxed); + sh[t].fetch_add(1, Ordering::Relaxed); + } + } + } + }); + } + }); + let el = t1.elapsed().as_secs_f64(); + let ck = checked.load(Ordering::Relaxed); + let hm = hash_mism.load(Ordering::Relaxed) as u64; + log!("{}: {} warps ({} nonces) interpreted in {el:.1} s ({:.3} ms per warp per thread); Epoch::hash_warp agreement on {ck} warps: {hm} mismatches{}", spec.label, warps, nonces, el * 1e3 * threads as f64 / warps as f64, if plant != Plant::None { " (library comparison skipped under the plant)" } else { "" }); + if hm > 0 { + log!("THE MIRROR DISAGREES WITH THE LIBRARY ({hm} warps); nothing below is trusted"); + } + match warp0_ok.load(Ordering::Relaxed) { + usize::MAX => {} + 1 => log!("{}: warp 0 lanes agree with the pack's vectors.json", spec.label), + _ => log!("{}: WARP 0 DOES NOT MATCH THE PACK'S VECTORS", spec.label), + } + if fingerprint { + let mut h: u64 = 0xcbf29ce484222325; + for x in &all_hashes { + for b in x.load(Ordering::Relaxed).to_le_bytes() { + h ^= b as u64; + h = h.wrapping_mul(0x100000001b3); + } + } + log!("{}: FNV-1a 64 fingerprint over the {nonces} outputs from base 0 (little-endian u64): {h:016x}", spec.label); + } + let total_reads = nonces * 128; + let snap = snapshot(&item_hist); + let s_items = stats_of(snap.iter().copied(), ITEMS_N); + log!("{}", s_items.line(&format!("{} item histogram (flat)", spec.label))); + // measured quarter and half shares + let q = ITEMS_N / 4; + let mut mq = [0f64; 4]; + for k in 0..4 { + mq[k] = snap[k * q..(k + 1) * q].iter().sum::() as f64 / total_reads as f64; + } + let mut mqs = mq; + mqs.sort_by(|a, b| b.partial_cmp(a).unwrap()); + let m_half = (mq[0] + mq[1]).max(mq[2] + mq[3]); + log!("{}: measured share per quarter {:.5} {:.5} {:.5} {:.5}; top quarter {:.5} (window model {:.5}, uniform 0.25); top aligned half {:.5} (model {:.5}, uniform 0.5)", spec.label, mq[0], mq[1], mq[2], mq[3], mqs[0], qsorted[0], m_half, exp_half); + let ctrl = atomic_vec(ITEMS_N); + windowed_control(&windows, total_reads, 0xADC2_3333 ^ igneum_pow::seed::fnv1a64(spec.label.as_bytes()), threads, &ctrl); + let cs = snapshot(&ctrl); + let c_items = stats_of(cs.iter().copied(), ITEMS_N); + log!("{}", c_items.line(&format!("{} WINDOWED CONTROL item histogram", spec.label))); + let flat = atomic_vec(ITEMS_N); + control(ITEMS_N, total_reads, 0xADC2_4444 ^ igneum_pow::seed::fnv1a64(spec.label.as_bytes()), threads, &flat); + let fs = snapshot(&flat); + let f_items = stats_of(fs.iter().copied(), ITEMS_N); + log!("{}", f_items.line(&format!("{} FLAT CONTROL item histogram", spec.label))); + let hot = hot_set_test(&format!("{} items (windowed control)", spec.label), &s_items, &c_items); + let _ = hot_set_test(&format!("{} items (flat control)", spec.label), &s_items, &f_items); + // the windowed 6-sigma: 64-item buckets against the window density + let mut z_w = f64::MIN; + let mut z_w_min = f64::MAX; + let mut e_min = f64::MAX; + for (b, c) in snap.chunks(64).enumerate() { + let e = qs[(b * 64) / q] * total_reads as f64 * 64.0 / ITEMS_N as f64 * 4.0; + if e <= 0.0 { + continue; + } + let s: u64 = c.iter().sum(); + let z = (s as f64 - e) / e.sqrt(); + e_min = e_min.min(e); + z_w = z_w.max(z); + z_w_min = z_w_min.min(z); + } + if e_min < 50.0 { + log!("6-sigma {} items buckets64 against the window density: smallest expected bucket {:.2} under 50, Poisson regime: largest {:+.2} sigma smallest {:+.2}; not a test at this nonce count", spec.label, e_min, z_w, z_w_min); + z_w = 0.0; + } else { + log!("6-sigma {} items buckets64 against the window density: largest {:+.2} sigma smallest {:+.2} -> {}", spec.label, z_w, z_w_min, if z_w > 6.0 || z_w_min < -6.0 { "FLAGGED" } else { "within 6 sigma" }); + } + // per site: top-f shares against the site's own window (uniform on the window) + let mut worst_site_ratio = 0f64; + let mut worst_site = 0usize; + for s in 0..16 { + let sh = snapshot(&site_hist[s]); + let (first, n) = windows[s]; + let st = stats_of(sh[first as usize..(first + n) as usize].iter().copied(), n as usize); + let outside: u64 = sh.iter().sum::() - st.total; + let sctrl = atomic_vec(n as usize); + control(n as usize, st.total, 0xADC2_5555 ^ (s as u64) ^ igneum_pow::seed::fnv1a64(spec.label.as_bytes()), threads, &sctrl); + let sc = snapshot(&sctrl); + let cst = stats_of(sc.iter().copied(), n as usize); + let ratio = st.top[0].1 / cst.top[0].1.max(1e-12); + if ratio > worst_site_ratio { + worst_site_ratio = ratio; + worst_site = s; + } + log!("site {s} (instr {}, win {}, off {}, window 2^{} items, reads outside the window {outside}): {}; control top0.1% {:.5}% top1% {:.5}%; ratio top0.1% {:.4}x top1% {:.4}x; z_max {:+.2}", sites[s], epoch.program.instrs[sites[s]].win, epoch.program.instrs[sites[s]].off, n.trailing_zeros(), st.line("hist"), cst.top[0].1 * 100.0, cst.top[2].1 * 100.0, ratio, st.top[2].1 / cst.top[2].1.max(1e-12), st.z_max); + } + // lines weighted by item reads, overall and under the windowed control + let lh = lines_from_items(&snap, table); + let s_lines = stats_of(lh.iter().copied(), LINES_N); + log!("{}", s_lines.line(&format!("{} LINE histogram (8 lines per item read)", spec.label))); + let ch = lines_from_items(&cs, table); + let c_lines = stats_of(ch.iter().copied(), LINES_N); + log!("{}", c_lines.line(&format!("{} LINE histogram under the WINDOWED CONTROL items", spec.label))); + let line_hot = hot_set_test(&format!("{} lines (windowed control)", spec.label), &s_lines, &c_lines); + let seg: Vec = lh.chunks(64).map(|c| c.iter().sum()).collect(); + let s_seg = stats_of(seg.iter().copied(), SEGS_N); + log!("{}", s_seg.line(&format!("{} SEGMENT histogram (lines / 64)", spec.label))); + six_sigma_test(&format!("{} segments", spec.label), &s_seg); + // a hottest-lines store at the measured line distribution: hit rates for f = 1/2 .. 1/64, plus stride + { + let mut sorted = lh.clone(); + sorted.sort_unstable_by(|a, b| b.cmp(a)); + let tot = s_lines.total as f64; + let mut acc = 0u64; + let mut marks: Vec<(usize, f64)> = Vec::new(); + for k in [2usize, 4, 8, 16, 32, 64] { + marks.push((LINES_N / k, 0.0)); + } + let mut mi = 0; + for (i, c) in sorted.iter().enumerate() { + acc += c; + while mi < marks.len() && i + 1 == marks[mi].0 { + marks[mi].1 = acc as f64 / tot; + mi += 1; + } + } + let mut stride = [0u64; 6]; + for (i, c) in lh.iter().enumerate() { + for (ki, k) in [2usize, 4, 8, 16, 32, 64].into_iter().enumerate() { + if (i & 63) % k == 0 { + stride[ki] += c; + } + } + } + log!("{} STORE hit rates (hottest-lines / stride / uniform): {}", spec.label, [2usize, 4, 8, 16, 32, 64].iter().enumerate().map(|(ki, k)| format!("f=1/{k}: {:.6} / {:.6} / {:.6}", marks[ki].1, stride[ki] as f64 / tot, 1.0 / *k as f64)).collect::>().join("; ")); + } + let (hottest, hottest_count) = snap.iter().enumerate().map(|(t, &c)| (t as u32, c)).max_by_key(|&(_, c)| c).unwrap(); + log!("hottest item {} ({:#08x}): {} reads ({:.5}% of all; uniform mean {:.3})", spec.label, hottest, hottest_count, hottest_count as f64 * 100.0 / total_reads as f64, total_reads as f64 / ITEMS_N as f64); + { + let tag = format!("warps-{}-d{day}-n{nonces}-{}", spec.label, plant.name()); + let mut g = std::fs::File::create(format!("{out_dir}/{tag}-items.u32le")).unwrap(); + let mut bytes = Vec::with_capacity(ITEMS_N * 4); + for c in &snap { + bytes.extend_from_slice(&(*c as u32).to_le_bytes()); + } + g.write_all(&bytes).unwrap(); + } + log!("program {} DONE: {} nonces; 6-sigma (windowed) {}; hot-set items {}; hot-set lines {}; verdict {}", spec.label, nonces, if z_w > 6.0 { "FLAGGED" } else { "clear" }, if hot { "FLAGGED" } else { "clear" }, if line_hot { "FLAGGED" } else { "clear" }, if z_w > 6.0 || hot || line_hot { "FLAGGED" } else { "PASS (no test fired)" }); + let summary = Summary { + label: spec.label.clone(), + id: epoch.program.program_id(), + attempt: epoch.program.attempt, + nonces, + s01: s_items.top[0].1, + e01_w: c_items.top[0].1, + e01_flat: f_items.top[0].1, + s10: s_items.top[2].1, + e10_w: c_items.top[2].1, + e10_flat: f_items.top[2].1, + hot: hot || line_hot, + z_w, + top_quarter: mqs[0], + top_half: m_half, + exp_quarter: qsorted[0], + exp_half, + line_s01: s_lines.top[0].1, + line_e01: c_lines.top[0].1, + line_s10: s_lines.top[2].1, + line_e10: c_lines.top[2].1, + worst_site_ratio, + worst_site, + hash_mism: hm, + }; + (summary, epoch.dataset) +} + +#[allow(clippy::too_many_arguments)] +fn census_warps(programs: &[String], day: u64, nonces: u64, threads: usize, plant: Plant, validate: &str, check_every: u64, fingerprint: bool, out_dir: &str) { + log!("warps: programs {:?} day {day} nonces {nonces} threads {threads} plant {} validate {validate} check_every {check_every} fingerprint {fingerprint}", programs, plant.name()); + let t0 = Instant::now(); + let mut ds = day_dataset(day); + let (table, table_mism) = { + let mh = ds.memhard().unwrap(); + log!("day {day}: cache filled in {:.2} s, fnv {:016x}", t0.elapsed().as_secs_f64(), mh.cache.fnv1a64()); + build_table(&mh.params, &mh.cache, threads, validate) + }; + if table_mism > 0 { + log!("THE TABLE DISAGREES WITH THE LIBRARY ({table_mism} items); nothing below is trusted"); + } + let devnet_warp0: [u64; 5] = [0xb9034ec6711e5fac, 0x3b65da691cea9030, 0x7b301843257b4173, 0x713bf5a5f38ccf4f, 0x6c9c3fd5c15f968e]; + let devnet3_warp0: [u64; 5] = [0xf3caba27739e3d45, 0xe8619c2f28d2cd00, 0xb8e8ada83bda800d, 0xdb17c754f5b28cec, 0x2f189e5404433cac]; + let mut summaries = Vec::new(); + for name in programs { + let spec = program_spec(name); + let exp: Option<&[u64]> = match (name.as_str(), day) { + ("devnet", DEVNET_DAY) => Some(&devnet_warp0), + ("devnet3", DEVNET3_DAY) => Some(&devnet3_warp0), + _ => None, + }; + let (s, d) = run_program(&spec, ds, &table, day, nonces, threads, plant, check_every, fingerprint, exp, out_dir); + ds = d; + summaries.push(s); + } + let mut f = std::fs::File::create(format!("{out_dir}/census-d{day}-n{nonces}-{}-{}.txt", programs[0].replace(':', "_"), programs[programs.len() - 1].replace(':', "_"))).unwrap(); + writeln!(f, "# label id attempt nonces S0.1% E0.1%_window E0.1%_flat ratio0.1%_window S1% E1%_window E1%_flat ratio1%_window hot z_w top_quarter model_quarter top_half model_half line_S0.1% line_E0.1% line_S1% line_E1% worst_site worst_site_ratio0.1% hash_mismatches").unwrap(); + let mut n_hot = 0; + let mut n_ratio = 0; + let mut n_z = 0; + let mut tq: Vec = Vec::new(); + for s in &summaries { + let r = s.s01 / s.e01_w.max(1e-12); + writeln!(f, "{} {:016x} {} {} {:.5} {:.5} {:.5} {:.4} {:.5} {:.5} {:.5} {:.4} {} {:.2} {:.5} {:.5} {:.5} {:.5} {:.5} {:.5} {:.5} {:.5} {} {:.4} {}", s.label, s.id, s.attempt, s.nonces, s.s01 * 100.0, s.e01_w * 100.0, s.e01_flat * 100.0, r, s.s10 * 100.0, s.e10_w * 100.0, s.e10_flat * 100.0, s.s10 / s.e10_w.max(1e-12), s.hot as u8, s.z_w, s.top_quarter, s.exp_quarter, s.top_half, s.exp_half, s.line_s01 * 100.0, s.line_e01 * 100.0, s.line_s10 * 100.0, s.line_e10 * 100.0, s.worst_site, s.worst_site_ratio, s.hash_mism).unwrap(); + log!("SEED {} id {:016x} attempt {} S0.1 {:.5}% window {:.5}% flat {:.5}% ratio {:.4}x; S1 {:.5}% window {:.5}% ratio {:.4}x; top quarter {:.5} (model {:.5}) top half {:.5} (model {:.5}); lines S0.1 {:.5}% E {:.5}% S1 {:.5}% E {:.5}%; worst site {} ratio {:.4}x; hot {}", s.label, s.id, s.attempt, s.s01 * 100.0, s.e01_w * 100.0, s.e01_flat * 100.0, r, s.s10 * 100.0, s.e10_w * 100.0, s.s10 / s.e10_w.max(1e-12), s.top_quarter, s.exp_quarter, s.top_half, s.exp_half, s.line_s01 * 100.0, s.line_e01 * 100.0, s.line_s10 * 100.0, s.line_e10 * 100.0, s.worst_site, s.worst_site_ratio, s.hot as u8); + n_hot += s.hot as usize; + n_ratio += (r > 1.2) as usize; + n_z += (s.z_w > 6.0) as usize; + tq.push(s.top_quarter); + } + tq.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let mean_tq = tq.iter().sum::() / tq.len() as f64; + log!("CENSUS {}: {} programs at {} nonces; hot set in {}; top 0.1% beyond 1.2x of the window-model control in {}; windowed buckets64 beyond 6 sigma in {}; top-quarter share min {:.4} median {:.4} mean {:.4} max {:.4} (uniform 0.25)", if n_hot + n_ratio + n_z == 0 { "PASS" } else { "FLAGGED" }, summaries.len(), nonces, n_hot, n_ratio, n_z, tq[0], tq[tq.len() / 2], mean_tq, tq[tq.len() - 1]); +} + +// -------------------------------------------------------------------------------------------------------------- +// Q3(1): bit sensitivity of the line index to t +// -------------------------------------------------------------------------------------------------------------- + +fn census_steer(day: u64, items_log2: u32, threads: usize, plant: Plant) { + log!("steer: day {day} items 2^{items_log2} threads {threads} plant {}", plant.name()); + let ds = day_dataset(day); + let mh = ds.memhard().unwrap(); + let n = 1usize << items_log2; + // flips[round][b][bit]: count of items where flipping bit b of t flips address bit `bit` in round `round` + let rounds = [0usize, 1, 7]; + let flips: Vec>> = (0..3).map(|_| (0..32).map(|_| atomic_vec(22)).collect()).collect(); + let equal_pairs: Vec = atomic_vec(3); + let next = AtomicUsize::new(0); + let t0 = Instant::now(); + std::thread::scope(|sc| { + for _ in 0..threads { + let (next, flips, equal_pairs, mp, cache) = (&next, &flips, &equal_pairs, &mh.params, &mh.cache); + sc.spawn(move || { + let mut ts = [0u32; 64]; + let mut out = [[0u32; 16]; 64]; + let mut lines = [[0u32; 8]; 64]; + let mut base_lines = [[0u32; 8]; 64]; + let mut local = vec![vec![[0u32; 22]; 32]; 3]; + let mut eq = [0u32; 3]; + loop { + let b = next.fetch_add(1, Ordering::Relaxed); + if b * 64 >= n { + break; + } + let mut rng = SplitMix64::new(0xADC2_7777 ^ (b as u64).wrapping_mul(0x9E3779B97F4A7C15)); + for k in 0..64 { + ts[k] = rng.next() as u32; + } + derive_traced(&ts, mp, cache, &mut out, &mut base_lines, plant); + for bit in 0..32u32 { + let mut tf = [0u32; 64]; + for k in 0..64 { + tf[k] = ts[k] ^ (1 << bit); + } + derive_traced(&tf, mp, cache, &mut out, &mut lines, plant); + for k in 0..64 { + for (ri, &r) in rounds.iter().enumerate() { + let x = lines[k][r] ^ base_lines[k][r]; + for ab in 0..22 { + if (x >> ab) & 1 != 0 { + local[ri][bit as usize][ab] += 1; + } + } + if x == 0 { + eq[ri] += 1; + } + } + } + } + } + for ri in 0..3 { + for bit in 0..32 { + for ab in 0..22 { + flips[ri][bit][ab].fetch_add(local[ri][bit][ab], Ordering::Relaxed); + } + } + equal_pairs[ri].fetch_add(eq[ri], Ordering::Relaxed); + } + }); + } + }); + let trials = n as f64; + let sigma = (trials * 0.25).sqrt(); + log!("steer: {} items x 32 bit flips x {} rounds in {:.1} s", n, rounds.len(), t0.elapsed().as_secs_f64()); + let mut worst = (0f64, 0usize, 0usize, 0usize); + let mut any = false; + for (ri, &r) in rounds.iter().enumerate() { + for bit in 0..32 { + let row: Vec = (0..22).map(|ab| flips[ri][bit][ab].load(Ordering::Relaxed) as f64 / trials).collect(); + for (ab, p) in row.iter().enumerate() { + let z = ((p - 0.5) * trials / sigma).abs(); + if z > worst.0 { + worst = (z, r, bit, ab); + } + if z > 6.0 { + any = true; + } + } + if r == 0 { + log!("steer round {r} t bit {bit:2}: P(address bit flips) min {:.4} max {:.4} mean {:.4}", row.iter().cloned().fold(1.0, f64::min), row.iter().cloned().fold(0.0, f64::max), row.iter().sum::() / 22.0); + } + } + let eq = equal_pairs[ri].load(Ordering::Relaxed) as f64; + let exp = trials * 32.0 / LINES_N as f64; + log!("steer round {r}: pairs (t, t XOR 2^b) with an equal line index {} of {} (uniform expectation {:.2}, {:+.2} sigma){}", eq, n * 32, exp, (eq - exp) / exp.sqrt(), if (eq - exp) / exp.sqrt() > 6.0 { " FLAGGED" } else { "" }); + any |= (eq - exp) / exp.sqrt() > 6.0; + } + log!("steer: worst cell |z| {:.2} at round {} t bit {} address bit {} (6-sigma gate); verdict {}", worst.0, worst.1, worst.2, worst.3, if any { "FLAGGED" } else { "PASS" }); +} + +// -------------------------------------------------------------------------------------------------------------- +// Q3(3): the window layer over drawn programs, by the draws alone +// -------------------------------------------------------------------------------------------------------------- + +fn census_windows(programs: &[String], threads: usize, plant: Plant, out_dir: &str) { + log!("windows: {} programs threads {threads} plant {}", programs.len(), plant.name()); + let next = AtomicUsize::new(0); + let mask = (1u32 << DATASET_LOG2) - 1; + let t0 = Instant::now(); + let rows: Vec> = std::thread::scope(|sc| { + let mut hs = Vec::new(); + for _ in 0..threads { + let next = &next; + hs.push(sc.spawn(move || { + let mut rows = Vec::new(); + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + if i >= programs.len() { + break; + } + let spec = program_spec(&programs[i]); + let mut program = Epoch::chain_program(&spec.epoch_seed, Some(&spec.era), ProgramClass::V4, &spec.label); + if plant == Plant::AllQuarter { + for ins in program.instrs.iter_mut() { + if ins.op == Op::Load { + ins.win = 2; + ins.off = 0; + } + } + } + let windows = site_item_windows(&program, mask, DATASET_LOG2); + let qs = quarter_shares(&windows); + let mut s = qs; + s.sort_by(|a, b| b.partial_cmp(a).unwrap()); + let half = (qs[0] + qs[1]).max(qs[2] + qs[3]); + let mut wins = [0usize; 3]; + for ins in program.instrs.iter().filter(|i| i.op == Op::Load) { + wins[ins.win as usize] += 1; + } + rows.push((spec.label.clone(), program.program_id(), program.attempt, s[0], half, wins)); + } + rows + })); + } + hs.into_iter().map(|h| h.join().unwrap()).collect() + }); + let all: Vec<_> = rows.into_iter().flatten().collect(); + let mut f = std::fs::File::create(format!("{out_dir}/windows-{}-{}.txt", programs[0].replace(':', "_"), programs[programs.len() - 1].replace(':', "_"))).unwrap(); + writeln!(f, "# label id attempt top_quarter_share top_half_share sites_win0 sites_win1 sites_win2").unwrap(); + let mut tq: Vec = Vec::new(); + let mut th: Vec = Vec::new(); + let mut wins = [0usize; 3]; + for r in &all { + writeln!(f, "{} {:016x} {} {:.5} {:.5} {} {} {}", r.0, r.1, r.2, r.3, r.4, r.5[0], r.5[1], r.5[2]).unwrap(); + tq.push(r.3); + th.push(r.4); + for k in 0..3 { + wins[k] += r.5[k]; + } + } + tq.sort_by(|a, b| a.partial_cmp(b).unwrap()); + th.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let pct = |v: &[f64], p: f64| v[((v.len() - 1) as f64 * p).round() as usize]; + log!("windows: {} programs in {:.1} s; win draws 0/1/2: {} {} {} (uniform third each); top-quarter share min {:.4} p10 {:.4} median {:.4} p90 {:.4} max {:.4} mean {:.4} (uniform 0.25); top-half share min {:.4} median {:.4} p90 {:.4} max {:.4} mean {:.4} (uniform 0.5)", all.len(), t0.elapsed().as_secs_f64(), wins[0], wins[1], wins[2], tq[0], pct(&tq, 0.1), pct(&tq, 0.5), pct(&tq, 0.9), tq[tq.len() - 1], tq.iter().sum::() / tq.len() as f64, th[0], pct(&th, 0.5), pct(&th, 0.9), th[th.len() - 1], th.iter().sum::() / th.len() as f64); + log!("windows DONE: {}", if plant == Plant::AllQuarter { if tq[0] > 0.999 { "plant fired (every program 100% in one quarter)" } else { "PLANT DID NOT FIRE" } } else { "result is the distribution above (by design above uniform); priced in Q4" }); +} + +// -------------------------------------------------------------------------------------------------------------- +// Self-test against the packs +// -------------------------------------------------------------------------------------------------------------- + +fn selftest(threads: usize) { + let mut ok = true; + for (d, want) in [(DEVNET_DAY, 0x448274a57f508cbcu64), (DEVNET3_DAY, 0x7334fa46e5d972eb)] { + let t0 = Instant::now(); + let ds = day_dataset(d); + let got = ds.memhard().unwrap().cache.fnv1a64(); + log!("selftest: day {d} cache fnv {got:016x} want {want:016x} {} ({:.2} s)", if got == want { "OK" } else { "MISMATCH" }, t0.elapsed().as_secs_f64()); + ok &= got == want; + } + for (name, want_id, want_attempt) in [("devnet", 0xa785001687d8688au64, 1u32), ("devnet3", 0xfce15bf61030be57, 0)] { + let spec = program_spec(name); + let t0 = Instant::now(); + let p = Epoch::chain_program(&spec.epoch_seed, Some(&spec.era), ProgramClass::V4, &spec.label); + log!("selftest: {name} program id {:016x} attempt {} want {want_id:016x} attempt {want_attempt} {} ({:.2} s)", p.program_id(), p.attempt, if p.program_id() == want_id && p.attempt == want_attempt { "OK" } else { "MISMATCH" }, t0.elapsed().as_secs_f64()); + ok &= p.program_id() == want_id && p.attempt == want_attempt; + } + // the warp-0 vectors through the mirror, 64 warps each, with the library comparison + let _ = threads; + log!("selftest: {}", if ok { "PASS" } else { "FAIL" }); +} + +fn usage() -> ! { + eprintln!( + "adv-cache-2 selftest\n\ + adv-cache-2 lines --day0 20730 --days 16 --items-log2 24 --threads 88 --out [--plant none|quarter-lines|half-lines] [--validate all|sample|none]\n\ + adv-cache-2 days --day0 20730 --days 4096 --items-log2 16 --threads 88 --out [--plant quarter-lines --plant-day ]\n\ + adv-cache-2 warps --programs devnet|devnet3|era-fixed:1..16|era-drawn:1..16 --day 20730 --nonces 16777216 --threads 88 --out [--plant none|const-item] [--validate all|sample|none] [--check-every 997] [--fingerprint 1]\n\ + adv-cache-2 steer --day 20730 --items-log2 16 --threads 88 [--plant t-low]\n\ + adv-cache-2 windows --programs era-fixed:1..1024 --threads 88 --out [--plant all-quarter]" + ); + std::process::exit(2); +} + +fn main() { + let args: Vec = std::env::args().collect(); + if args.len() < 2 { + usage(); + } + let get = |k: &str, d: &str| -> String { + let mut i = 2; + while i + 1 < args.len() { + if args[i] == k { + return args[i + 1].clone(); + } + i += 1; + } + d.to_string() + }; + let threads: usize = get("--threads", "32").parse().unwrap(); + let out = get("--out", "."); + let plant = Plant::parse(&get("--plant", "none")); + let validate = get("--validate", "sample"); + log!("adv-cache-2 {} (igneum-pow generator v{}); args {:?}", env!("CARGO_PKG_VERSION"), igneum_pow::generator::GENERATOR_VERSION_V4, &args[1..]); + match args[1].as_str() { + "selftest" => selftest(threads), + "lines" => census_lines(get("--day0", &DEVNET_DAY.to_string()).parse().unwrap(), get("--days", "16").parse().unwrap(), get("--items-log2", "24").parse().unwrap(), threads, plant, &validate, &out), + "days" => census_days(get("--day0", &DEVNET_DAY.to_string()).parse().unwrap(), get("--days", "4096").parse().unwrap(), get("--items-log2", "16").parse().unwrap(), threads, plant, get("--plant-day", "0").parse().unwrap(), &out), + "warps" => census_warps(&expand_programs(&get("--programs", "devnet")), get("--day", &DEVNET_DAY.to_string()).parse().unwrap(), get("--nonces", "1048576").parse().unwrap(), threads, plant, &validate, get("--check-every", "997").parse().unwrap(), get("--fingerprint", "0") == "1", &out), + "steer" => census_steer(get("--day", &DEVNET_DAY.to_string()).parse().unwrap(), get("--items-log2", "16").parse().unwrap(), threads, plant), + "windows" => census_windows(&expand_programs(&get("--programs", "era-fixed:1..64")), threads, plant, &out), + _ => usage(), + } +}