diff --git a/app/igneum-app/src/engine.rs b/app/igneum-app/src/engine.rs index 70545e3f..7b26e99d 100644 --- a/app/igneum-app/src/engine.rs +++ b/app/igneum-app/src/engine.rs @@ -120,6 +120,7 @@ impl Shared { st.setup_done = settings.setup_done; st.phase = if settings.setup_done { "dashboard".into() } else { "welcome".into() }; st.node.state = "stopped".into(); + st.node.tip_age_s = -1.0; st.mining.state = "idle".into(); st.mining.paused = settings.paused; st.mining.accepted_total = settings.accepted_total; @@ -1469,6 +1470,9 @@ impl Engine { let mut st = self.st(); st.node.pid = p.pid(); st.node.state = "starting".into(); + st.node.stall_exits = 0; + st.node.tip_age_s = -1.0; + st.node.sync_cause = String::new(); st.node.starts = self.node_starts; st.node.message = "opening the database".into(); st.node.restart_in_s = 0; @@ -3385,6 +3389,30 @@ impl Engine { } else if verdict != crate::watchdog::Action::None { // exit 43: the miner gave up on its worker; once more, then the card is faulted self.watchdog_verdict(i, verdict, &tail); + } else if is_stall_exit(code, &tail) { + // N4: the miner found no new template for its stall span and stopped hashing and voting. Once: + // the miner restarts and the node gets another look. Twice since the node started: the node + // itself restarts and re-dials its peers (the seed list), because the miner alone cannot fix a + // node that fell off the network. + let n = { let mut st = self.st(); st.node.stall_exits += 1; st.node.stall_exits }; + let label = self.miners[i].label.clone(); + for l in &tail { + self.shared.log(&format!(" miner: {l}")); + } + if n >= 2 { + self.shared.event("error", &format!("{label}: no new block to work on, twice; the node restarts and re-dials its peers")); + self.st().node.stall_exits = 0; + self.restart_node("the miner saw no new block twice", Duration::from_secs(2)); + } else { + let delay = jitter_secs(crate::platform::unix_now() + i as u64); + self.miners[i].restart_at = Some(now + Duration::from_secs(delay)); + self.shared.event("error", &format!("{label}: no new block to work on for the stall span; the miner restarts in {delay} s (a second stall restarts the node)")); + if let Some(c) = self.st().mining.cards.get_mut(card_idx) { + c.state = "restarting".into(); + c.hash_now = 0.0; + c.message = "no new block to work on; waiting for the node".into(); + } + } } else { self.miners[i].restarts += 1; let delay = jitter_secs(crate::platform::unix_now() + i as u64); @@ -3729,6 +3757,8 @@ impl Engine { let blue = kv_u64(text, "blue").unwrap_or(0); let difficulty = kv_f64(text, "difficulty").unwrap_or(0.0); let flag = kv(text, "synced").map(|s| s == "true"); + // N4: the sink header's age from the node (ca3-v4-0316 appends tip_age_s after synced=); -1 on an older node + let tip_age_s = kv(text, "tip_age_s").and_then(|s| s.parse::().ok()).unwrap_or(-1.0); let was_synced = self.st().node.synced; // Derived from every reading, never a one-shot transition (PC 2, 4 October 2026: the node caught up after a // clock fix and mined at 118 MH/s while the card still said "syncing", because getInfo's flag stayed false). @@ -3741,7 +3771,7 @@ impl Engine { let stable_s = self.sync_stable_since.map(|t| now.duration_since(t).as_secs_f64()).unwrap_or(0.0); let accepted_recent = self.last_accepted.map(|t| now.duration_since(t) <= Duration::from_secs(60)).unwrap_or(false); let private = self.shared.runtime.unsynced_mining && self.shared.runtime.peers.is_empty(); - let synced = sync_decision(&Reading { blocks, headers, peers, flag }, caught_up_moving, stable_s, accepted_recent, private); + let (synced, cause) = sync_decision_v2(&Reading { blocks, headers, peers, flag }, caught_up_moving, stable_s, accepted_recent, private, tip_age_s); self.sync_prev = Some(blocks); // the first clock source: local time against the latest block the peers produced, once blocks arrive if blocks > 0 && (peers > 0 || self.shared.runtime.peers.is_empty()) && now.duration_since(self.clock_last_sample) >= Duration::from_secs(9) { @@ -3764,11 +3794,17 @@ impl Engine { st.node.blue = blue; st.node.difficulty = difficulty; st.node.synced = synced; - st.node.state = if synced { "synced".into() } else { "syncing".into() }; + st.node.tip_age_s = tip_age_s; + st.node.sync_cause = cause.to_string(); + // the state word: "behind" for a frozen tip, "no peers" for none, "syncing" for the rest (N4: never + // "synced" on a tip that stopped moving) + st.node.state = if synced { "synced".into() } else if cause == "frozen" { "behind".into() } else if cause == "no peers" { "no peers".into() } else { "syncing".into() }; st.node.message = if synced { String::new() + } else if cause == "frozen" { + format!("no new block for {} s with {peers} peer{}; the chain may have moved on without this node", tip_age_s.round() as i64, if peers == 1 { "" } else { "s" }) } else if peers == 0 { - "waiting for a peer".into() + "waiting for a peer on our chain".into() } else if headers > blocks + 2 { format!("{blocks} of {headers} blocks") } else { @@ -4306,6 +4342,38 @@ pub fn sync_decision(r: &Reading, moving: bool, stable_s: f64, accepted_recent: caught_up && (moving || stable_s >= 60.0) } +/// The tip age at which a node that still says "synced" is treated as frozen (ledger N4, 6 October 2026: a home +/// miner's one same-digest peer relayed nothing for 53 minutes while the app read "mining, synced"). +pub const TIP_FROZEN_S: f64 = 120.0; + +/// N4: what "synced" means on every surface. The old decision, AND the tip moved in the last 120 s (the watch +/// line's tip_age_s, the sink header's age; -1 = the node does not report it yet, which gates nothing so an older node +/// keeps today's words), AND at least one peer (the handshake refuses a peer with another digest, so `peers` already +/// counts peers on our chain). Returns (synced, cause): the cause is "" when synced, else "frozen" (a peer, caught up, +/// no new block for over 120 s), "no peers", "syncing", or "behind" (the node's own flag false while the count moves). +pub fn sync_decision_v2(r: &Reading, moving: bool, stable_s: f64, accepted_recent: bool, private: bool, tip_age_s: f64) -> (bool, &'static str) { + if private { + return (true, ""); + } + let frozen = tip_age_s > TIP_FROZEN_S && !accepted_recent; + if r.peers == 0 && !accepted_recent { + return (false, "no peers"); + } + if frozen { + return (false, "frozen"); + } + if sync_decision(r, moving, stable_s, accepted_recent, private) { + return (true, ""); + } + (false, if r.headers > r.blocks + 2 { "syncing" } else { "behind" }) +} + +/// The miner's stall exit (the node lane, ca3-v4-0316): code 45, or a tail line starting "STALLED '". +pub const STALL_EXIT_CODE: i32 = 45; +pub fn is_stall_exit(code: i32, tail: &[String]) -> bool { + code == STALL_EXIT_CODE || tail.iter().any(|l| l.contains(" STALLED '") || l.starts_with("STALLED '")) +} + /// "...block timestamp is 1791112663000 but maximum timestamp allowed is 1791112600000" -> at least 63 s behind. fn behind_from_warning(text: &str) -> Option { let nums: Vec = text.split(|c: char| !c.is_ascii_digit()).filter(|s| s.len() >= 12).filter_map(|s| s.parse::().ok()).collect(); @@ -4536,6 +4604,44 @@ mod tests { assert!(sync_decision(&r(0, 0, 0, None), false, 0.0, false, true)); } + /// Ledger N4 (6 October 2026): a home miner's node kept one same-digest peer that relayed nothing for 53 minutes; + /// today's rule said "synced" the whole time. The known-failed case first, then the fix. + #[test] + fn n4_a_frozen_tip_is_never_synced() { + // the frozen tip: caught up, one peer, the count stable for an hour, getInfo's flag true. Today: synced. + let frozen = r(133_000, 133_000, 1, Some(true)); + assert!(sync_decision(&frozen, false, 3_180.0, false, false), "the old rule says synced on the frozen tip (the N4 failure)"); + // the fix: the tip age gates it + let (ok, cause) = sync_decision_v2(&frozen, false, 3_180.0, false, false, 3_180.0); + assert!(!ok); + assert_eq!(cause, "frozen"); + // the same node moving: synced + assert_eq!(sync_decision_v2(&frozen, true, 0.0, false, false, 4.0), (true, "")); + // exactly at the bound is still synced; one second over is not + assert_eq!(sync_decision_v2(&frozen, false, 60.0, false, false, 120.0), (true, "")); + assert_eq!(sync_decision_v2(&frozen, false, 60.0, false, false, 121.0), (false, "frozen")); + // an older node that does not report the age (-1) keeps today's words + assert_eq!(sync_decision_v2(&frozen, false, 60.0, false, false, -1.0), (true, "")); + // our own block accepted in the last minute proves the tip moves, whatever the reported age + assert_eq!(sync_decision_v2(&frozen, false, 60.0, true, false, 900.0), (true, "")); + // no peer at all: "no peers", not "frozen" and never synced + assert_eq!(sync_decision_v2(&r(133_000, 133_000, 0, Some(true)), false, 600.0, false, false, 500.0), (false, "no peers")); + // still downloading: "syncing" + assert_eq!(sync_decision_v2(&r(100, 700, 1, Some(false)), true, 0.0, false, false, 2.0), (false, "syncing")); + // caught up but the flag false and the count not yet stable: "behind" + assert_eq!(sync_decision_v2(&r(700, 700, 1, Some(false)), false, 10.0, false, false, 5.0), (false, "behind")); + // a private test node is always synced + assert_eq!(sync_decision_v2(&r(0, 0, 0, None), false, 0.0, false, true, 9_999.0), (true, "")); + } + + #[test] + fn n4_the_stall_exit_is_code_45_or_the_stalled_line() { + assert!(is_stall_exit(45, &[])); + assert!(is_stall_exit(1, &["1791327000.123 STALLED 'nvidia-1ccfe586-1': no new template for 300 s (tip d3dc2f78, daa 197752); the node has fallen off the network or stopped moving; hashing and voting paused, exiting with code 45 so the launcher restarts (and after a second stall, restarts the node)".to_string()])); + assert!(!is_stall_exit(1, &["worker fault: rejected nonce".to_string()])); + assert!(!is_stall_exit(42, &[])); + } + #[test] fn node_warning() { let l = "2026-10-04 11:02:11.123+00:00 [WARN ] HandleRelayInvsFlow flow error: the block timestamp is too far into the future: block timestamp is 1791112663000 but maximum timestamp allowed is 1791112600000"; diff --git a/app/igneum-app/src/state.rs b/app/igneum-app/src/state.rs index 19665216..8da9c2bf 100644 --- a/app/igneum-app/src/state.rs +++ b/app/igneum-app/src/state.rs @@ -21,6 +21,13 @@ pub struct NodeState { pub version: String, pub last_reading_age_s: f64, pub message: String, + /// N4 (6 October 2026, the frozen tip): seconds since the sink block's header time, from the watch line's + /// tip_age_s (the node lane, ca3-v4-0316); -1 until the node reports it. "synced" needs it at or under 120. + pub tip_age_s: f64, + /// why the node is not "synced", in plain words ("" when synced): "behind" | "no peers" | "syncing" | "frozen" + pub sync_cause: String, + /// the miner's stall exits (code 45, "STALLED") since the node last started; the second one restarts the node + pub stall_exits: u32, /// a consensus switch the signed manifest announced (difficulty v2 activation DAA); 0 = none pub consensus_switch_daa: u64, pub override_restart_wait: String, diff --git a/app/igneum-app/ui/app.js b/app/igneum-app/ui/app.js index 6645dcaf..7e9e1218 100644 --- a/app/igneum-app/ui/app.js +++ b/app/igneum-app/ui/app.js @@ -450,7 +450,7 @@ var View = (function () { if (!on) return { label: 'Start mining', sub: cards.length ? 'switch a card on in Cards first' : 'no card this app can use', cls: '', disabled: true, act: '' }; if (m.paused) return { label: 'Start mining', sub: 'paused · the node keeps running', cls: '', disabled: false, act: 'resume' }; if (m.state === 'mining') return { label: 'Stop mining', sub: 'mining on ' + mining + ' of ' + on + ' card' + (on === 1 ? '' : 's'), cls: 'stop', disabled: false, act: 'pause' }; - if (m.state === 'waiting') return { label: 'Stop mining', sub: n && n.synced ? 'card starting' : 'waiting for the node to sync', cls: 'stop', disabled: false, act: 'pause' }; + if (m.state === 'waiting') return { label: 'Stop mining', sub: n && n.synced ? 'card starting' : n && n.state === 'behind' ? 'waiting: the node is behind the chain' : n && n.state === 'no peers' ? 'waiting: no peers on our chain' : 'waiting for the node to sync', cls: 'stop', disabled: false, act: 'pause' }; return { label: 'Stop mining', sub: m.state || 'starting', cls: 'stop', disabled: false, act: 'pause' }; } // the row's state word while the miner is paused: the card is on, the miner is not @@ -461,7 +461,10 @@ var View = (function () { var bad = clock && clock.severity === 'block'; if (bad) return { word: n.state === 'synced' ? 'synced' : n.state, line: 'Your clock is off by ' + skewWord(clock.skew_s) + '. Mining waits until it is synced.', tone: 'bad' }; switch (n.state) { - case 'synced': return { word: 'synced', line: 'This node has every block the network has made.' + (n.last_reading_age_s >= 0 ? ' Read ' + Math.round(n.last_reading_age_s) + ' s ago.' : ''), tone: 'ok' }; + case 'synced': return { word: 'synced', line: 'This node has every block the network has made.' + (n.tip_age_s >= 0 ? ' Last block ' + Math.round(n.tip_age_s) + ' s ago.' : n.last_reading_age_s >= 0 ? ' Read ' + Math.round(n.last_reading_age_s) + ' s ago.' : ''), tone: 'ok' }; + // N4 (6 October 2026): a tip that stopped moving is never "synced"; the node says why + case 'behind': return { word: 'behind', line: (n.tip_age_s >= 0 ? 'No new block for ' + tipAge(n.tip_age_s) + '. ' : '') + 'The chain may have moved on without this node. Mining pauses until it catches up; the node re-dials its peers by itself.', tone: 'bad' }; + case 'no peers': return { word: 'no peers', line: 'No other node on our chain is connected. The node keeps dialling the seed list; mining waits.', tone: 'bad' }; case 'syncing': var pct = n.headers > n.blocks && n.headers > 0 ? Math.floor(n.blocks / n.headers * 100) : 0; var line = n.headers > n.blocks ? (eta || n.message || 'downloading blocks') + ' · ' + withCommas(n.blocks) + ' of ' + withCommas(n.headers) + ' (' + pct + '%)' : (n.message || 'caught up, waiting for the next block'); @@ -473,11 +476,14 @@ var View = (function () { } return { word: n.state || 'starting', line: n.message || '', tone: '' }; } - // the one node line on Mine: "Node synced · 6 peers · 109,859 blocks" + // "12 s" / "4 min" / "53 min" / "2 h" for a tip age + function tipAge(s) { s = Math.max(0, Math.round(s)); return s < 90 ? s + ' s' : s < 5400 ? Math.round(s / 60) + ' min' : Math.round(s / 3600) + ' h'; } + // the one node line on Mine: "Node synced · 6 peers · 109,859 blocks · last block 4 s ago" function nodeLine(n, clock, eta) { var w = nodeWords(n, clock, eta), parts = ['Node ' + w.word]; if (n.peers > 0) parts.push(n.peers + ' peer' + (n.peers === 1 ? '' : 's')); if (n.blocks > 0) parts.push(withCommas(n.blocks) + ' blocks'); + if (n.tip_age_s >= 0 && (n.state === 'synced' || n.state === 'behind')) parts.push('last block ' + tipAge(n.tip_age_s) + ' ago'); return { text: parts.join(' · '), sub: w.line, tone: w.tone }; } function skewWord(sk) { sk = Math.round(Math.abs(sk || 0)); return sk < 120 ? sk + ' s' : sk < 7200 ? Math.floor(sk / 60) + ' min ' + (sk % 60) + ' s' : Math.floor(sk / 3600) + ' h ' + Math.floor(sk % 3600 / 60) + ' min'; } @@ -653,6 +659,8 @@ var View = (function () { if (n.state === 'syncing') return { text: 'Syncing the node', tone: '' }; if (n.state === 'starting') return { text: 'Starting the node', tone: '' }; if (n.state === 'restarting') return { text: 'Node restarting', tone: 'bad' }; + if (n.state === 'behind') return { text: 'Node behind', tone: 'bad' }; + if (n.state === 'no peers') return { text: 'No peers', tone: 'bad' }; if (n.state === 'failed') return { text: 'Node failed', tone: 'bad' }; if (n.state === 'stopped') return { text: 'Node stopped', tone: 'bad' }; if (m.state === 'waiting') return { text: 'Starting the worker', tone: '' }; @@ -744,7 +752,7 @@ var View = (function () { // the confirmations' words (in place, never a dialog) function addressAsk(a) { return 'Pay ' + shortHex(a, 6, 4) + ' from the next block? Mining restarts.'; } function trustAsk(on) { return on ? 'Include proofs the node never checked? Devnet only. The node restarts.' : 'Check proofs again? The node restarts.'; } - return { PAGES: PAGES, page: page, withCommas: withCommas, compact: compact, rel: rel, ago: ago, shortHex: shortHex, kindWord: kindWord, shownCards: shownCards, present: present, cardTitle: cardTitle, cardRow: cardRow, rowWord: rowWord, tuneWord: tuneWord, tuneEta: tuneEta, tuningCard: tuningCard, poundsPerDay: poundsPerDay, money: money, effOf: effOf, effText: effText, wattsTotal: wattsTotal, noReading: noReading, cells: cells, nextCheck: nextCheck, tunePoint: tunePoint, tunable: tunable, tuneWords: tuneWords, schedule: schedule, GOALS: GOALS, GOAL_NAMES: GOAL_NAMES, goalWords: goalWords, ign: ign, balanceWords: balanceWords, fleetWatts: fleetWatts, fleetPounds: fleetPounds, isMine: isMine, liveNote: liveNote, toggle: toggle, nodeWords: nodeWords, nodeLine: nodeLine, skewWord: skewWord, peersLine: peersLine, heightLine: heightLine, nextSwitch: nextSwitch, switchLine: switchLine, proveWords: proveWords, verifierWords: verifierWords, proveTier: proveTier, proveLine: proveLine, devFeeText: devFeeText, devFeeLine: devFeeLine, jobsNote: jobsNote, eventLine: eventLine, plainEvent: plainEvent, pill: pill, beforeOf: beforeOf, ember: ember, fleetSaving: fleetSaving, addressAsk: addressAsk, trustAsk: trustAsk }; + return { PAGES: PAGES, page: page, withCommas: withCommas, compact: compact, rel: rel, ago: ago, shortHex: shortHex, kindWord: kindWord, shownCards: shownCards, present: present, cardTitle: cardTitle, cardRow: cardRow, rowWord: rowWord, tuneWord: tuneWord, tuneEta: tuneEta, tuningCard: tuningCard, poundsPerDay: poundsPerDay, money: money, effOf: effOf, effText: effText, wattsTotal: wattsTotal, noReading: noReading, cells: cells, nextCheck: nextCheck, tunePoint: tunePoint, tunable: tunable, tuneWords: tuneWords, schedule: schedule, GOALS: GOALS, GOAL_NAMES: GOAL_NAMES, goalWords: goalWords, ign: ign, balanceWords: balanceWords, fleetWatts: fleetWatts, fleetPounds: fleetPounds, isMine: isMine, liveNote: liveNote, toggle: toggle, nodeWords: nodeWords, nodeLine: nodeLine, skewWord: skewWord, peersLine: peersLine, heightLine: heightLine, tipAge: tipAge, nextSwitch: nextSwitch, switchLine: switchLine, proveWords: proveWords, verifierWords: verifierWords, proveTier: proveTier, proveLine: proveLine, devFeeText: devFeeText, devFeeLine: devFeeLine, jobsNote: jobsNote, eventLine: eventLine, plainEvent: plainEvent, pill: pill, beforeOf: beforeOf, ember: ember, fleetSaving: fleetSaving, addressAsk: addressAsk, trustAsk: trustAsk }; })(); /* ---------- Ember Tune: the card row's tuning line (pure; tune-line.test.mjs loads this block) ---------- One line per card from the card state (src/state.rs, src/ember.rs): running (the phase and the step), tuned diff --git a/app/igneum-app/ui/view.test.mjs b/app/igneum-app/ui/view.test.mjs index d113c5c5..b85b30bf 100644 --- a/app/igneum-app/ui/view.test.mjs +++ b/app/igneum-app/ui/view.test.mjs @@ -410,3 +410,23 @@ test('the 0.3.16 engine fields (ember-tune fd03f35): the balance row, the fleet assert.equal(V.isMine('8faf', mineCards), false, 'under 6 hex never matches'); assert.equal(V.isMine('', mineCards), false); }); + +test('N4 (6 October 2026): a frozen tip reads behind, never synced; the tip age is on the node line', () => { + const n = (over) => ({ state: 'synced', blocks: 133000, headers: 133000, peers: 1, daa: 140000, last_reading_age_s: 4, tip_age_s: 3, message: '', restart_in_s: 0, ...over }); + const ok = V.nodeLine(n(), { severity: 'none' }, ''); + assert.equal(ok.text, 'Node synced · 1 peer · 133,000 blocks · last block 3 s ago'); assert.equal(ok.tone, 'ok'); + assert.match(ok.sub, /Last block 3 s ago/); + const behind = V.nodeLine(n({ state: 'behind', synced: false, tip_age_s: 3180, sync_cause: 'frozen' }), { severity: 'none' }, ''); + assert.equal(behind.text, 'Node behind · 1 peer · 133,000 blocks · last block 53 min ago'); assert.equal(behind.tone, 'bad'); + assert.match(behind.sub, /^No new block for 53 min\. The chain may have moved on without this node\./); + const none = V.nodeLine(n({ state: 'no peers', synced: false, peers: 0, tip_age_s: 500 }), { severity: 'none' }, ''); + assert.equal(none.text, 'No peers · 133,000 blocks'.replace('No peers', 'Node no peers')); assert.equal(none.tone, 'bad'); + assert.match(none.sub, /^No other node on our chain is connected/); + // an older node without the age keeps the old words + assert.equal(V.nodeLine(n({ tip_age_s: -1 }), { severity: 'none' }, '').text, 'Node synced · 1 peer · 133,000 blocks'); + assert.equal(V.tipAge(12), '12 s'); assert.equal(V.tipAge(240), '4 min'); assert.equal(V.tipAge(3180), '53 min'); assert.equal(V.tipAge(7300), '2 h'); + const base = { setup_done: true, quitting: false, mining: { state: 'waiting', hash_total: 0, cards: [] }, node: { state: 'behind' } }; + assert.deepEqual(V.pill(base), { text: 'Node behind', tone: 'bad' }); + assert.deepEqual(V.pill({ ...base, node: { state: 'no peers' } }), { text: 'No peers', tone: 'bad' }); + assert.equal(V.toggle({ state: 'waiting', paused: false, cards: [card({ state: 'waiting' })] }, { synced: false, state: 'behind' }, {}).sub, 'waiting: the node is behind the chain'); +});