igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

observer: a watchdog forces a reconnect after ten failed ticks (the live page went stale for an hour after a node restart)

Browse source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-josh 2026-10-05 18:22:22 +01:00
parent a05391ee11
commit 22ffc02566
1 changed files with 18 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
tools/observer/observer.mjs
View file

@ -440,6 +440,9 @@ async function flushChain() {
} }
async function tick(rpc) { async function tick(rpc) {
try { await tickInner(rpc); rpc.noteTick && rpc.noteTick(true); } catch (e) { rpc.noteTick && rpc.noteTick(false); }
}
async function tickInner(rpc) {
const now = Date.now(); const now = Date.now();
let dag, info, peersRes, hps; let dag, info, peersRes, hps;
try { try {
@ -449,7 +452,7 @@ async function tick(rpc) {
rpc.call('getConnectedPeerInfo', {}), rpc.call('getConnectedPeerInfo', {}),
rpc.call('estimateNetworkHashesPerSecond', { windowSize: 1000, startHash: null }).catch(() => null), rpc.call('estimateNetworkHashesPerSecond', { windowSize: 1000, startHash: null }).catch(() => null),
]); ]);
} catch (e) { log('tick failed', e.message); return; } } catch (e) { log('tick failed', e.message); throw e; }
if (dag.network) network = String(dag.network).startsWith('igneum') ? dag.network : `igneum-${dag.network}`; if (dag.network) network = String(dag.network).startsWith('igneum') ? dag.network : `igneum-${dag.network}`;
nodeVersion = info.serverVersion || nodeVersion; nodeVersion = info.serverVersion || nodeVersion;
if (dag.sink && dag.sink !== sinkHash) { sinkHash = dag.sink; pendingChain.add.add(dag.sink); } if (dag.sink && dag.sink !== sinkHash) { sinkHash = dag.sink; pendingChain.add.add(dag.sink); }
@ -963,6 +966,20 @@ async function main() {
}; };
rpc.onClose = () => { setTimeout(connect, 2000); }; rpc.onClose = () => { setTimeout(connect, 2000); };
await connect(); await connect();
// Watchdog (5 October 2026): after the observer node was restarted for 0.3.9 the reporter logged "rpc not connected" for
// an hour without reconnecting (the socket's close never resolved into a new connect). Ten failed ticks in a row force
// the socket shut and a fresh connect; the live page must never go stale while the node answers.
let failedTicks = 0;
rpc.noteTick = (ok) => {
failedTicks = ok ? 0 : failedTicks + 1;
if (failedTicks === 10) {
log('watchdog: 10 failed ticks, forcing a reconnect');
failedTicks = 0;
try { rpc.ws && rpc.ws.close(); } catch { }
rpc.open = false;
setTimeout(connect, 1000);
}
};
setInterval(async () => { if (flushBusy) return; flushBusy = true; try { await flushBlocks(); await flushChain(); } finally { flushBusy = false; } }, FLUSH_EVERY_MS); setInterval(async () => { if (flushBusy) return; flushBusy = true; try { await flushBlocks(); await flushChain(); } finally { flushBusy = false; } }, FLUSH_EVERY_MS);
setInterval(() => flushColors(rpc), FLUSH_EVERY_MS); setInterval(() => flushColors(rpc), FLUSH_EVERY_MS);