2.0.2: a quit that quits, whatever the node's state (the quit guard)
PC 2, 8 October 2026, 21:24 UK: api/quit on 2.0.1 left all four processes up 90 s later while the node sat in initial
sync. Now: a quit guard armed at the ask (the server's /api/quit and the engine's Cmd::Quit alike) ends the node and
the workers by the pids the state records, never by name, 45 s after the ask when the engine has not left by itself,
prints the exit line and leaves; the node's grace is 10 s then the kill; the last log upload waits 10 s at most; the
window reads the stage ("quitting: the node is being stopped") on the pill and the big button.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
parent
ec35d31c50
commit
c00e5dc42e
7 changed files with 213 additions and 6 deletions
|
|
@ -167,6 +167,8 @@ pub struct Shared {
|
|||
/// miner-ui-5: the machine's own ladder record (src/ladder.rs), `<app dir>/ladder.json`
|
||||
pub ladder: Mutex<crate::ladder::Ladder>,
|
||||
pub ladder_path: PathBuf,
|
||||
/// 2.0.2: the quit guard is armed once per process (src/quitguard.rs)
|
||||
pub quit_armed: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
impl Shared {
|
||||
|
|
@ -230,6 +232,7 @@ impl Shared {
|
|||
started: Instant::now(),
|
||||
started_unix: crate::platform::unix_now_f(),
|
||||
engine_log: Mutex::new(engine_log),
|
||||
quit_armed: std::sync::atomic::AtomicBool::new(false),
|
||||
log_path,
|
||||
port: std::sync::atomic::AtomicU16::new(0),
|
||||
state_error_logged: std::sync::atomic::AtomicBool::new(false),
|
||||
|
|
@ -339,7 +342,7 @@ impl Shared {
|
|||
})
|
||||
}
|
||||
|
||||
fn save_settings(&self) {
|
||||
pub fn save_settings(&self) {
|
||||
self.settings.lock().unwrap().save(&self.settings_path);
|
||||
}
|
||||
|
||||
|
|
@ -1829,6 +1832,8 @@ impl Engine {
|
|||
self.quit_source = source;
|
||||
self.quitting = true;
|
||||
self.st().quitting = true;
|
||||
// 2.0.2: the bound over the whole quit, whatever the node does (PC 2, 21:24 UK)
|
||||
crate::quitguard::arm(self.shared.clone(), exit_line(self.quit_source));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2419,8 +2424,8 @@ impl Engine {
|
|||
|
||||
fn stop_node(&mut self) {
|
||||
if let Some(mut n) = self.node.take() {
|
||||
self.shared.log("stopping the node");
|
||||
let code = n.stop(30);
|
||||
self.shared.log(&format!("stopping the node (pid {}, {} s grace, then the kill)", n.pid(), crate::quitguard::NODE_GRACE_S));
|
||||
let code = n.stop(crate::quitguard::NODE_GRACE_S);
|
||||
self.shared.log(&format!("node stopped (exit {code:?})"));
|
||||
}
|
||||
self.st().node.state = "stopped".into();
|
||||
|
|
@ -5753,7 +5758,16 @@ impl Engine {
|
|||
}
|
||||
});
|
||||
if wait {
|
||||
let _ = t.join();
|
||||
// 2.0.2: bounded (the quit's last step); an intake that does not answer never holds the exit
|
||||
let until = Instant::now() + Duration::from_secs(crate::quitguard::UPLOAD_WAIT_S);
|
||||
while !t.is_finished() && Instant::now() < until {
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
if t.is_finished() {
|
||||
let _ = t.join();
|
||||
} else {
|
||||
self.shared.log(&format!("the last log upload did not finish in {} s; leaving without it", crate::quitguard::UPLOAD_WAIT_S));
|
||||
}
|
||||
} else {
|
||||
self.upload_thread = Some(t);
|
||||
}
|
||||
|
|
@ -5769,6 +5783,7 @@ impl Engine {
|
|||
self.sweep_abort("the app is quitting");
|
||||
}
|
||||
self.sweep_helper_quit();
|
||||
crate::quitguard::set_stage(&self.shared, crate::quitguard::Stage::Miners);
|
||||
self.stop_miners("quit");
|
||||
self.stability_line();
|
||||
if let Some(mut t) = self.telemetry.take() {
|
||||
|
|
@ -5786,10 +5801,12 @@ impl Engine {
|
|||
if let Some(mut w) = self.watch.take() {
|
||||
w.stop(2);
|
||||
}
|
||||
crate::quitguard::set_stage(&self.shared, crate::quitguard::Stage::Node);
|
||||
self.stop_node();
|
||||
if let Some(mut k) = self.keep_awake.take() {
|
||||
k.stop();
|
||||
}
|
||||
crate::quitguard::set_stage(&self.shared, crate::quitguard::Stage::Logs);
|
||||
let st = self.st();
|
||||
let summary = format!(
|
||||
"SUMMARY after {}: node started {} time(s), restarts {}, {} accepted blocks this run ({} lifetime); data stays in {}",
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ mod tiertable;
|
|||
mod heat;
|
||||
mod powertask;
|
||||
mod watchdog;
|
||||
mod quitguard;
|
||||
mod device;
|
||||
mod edition;
|
||||
mod live;
|
||||
|
|
|
|||
181
app/igneum-app/src/quitguard.rs
Normal file
181
app/igneum-app/src/quitguard.rs
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
//! A quit that quits (PC 2, 8 October 2026, 21:24 UK: api/quit on 2.0.1 left all four processes up 90 s later while the
|
||||
//! node sat in initial sync; the founder's plug-tune-play rule says no fault a user fixes by hand). The engine's own
|
||||
//! quit path stops the miners, then the node, by their process handles with a short grace and a kill; this module is
|
||||
//! the bound over the whole of it: a guard armed the moment a quit is asked, which, when the process is still here
|
||||
//! after QUIT_BOUND_S whatever the engine is doing (a tick blocked on a syncing node's RPC, a stop that never
|
||||
//! answers), ends the node and the workers by the pids the state records, never by name, prints the exit line and
|
||||
//! leaves. The window reads the stage words from `quit_stage` meanwhile.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
/// The whole quit, whatever the node's state: miners, node, the last upload. Past this the guard ends what is left.
|
||||
pub const QUIT_BOUND_S: u64 = 45;
|
||||
/// The node's grace after its stop signal before the kill (30 before 2.0.2; a syncing node ignored it for longer).
|
||||
pub const NODE_GRACE_S: u64 = 10;
|
||||
/// How long the quit waits for the last log upload.
|
||||
pub const UPLOAD_WAIT_S: u64 = 10;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Stage {
|
||||
Miners,
|
||||
Node,
|
||||
Logs,
|
||||
Forced,
|
||||
}
|
||||
|
||||
impl Stage {
|
||||
pub fn words(self) -> &'static str {
|
||||
match self {
|
||||
Stage::Miners => "quitting: the miners are being stopped",
|
||||
Stage::Node => "quitting: the node is being stopped",
|
||||
Stage::Logs => "quitting: the last log upload",
|
||||
Stage::Forced => "quitting: the node did not stop in time; ending it by pid",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The clock over a quit (seconds since the engine started, the engine's own `secs`).
|
||||
pub struct Guard {
|
||||
armed_at: f64,
|
||||
}
|
||||
|
||||
impl Guard {
|
||||
pub fn new(now: f64) -> Guard {
|
||||
Guard { armed_at: now }
|
||||
}
|
||||
pub fn overdue(&self, now: f64) -> bool {
|
||||
now - self.armed_at >= QUIT_BOUND_S as f64
|
||||
}
|
||||
}
|
||||
|
||||
/// The pids the state records: the node's and every card's worker (0 = none).
|
||||
pub fn recorded_pids(st: &crate::state::State) -> Vec<(&'static str, u32)> {
|
||||
let mut v = Vec::new();
|
||||
if st.node.pid > 0 {
|
||||
v.push(("node", st.node.pid));
|
||||
}
|
||||
for c in &st.mining.cards {
|
||||
if c.pid > 0 {
|
||||
v.push(("worker", c.pid));
|
||||
}
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
/// Ends each pid (platform::kill_pid: taskkill /PID /T /F on Windows, SIGKILL elsewhere); the record lines.
|
||||
pub fn enforce_pids(pids: &[(&str, u32)]) -> Vec<String> {
|
||||
pids.iter()
|
||||
.map(|(what, pid)| {
|
||||
crate::platform::kill_pid(*pid);
|
||||
format!("QUIT event=forced what={what} pid={pid}")
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Whether a pid is still a live process (unix: kill -0; Windows: tasklist by pid).
|
||||
pub fn pid_alive(pid: u32) -> bool {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
unsafe { libc::kill(pid as i32, 0) == 0 }
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let out = crate::platform::quiet(&mut std::process::Command::new(crate::platform::tool("tasklist"))).args(["/FI", &format!("PID eq {pid}"), "/NH"]).output();
|
||||
out.map(|o| String::from_utf8_lossy(&o.stdout).contains(&pid.to_string())).unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets the stage words the window shows.
|
||||
pub fn set_stage(shared: &crate::engine::Shared, stage: Stage) {
|
||||
let mut st = shared.state.lock().unwrap();
|
||||
st.quitting = true;
|
||||
st.quit_stage = stage.words().into();
|
||||
}
|
||||
|
||||
/// Arms the guard once per process (the server's /api/quit and the engine's Cmd::Quit both call it): a thread that,
|
||||
/// QUIT_BOUND_S after the ask, ends the recorded pids and leaves with the exit line, when the engine has not left
|
||||
/// by itself. `exit_line` is what the window host reads ("EXIT" or "EXIT update").
|
||||
pub fn arm(shared: Arc<crate::engine::Shared>, exit_line: &'static str) {
|
||||
if shared.quit_armed.swap(true, std::sync::atomic::Ordering::SeqCst) {
|
||||
return;
|
||||
}
|
||||
std::thread::Builder::new()
|
||||
.name("quit-guard".into())
|
||||
.spawn(move || {
|
||||
std::thread::sleep(std::time::Duration::from_secs(QUIT_BOUND_S));
|
||||
let pids = recorded_pids(&shared.state.lock().unwrap());
|
||||
shared.log(&format!("quit guard: still here {QUIT_BOUND_S} s after the ask; ending {} recorded process(es) by pid and leaving", pids.len()));
|
||||
set_stage(&shared, Stage::Forced);
|
||||
for l in enforce_pids(&pids) {
|
||||
shared.log(&l);
|
||||
}
|
||||
shared.save_settings();
|
||||
shared.log("stopped (by the quit guard)");
|
||||
println!("{exit_line}");
|
||||
use std::io::Write;
|
||||
let _ = std::io::stdout().flush();
|
||||
std::process::exit(0);
|
||||
})
|
||||
.expect("quit guard thread");
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Known failed first: a node that never answers its stop (here a child that ignores SIGTERM, as a node stuck in
|
||||
/// initial sync did on PC 2) is ended by its pid within the grace plus a second, never left running.
|
||||
#[test]
|
||||
fn a_node_that_never_answers_its_stop_is_ended_by_pid_within_the_bound() {
|
||||
if cfg!(windows) {
|
||||
return;
|
||||
}
|
||||
let (tx, _rx) = std::sync::mpsc::channel();
|
||||
let log = std::env::temp_dir().join(format!("igneum-quit-node-{}.log", std::process::id()));
|
||||
let mut p = crate::procs::spawn(crate::procs::Source::Node, std::path::Path::new("sh"), &["-c".into(), "trap '' TERM; while :; do sleep 1; done".into()], None, &log, &tx, &[]).unwrap();
|
||||
std::thread::sleep(std::time::Duration::from_millis(300));
|
||||
let pid = p.pid();
|
||||
let started = std::time::Instant::now();
|
||||
let code = p.stop(NODE_GRACE_S.min(2));
|
||||
assert!(started.elapsed() < std::time::Duration::from_secs(4), "the stop took {:?}", started.elapsed());
|
||||
assert_eq!(code, Some(-9), "the grace ran out and the kill ended it");
|
||||
assert!(!pid_alive(pid), "pid {pid} is still here");
|
||||
let _ = std::fs::remove_file(log);
|
||||
}
|
||||
|
||||
/// The guard: armed at the ask, overdue after the bound; it ends exactly the pids the state records (the node and
|
||||
/// every card's worker), each by pid, and the words name the stage.
|
||||
#[test]
|
||||
fn the_guard_ends_the_recorded_pids_and_names_the_stage() {
|
||||
assert_eq!(QUIT_BOUND_S, 45);
|
||||
assert!(NODE_GRACE_S <= 10 && UPLOAD_WAIT_S <= 10, "the normal path stays well inside the bound");
|
||||
let g = Guard::new(0.0);
|
||||
assert!(!g.overdue(QUIT_BOUND_S as f64 - 1.0));
|
||||
assert!(g.overdue(QUIT_BOUND_S as f64));
|
||||
let mut st = crate::state::State::default();
|
||||
st.node.pid = 4242;
|
||||
st.mining.cards.push(crate::state::CardState { pid: 777, ..Default::default() });
|
||||
st.mining.cards.push(crate::state::CardState { pid: 0, ..Default::default() });
|
||||
assert_eq!(recorded_pids(&st), vec![("node", 4242), ("worker", 777)]);
|
||||
assert_eq!(Stage::Miners.words(), "quitting: the miners are being stopped");
|
||||
assert_eq!(Stage::Node.words(), "quitting: the node is being stopped");
|
||||
assert_eq!(Stage::Logs.words(), "quitting: the last log upload");
|
||||
assert_eq!(Stage::Forced.words(), "quitting: the node did not stop in time; ending it by pid");
|
||||
if cfg!(windows) {
|
||||
return;
|
||||
}
|
||||
let mut child = std::process::Command::new("sleep").arg("100").spawn().unwrap();
|
||||
let pid = child.id();
|
||||
let lines = enforce_pids(&[("node", pid)]);
|
||||
assert_eq!(lines, vec![format!("QUIT event=forced what=node pid={pid}")]);
|
||||
// the child is reaped here (a killed but unreaped child still answers kill -0 as a zombie)
|
||||
let started = std::time::Instant::now();
|
||||
let mut ended = None;
|
||||
while ended.is_none() && started.elapsed() < std::time::Duration::from_secs(3) {
|
||||
ended = child.try_wait().unwrap();
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
}
|
||||
assert!(ended.is_some(), "pid {pid} survived the guard");
|
||||
assert!(!pid_alive(pid), "pid {pid} is still a process after the reap");
|
||||
}
|
||||
}
|
||||
|
|
@ -545,6 +545,8 @@ fn api_post(shared: &Arc<Shared>, path: &str, body: Value) -> Result<Value, Stri
|
|||
"/api/quit" => {
|
||||
// the caller is on 127.0.0.1 and holds the token: the installer, the OTA apply, a script that read app.url
|
||||
shared.send(Cmd::Quit("POST /api/quit (a local caller with the token: the installer, the OTA apply, or a script that read app.url)"));
|
||||
// 2.0.2: armed here too, so a tick blocked on a syncing node (PC 2, 21:24 UK) cannot hold the quit
|
||||
crate::quitguard::arm(shared.clone(), "EXIT update");
|
||||
Ok(json!({ "ok": true }))
|
||||
}
|
||||
_ => Err("unknown api".into()),
|
||||
|
|
|
|||
|
|
@ -628,6 +628,8 @@ pub struct State {
|
|||
pub started_at: f64,
|
||||
pub now: f64,
|
||||
pub quitting: bool,
|
||||
/// 2.0.2: the quit's stage in words (src/quitguard.rs Stage), what the window shows while it waits
|
||||
pub quit_stage: String,
|
||||
pub live_page: String,
|
||||
/// miner-ui-5: this machine's own ladder record (src/ladder.rs), summarised at `now`
|
||||
pub ladder: crate::ladder::LadderState,
|
||||
|
|
|
|||
|
|
@ -612,7 +612,7 @@ var View = (function () {
|
|||
var cards = shownCards(m.cards).filter(present);
|
||||
var on = cards.filter(function (c) { return c.enabled; }).length;
|
||||
var mining = cards.filter(function (c) { return c.state === 'mining'; }).length;
|
||||
if (s && s.quitting) return { label: 'Stopping', sub: 'miners first, then the node', cls: 'stop', disabled: true, act: '' };
|
||||
if (s && s.quitting) return { label: 'Stopping', sub: s.quit_stage || 'miners first, then the node', cls: 'stop', disabled: true, act: '' };
|
||||
var tc = tuningCard(cards);
|
||||
if (tc) return { label: 'Tuning', sub: 'mining again in ' + tuneEta(tc.tune_eta_s) + ' · ' + tc.name, cls: 'stop', disabled: true, act: '' };
|
||||
var heldCards = cards.filter(function (c) { return c.state === 'held'; }).length;
|
||||
|
|
@ -848,7 +848,7 @@ var View = (function () {
|
|||
}
|
||||
// the pill in the top bar: a verb a user understands, and its tone (on = molten, bad = ember, '' = grey)
|
||||
function pill(s) {
|
||||
if (s.quitting) return { text: 'Stopping', tone: '' };
|
||||
if (s.quitting) return { text: s.quit_stage ? 'Stopping · ' + s.quit_stage.replace(/^quitting: /, '') : 'Stopping', tone: '' };
|
||||
if (!s.setup_done) return { text: s.detecting ? 'Detecting cards' : 'Setting up', tone: '' };
|
||||
var m = s.mining || {}, n = s.node || {};
|
||||
// app-ia-26: the state word only; the rate lives on Mine, the reason on the node line and the card rows
|
||||
|
|
|
|||
|
|
@ -63,6 +63,10 @@ test('T-R8: the pill is a state word, the rate lives on Mine', () => {
|
|||
assert.deepEqual(V.pill({ ...base, mining: { state: 'idle', hash_total: 0, cards: [] } }), { text: 'Not mining', tone: '' });
|
||||
assert.deepEqual(V.pill({ ...base, setup_done: false, detecting: true }), { text: 'Detecting cards', tone: '' });
|
||||
assert.deepEqual(V.pill({ ...base, quitting: true }), { text: 'Stopping', tone: '' });
|
||||
// 2.0.2 (PC 2, 21:24 UK): the quit's stage stands on the pill and the big button while the node is being stopped
|
||||
assert.deepEqual(V.pill({ ...base, quitting: true, quit_stage: 'quitting: the node is being stopped' }), { text: 'Stopping · the node is being stopped', tone: '' });
|
||||
assert.equal(V.toggle({ state: 'mining', cards: [] }, { state: 'syncing' }, { quitting: true, quit_stage: 'quitting: the node is being stopped' }).sub, 'quitting: the node is being stopped');
|
||||
assert.equal(V.toggle({ state: 'mining', cards: [] }, { state: 'syncing' }, { quitting: true }).sub, 'miners first, then the node');
|
||||
});
|
||||
// the toggle's sub-line (T-R8's second half) goes with item 3, Mine
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue