Igneum Miner jobs: a collect job fails when its command exits non-zero (collect_done, unit test); collect-pc1-board3 had reported done (exit 0) over command exit Some(1)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs 2026-10-04 19:18:13 +00:00
parent f9ed07807e
commit 5ccafcf4d2

View file

@ -1009,6 +1009,7 @@ fn run_collect(shared: &Arc<Shared>, job: &Job, sink: &Sink, data_root: &Path, c
}
}
let command = job.str_param("command");
let mut ran: Option<Ran> = None;
if !command.trim().is_empty() {
sink.stage(&format!("running: {}", short(&command, 120)));
let mut cmd = if cfg!(windows) {
@ -1022,11 +1023,55 @@ fn run_collect(shared: &Arc<Shared>, job: &Job, sink: &Sink, data_root: &Path, c
};
cmd.current_dir(data_root);
job_env(&mut cmd, shared, job, &sink.dir, data_root);
let ran = run_streamed(&mut cmd, sink, ctl, Duration::from_secs(600), shared, job, crate::platform::unix_now(), "collect command running")?;
sink.line(&format!("command exit {:?}", ran.code));
let r = run_streamed(&mut cmd, sink, ctl, Duration::from_secs(600), shared, job, crate::platform::unix_now(), "collect command running")?;
sink.line(&format!("command exit {:?}", r.code));
ran = Some(r);
}
Ok(collect_done(uploaded, failed, names, ran))
}
/// The outcome of a collect job from its parts: the upload counts and, when a command ran, how it ended. A command
/// that exits non-zero fails the job and its code is the job's exit; one ended by the cap is a timeout; one ended
/// otherwise is failed (4 October 2026, collect-pc1-board3: PowerShell could not parse the command, "command exit
/// Some(1)" was in the report and the job still said "done (exit 0)" because only the upload count was judged).
fn collect_done(uploaded: u32, failed: u32, names: Vec<String>, ran: Option<Ran>) -> Done {
let files = format!("{uploaded} file{} uploaded{}", if uploaded == 1 { "" } else { "s" }, if failed > 0 { format!(", {failed} failed") } else { String::new() });
let extra = json!({ "uploaded_files": names });
let by_files = || (if failed > 0 { "failed" } else { "done" }.to_string(), failed as i64);
match ran {
None => { let (status, exit) = by_files(); Done { status, exit, summary: files, extra } }
Some(Ran { code: Some(0), .. }) => { let (status, exit) = by_files(); Done { status, exit, summary: format!("{files}, command exit 0, output in the report"), extra } }
Some(Ran { code: Some(c), .. }) => Done { status: "failed".into(), exit: c as i64, summary: format!("{files}, command exited with code {c}, output in the report"), extra },
Some(Ran { code: None, timed_out: true }) => Done { status: "timeout".into(), exit: -1, summary: format!("{files}, command hit the time cap and was ended"), extra },
Some(Ran { code: None, timed_out: false }) => Done { status: "failed".into(), exit: -1, summary: format!("{files}, command was ended"), extra },
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn collect_outcome_follows_the_command_exit() {
let d = collect_done(0, 0, vec![], None);
assert_eq!((d.status.as_str(), d.exit, d.summary.as_str()), ("done", 0, "0 files uploaded"));
let d = collect_done(2, 0, vec!["a".into(), "b".into()], Some(Ran { code: Some(0), timed_out: false }));
assert_eq!((d.status.as_str(), d.exit), ("done", 0));
assert_eq!(d.summary, "2 files uploaded, command exit 0, output in the report");
assert_eq!(d.extra["uploaded_files"].as_array().map(|a| a.len()), Some(2));
// the board3 case: the command failed, the job must not say done
let d = collect_done(0, 0, vec![], Some(Ran { code: Some(1), timed_out: false }));
assert_eq!((d.status.as_str(), d.exit), ("failed", 1));
assert_eq!(d.summary, "0 files uploaded, command exited with code 1, output in the report");
// a failed upload fails the job even when the command was fine
let d = collect_done(1, 2, vec!["a".into()], Some(Ran { code: Some(0), timed_out: false }));
assert_eq!((d.status.as_str(), d.exit), ("failed", 2));
assert!(d.summary.starts_with("1 file uploaded, 2 failed, command exit 0"));
let d = collect_done(0, 0, vec![], Some(Ran { code: None, timed_out: true }));
assert_eq!((d.status.as_str(), d.exit), ("timeout", -1));
let d = collect_done(0, 0, vec![], Some(Ran { code: None, timed_out: false }));
assert_eq!((d.status.as_str(), d.exit), ("failed", -1));
}
let summary = format!("{uploaded} file{} uploaded{}{}", if uploaded == 1 { "" } else { "s" }, if failed > 0 { format!(", {failed} failed") } else { String::new() }, if command.trim().is_empty() { String::new() } else { ", command output in the report".into() });
Ok(Done { status: if failed > 0 { "failed".into() } else { "done".into() }, exit: failed as i64, summary, extra: json!({ "uploaded_files": names }) })
}
// ---- kind: shard-benchmark ----------------------------------------------------------------------------------------------