Prover service: the host and the exporter are polled children, killed when the app quits, proving is switched off or the limit passes; their output goes to per-run log files
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
parent
2278714a09
commit
b338db3fac
1 changed files with 36 additions and 10 deletions
|
|
@ -170,8 +170,10 @@ fn find_tools(bin_dir: &Path) -> Result<Tools, String> {
|
|||
}
|
||||
}
|
||||
|
||||
/// Runs the host or the exporter: directly, or through WSL on Windows. Returns (exit ok, output).
|
||||
fn run_tool(t: &Tools, exe: &Path, args: &[String], env: &[(&str, &str)], limit: Duration) -> (bool, String) {
|
||||
/// Runs the host or the exporter: directly, or through WSL on Windows. Output goes to `log`; the child is polled
|
||||
/// every second and killed when the app quits, the setting goes off or `limit` passes (a proof must never outlive
|
||||
/// the app). Returns (exit ok, output).
|
||||
fn run_tool(shared: &Shared, t: &Tools, exe: &Path, args: &[String], env: &[(&str, &str)], limit: Duration, log: &Path) -> (bool, String) {
|
||||
let mut cmd = if t.wsl {
|
||||
let mut c = Command::new(crate::platform::tool("wsl"));
|
||||
let envs: String = env.iter().map(|(k, v)| format!("{k}={v} ")).collect();
|
||||
|
|
@ -187,13 +189,38 @@ fn run_tool(t: &Tools, exe: &Path, args: &[String], env: &[(&str, &str)], limit:
|
|||
c
|
||||
};
|
||||
crate::platform::quiet(&mut cmd);
|
||||
let Ok(file) = std::fs::File::create(log) else { return (false, format!("cannot write {}", log.display())) };
|
||||
let Ok(err) = file.try_clone() else { return (false, "cannot clone the log handle".into()) };
|
||||
cmd.stdin(std::process::Stdio::null()).stdout(file).stderr(err);
|
||||
let mut child = match cmd.spawn() {
|
||||
Ok(c) => c,
|
||||
Err(e) => return (false, format!("{}: {e}", exe.display())),
|
||||
};
|
||||
let started = Instant::now();
|
||||
match crate::detect::run_timeout(&mut cmd, None, limit) {
|
||||
Some(out) => {
|
||||
let ok = out.lines().any(|l| l.starts_with("results written to") || l.contains("fixture written") || l.starts_with("wrote ")) || !out.contains("Error") && started.elapsed() < limit;
|
||||
(ok, out)
|
||||
let status = loop {
|
||||
match child.try_wait() {
|
||||
Ok(Some(st)) => break Some(st),
|
||||
Ok(None) => {}
|
||||
Err(e) => {
|
||||
let _ = child.kill();
|
||||
return (false, format!("{}: {e}", exe.display()));
|
||||
}
|
||||
}
|
||||
None => (false, format!("{} did not finish in {} s", exe.display(), limit.as_secs())),
|
||||
let stop = {
|
||||
let st = shared.state.lock().unwrap();
|
||||
st.quitting
|
||||
} || !shared.settings.lock().unwrap().prove;
|
||||
if stop || started.elapsed() > limit {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
break None;
|
||||
}
|
||||
std::thread::sleep(Duration::from_secs(1));
|
||||
};
|
||||
let out = std::fs::read_to_string(log).unwrap_or_default();
|
||||
match status {
|
||||
Some(st) => (st.success(), out),
|
||||
None => (false, format!("{} stopped after {} s (quit, proving switched off, or the {} s limit)", exe.display(), started.elapsed().as_secs(), limit.as_secs())),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -359,14 +386,13 @@ fn loop_forever(shared: Arc<Shared>, bin_dir: PathBuf) {
|
|||
let export = evm_rpc(&shared, "igneum_exportSegments", json!(["0x0", format!("{:#x}", w.number)]), Duration::from_secs(120))?;
|
||||
std::fs::write(&seq, export.to_string()).map_err(|e| e.to_string())?;
|
||||
let (seq_p, fix_p, res_p) = if t.wsl { (wsl_path(&seq), wsl_path(&fixture), wsl_path(&results)) } else { (seq.display().to_string(), fixture.display().to_string(), results.display().to_string()) };
|
||||
let (ok, out) = run_tool(t, &t.export, &[seq_p, w.number.to_string(), fix_p.clone()], &[], Duration::from_secs(600));
|
||||
let (ok, out) = run_tool(&shared, t, &t.export, &[seq_p, w.number.to_string(), fix_p.clone()], &[], Duration::from_secs(600), &dir.join(format!("export-{}.log", w.number)));
|
||||
if !ok || !fixture.exists() {
|
||||
return Err(format!("exporter: {}", out.lines().rev().find(|l| !l.trim().is_empty()).unwrap_or("failed")));
|
||||
}
|
||||
set(&shared, |p| p.message = if t.cuda { "proving on the GPU".into() } else { "proving on the CPU (slow)".into() });
|
||||
let prover_env = if t.cuda { "cuda" } else { "cpu" };
|
||||
let (ok, out) = run_tool(t, &t.host, &[fix_p, "--mode".into(), "compressed".into(), "--shard".into(), w.shard.to_string(), "--prover".into(), payout.clone(), "--out".into(), res_p], &[("SP1_PROVER", prover_env), ("RUST_LOG", "off")], Duration::from_secs(3 * 3600));
|
||||
let _ = std::fs::write(dir.join(format!("prove-{}-{}.log", w.number, w.shard)), &out);
|
||||
let (ok, out) = run_tool(&shared, t, &t.host, &[fix_p, "--mode".into(), "compressed".into(), "--shard".into(), w.shard.to_string(), "--prover".into(), payout.clone(), "--out".into(), res_p], &[("SP1_PROVER", prover_env), ("RUST_LOG", "off")], Duration::from_secs(3 * 3600), &dir.join(format!("prove-{}-{}.log", w.number, w.shard)));
|
||||
if !ok || !results.exists() {
|
||||
return Err(format!("prover: {}", out.lines().rev().find(|l| l.contains("RESULT") || l.contains("rror")).unwrap_or("failed")));
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue