diff --git a/app/igneum-app/src/main.rs b/app/igneum-app/src/main.rs index 809453382..12f23b995 100644 --- a/app/igneum-app/src/main.rs +++ b/app/igneum-app/src/main.rs @@ -30,6 +30,7 @@ mod jobrun; mod jobbuild; mod prover; mod provedefault; +mod provingdir; mod segments; mod verifier; mod wslhost; diff --git a/app/igneum-app/src/provingdir.rs b/app/igneum-app/src/provingdir.rs new file mode 100644 index 000000000..c7564d7aa --- /dev/null +++ b/app/igneum-app/src/provingdir.rs @@ -0,0 +1,178 @@ +//! The prover's working directory (`/proving`) owns its disk (0.3.16, 6 October 2026): the fleet's segment +//! exports (50 to 500 MB each with the old [0, last] export, never pruned) filled 60 GB on the hub and 3 to 46 GB on +//! every standing box, and the hub's node died three times on "No space left on device" (the last at 21:42 UK). +//! Rules: a size cap and an age cap on the directory (defaults that fit a 100 GB box for a week), enforced before +//! every export; a segment's directory is deleted the moment its record is submitted or paid; no export starts +//! below 10% free disk, and the skip is logged. +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +/// Defaults: 20 GB and 7 days. `IGNEUM_PROVING_DIR_MAX_GB` and `IGNEUM_PROVING_DIR_MAX_DAYS` change them. +pub const DEFAULT_MAX_BYTES: u64 = 20 * 1024 * 1024 * 1024; +pub const DEFAULT_MAX_AGE: Duration = Duration::from_secs(7 * 24 * 3600); +/// No export below this share of free disk. +pub const MIN_FREE_FRACTION: f64 = 0.10; + +pub fn max_bytes() -> u64 { + std::env::var("IGNEUM_PROVING_DIR_MAX_GB").ok().and_then(|v| v.parse::().ok()).map(|g| g * 1024 * 1024 * 1024).unwrap_or(DEFAULT_MAX_BYTES) +} + +pub fn max_age() -> Duration { + std::env::var("IGNEUM_PROVING_DIR_MAX_DAYS").ok().and_then(|v| v.parse::().ok()).map(|d| Duration::from_secs(d * 24 * 3600)).unwrap_or(DEFAULT_MAX_AGE) +} + +/// One entry of the directory as the planner sees it: a top-level file or a segment directory, its total bytes +/// and its age. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Entry { + pub path: PathBuf, + pub bytes: u64, + pub age: Duration, +} + +/// What to delete so the directory fits: everything older than `max_age`, then the oldest entries until the total +/// is at or under `max_bytes`. Pure, so the cap is unit-tested. +pub fn prune_plan(entries: &[Entry], max_bytes: u64, max_age: Duration) -> Vec { + let mut keep: Vec<&Entry> = Vec::new(); + let mut out: Vec = Vec::new(); + for e in entries { + if e.age > max_age { + out.push(e.path.clone()); + } else { + keep.push(e); + } + } + let mut total: u64 = keep.iter().map(|e| e.bytes).sum(); + // oldest first + keep.sort_by(|a, b| b.age.cmp(&a.age)); + for e in keep { + if total <= max_bytes { + break; + } + total = total.saturating_sub(e.bytes); + out.push(e.path.clone()); + } + out +} + +fn dir_bytes(p: &Path) -> u64 { + let mut total = 0; + if let Ok(rd) = std::fs::read_dir(p) { + for e in rd.flatten() { + let m = match e.metadata() { + Ok(m) => m, + Err(_) => continue, + }; + total += if m.is_dir() { dir_bytes(&e.path()) } else { m.len() }; + } + } + total +} + +/// The directory's entries, oldest by modification time. +pub fn scan(dir: &Path) -> Vec { + let now = SystemTime::now(); + let mut out = Vec::new(); + if let Ok(rd) = std::fs::read_dir(dir) { + for e in rd.flatten() { + let Ok(m) = e.metadata() else { continue }; + let age = m.modified().ok().and_then(|t| now.duration_since(t).ok()).unwrap_or_default(); + let bytes = if m.is_dir() { dir_bytes(&e.path()) } else { m.len() }; + out.push(Entry { path: e.path(), bytes, age }); + } + } + out +} + +/// Enforces the caps on `dir`; returns (entries deleted, bytes freed). +pub fn enforce(dir: &Path) -> (usize, u64) { + let entries = scan(dir); + let plan = prune_plan(&entries, max_bytes(), max_age()); + let mut freed = 0; + for p in &plan { + if let Some(e) = entries.iter().find(|e| &e.path == p) { + freed += e.bytes; + } + let _ = if p.is_dir() { std::fs::remove_dir_all(p) } else { std::fs::remove_file(p) }; + } + (plan.len(), freed) +} + +/// Free disk as a share of the volume `path` is on; None when the platform call fails. +#[cfg(unix)] +pub fn free_fraction(path: &Path) -> Option { + use std::ffi::CString; + use std::os::unix::ffi::OsStrExt; + let c = CString::new(path.as_os_str().as_bytes()).ok()?; + let mut st: libc::statvfs = unsafe { std::mem::zeroed() }; + if unsafe { libc::statvfs(c.as_ptr(), &mut st) } != 0 { + return None; + } + let total = st.f_blocks as f64 * st.f_frsize as f64; + if total <= 0.0 { + return None; + } + Some(st.f_bavail as f64 * st.f_frsize as f64 / total) +} + +#[cfg(windows)] +pub fn free_fraction(path: &Path) -> Option { + use std::os::windows::ffi::OsStrExt; + #[link(name = "kernel32")] + extern "system" { + fn GetDiskFreeSpaceExW(dir: *const u16, avail: *mut u64, total: *mut u64, free: *mut u64) -> i32; + } + let wide: Vec = path.as_os_str().encode_wide().chain(std::iter::once(0)).collect(); + let (mut avail, mut total, mut free) = (0u64, 0u64, 0u64); + if unsafe { GetDiskFreeSpaceExW(wide.as_ptr(), &mut avail, &mut total, &mut free) } == 0 || total == 0 { + return None; + } + Some(avail as f64 / total as f64) +} + +/// The pre-export gate: the caps enforced, then the free-disk check. Err carries the line to log when the export +/// must be skipped. +pub fn before_export(dir: &Path) -> Result<(), String> { + let _ = std::fs::create_dir_all(dir); + let (n, freed) = enforce(dir); + if n > 0 { + eprintln!("prover: proving dir cap: {n} entries deleted, {} MB freed", freed / (1024 * 1024)); + } + match free_fraction(dir) { + Some(f) if f < MIN_FREE_FRACTION => Err(format!("no export: {:.1}% of the disk is free, under the {:.0}% floor; the proving dir is {} MB after its cap; free space or lower IGNEUM_PROVING_DIR_MAX_GB", f * 100.0, MIN_FREE_FRACTION * 100.0, dir_bytes(dir) / (1024 * 1024))), + _ => Ok(()), + } +} + +/// A segment's directory goes the moment its record is submitted or paid. +pub fn remove_segment(dir: &Path, first: u64) { + let _ = std::fs::remove_dir_all(dir.join(format!("seg-{first}"))); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn e(name: &str, mb: u64, days: u64) -> Entry { + Entry { path: PathBuf::from(name), bytes: mb * 1024 * 1024, age: Duration::from_secs(days * 24 * 3600) } + } + + #[test] + fn the_cap_deletes_the_old_then_the_oldest_until_it_fits() { + let entries = vec![e("seg-1", 400, 9), e("seg-2", 300, 3), e("seg-3", 300, 2), e("seg-4", 200, 1), e("block-5.json", 1, 0)]; + // the age cap takes seg-1; the size cap (600 MB) then takes seg-2 (oldest kept), leaving 501 MB + let plan = prune_plan(&entries, 600 * 1024 * 1024, Duration::from_secs(7 * 24 * 3600)); + assert_eq!(plan, vec![PathBuf::from("seg-1"), PathBuf::from("seg-2")]); + // under both caps: nothing + assert!(prune_plan(&entries[1..], 2 * 1024 * 1024 * 1024, Duration::from_secs(30 * 24 * 3600)).is_empty()); + // a 100 GB box for a week: the default caps leave 80 GB to the node and the system + assert_eq!(DEFAULT_MAX_BYTES, 20 * 1024 * 1024 * 1024); + assert_eq!(DEFAULT_MAX_AGE, Duration::from_secs(7 * 24 * 3600)); + } + + #[test] + fn the_free_check_answers_on_this_machine() { + let f = free_fraction(Path::new(".")).expect("statvfs"); + assert!((0.0..=1.0).contains(&f)); + } +}