prover: the proving directory owns its disk (size and age caps, delete on submit, a free-disk floor before each export)

The fleet's segment exports (50 to 500 MB each with the old [0, last] export, never pruned) filled 42 GB on the hub
(586 segment dirs, 60 GB disk, 100 percent) and 3 to 46 GB on every standing box between about 13:00Z and 20:44Z on
6 October 2026, 4 to 6 GB an hour a box; the hub's node died on the full disk at 20:42:31Z (its two earlier deaths
at 19:58:08Z and 20:01:55Z were the sync KeyNotFound). The app's prover now:
- enforces a cap on <app dir>/proving before every export: entries older than IGNEUM_PROVING_DIR_MAX_DAYS (7) go,
  then the oldest until the total is under IGNEUM_PROVING_DIR_MAX_GB (20); the plan is a pure function with a
  unit test (provingdir::prune_plan); the defaults leave 80 GB of a 100 GB box to the node and the system
- skips the export with a logged line when the disk is under 10 percent free (statvfs on unix, GetDiskFreeSpaceExW
  on Windows), and the Prove page says so
- deletes a segment's directory (fixtures, proofs, logs) the moment its record is submitted, on the first try or on
  a retry; seq.json was already removed after the cut
The fleet's kit (box-prover.py, gpu-fleet 86c9854) carries the same rule on the boxes since 20:5xZ.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

app/igneum-app/src/main.rs

@@ -30,6 +30,7 @@ mod jobrun;
 mod jobbuild;
 mod prover;
 mod provedefault;
+mod provingdir;
 mod segments;
 mod verifier;
 mod wslhost;

app/igneum-app/src/provingdir.rs

@@ -0,0 +1,178 @@
+//! The prover's working directory (`<app dir>/proving`) owns its disk (0.3.16, 6 October 2026): the fleet's segment
+//! exports (50 to 500 MB each with the old [0, last] export, never pruned) filled 60 GB on the hub and 3 to 46 GB on
+//! every standing box, and the hub's node died three times on "No space left on device" (the last at 21:42 UK).
+//! Rules: a size cap and an age cap on the directory (defaults that fit a 100 GB box for a week), enforced before
+//! every export; a segment's directory is deleted the moment its record is submitted or paid; no export starts
+//! below 10% free disk, and the skip is logged.
+use std::path::{Path, PathBuf};
+use std::time::{Duration, SystemTime};
+
+/// Defaults: 20 GB and 7 days. `IGNEUM_PROVING_DIR_MAX_GB` and `IGNEUM_PROVING_DIR_MAX_DAYS` change them.
+pub const DEFAULT_MAX_BYTES: u64 = 20 * 1024 * 1024 * 1024;
+pub const DEFAULT_MAX_AGE: Duration = Duration::from_secs(7 * 24 * 3600);
+/// No export below this share of free disk.
+pub const MIN_FREE_FRACTION: f64 = 0.10;
+
+pub fn max_bytes() -> u64 {
+    std::env::var("IGNEUM_PROVING_DIR_MAX_GB").ok().and_then(|v| v.parse::<u64>().ok()).map(|g| g * 1024 * 1024 * 1024).unwrap_or(DEFAULT_MAX_BYTES)
+}
+
+pub fn max_age() -> Duration {
+    std::env::var("IGNEUM_PROVING_DIR_MAX_DAYS").ok().and_then(|v| v.parse::<u64>().ok()).map(|d| Duration::from_secs(d * 24 * 3600)).unwrap_or(DEFAULT_MAX_AGE)
+}
+
+/// One entry of the directory as the planner sees it: a top-level file or a segment directory, its total bytes
+/// and its age.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Entry {
+    pub path: PathBuf,
+    pub bytes: u64,
+    pub age: Duration,
+}
+
+/// What to delete so the directory fits: everything older than `max_age`, then the oldest entries until the total
+/// is at or under `max_bytes`. Pure, so the cap is unit-tested.
+pub fn prune_plan(entries: &[Entry], max_bytes: u64, max_age: Duration) -> Vec<PathBuf> {
+    let mut keep: Vec<&Entry> = Vec::new();
+    let mut out: Vec<PathBuf> = Vec::new();
+    for e in entries {
+        if e.age > max_age {
+            out.push(e.path.clone());
+        } else {
+            keep.push(e);
+        }
+    }
+    let mut total: u64 = keep.iter().map(|e| e.bytes).sum();
+    // oldest first
+    keep.sort_by(|a, b| b.age.cmp(&a.age));
+    for e in keep {
+        if total <= max_bytes {
+            break;
+        }
+        total = total.saturating_sub(e.bytes);
+        out.push(e.path.clone());
+    }
+    out
+}
+
+fn dir_bytes(p: &Path) -> u64 {
+    let mut total = 0;
+    if let Ok(rd) = std::fs::read_dir(p) {
+        for e in rd.flatten() {
+            let m = match e.metadata() {
+                Ok(m) => m,
+                Err(_) => continue,
+            };
+            total += if m.is_dir() { dir_bytes(&e.path()) } else { m.len() };
+        }
+    }
+    total
+}
+
+/// The directory's entries, oldest by modification time.
+pub fn scan(dir: &Path) -> Vec<Entry> {
+    let now = SystemTime::now();
+    let mut out = Vec::new();
+    if let Ok(rd) = std::fs::read_dir(dir) {
+        for e in rd.flatten() {
+            let Ok(m) = e.metadata() else { continue };
+            let age = m.modified().ok().and_then(|t| now.duration_since(t).ok()).unwrap_or_default();
+            let bytes = if m.is_dir() { dir_bytes(&e.path()) } else { m.len() };
+            out.push(Entry { path: e.path(), bytes, age });
+        }
+    }
+    out
+}
+
+/// Enforces the caps on `dir`; returns (entries deleted, bytes freed).
+pub fn enforce(dir: &Path) -> (usize, u64) {
+    let entries = scan(dir);
+    let plan = prune_plan(&entries, max_bytes(), max_age());
+    let mut freed = 0;
+    for p in &plan {
+        if let Some(e) = entries.iter().find(|e| &e.path == p) {
+            freed += e.bytes;
+        }
+        let _ = if p.is_dir() { std::fs::remove_dir_all(p) } else { std::fs::remove_file(p) };
+    }
+    (plan.len(), freed)
+}
+
+/// Free disk as a share of the volume `path` is on; None when the platform call fails.
+#[cfg(unix)]
+pub fn free_fraction(path: &Path) -> Option<f64> {
+    use std::ffi::CString;
+    use std::os::unix::ffi::OsStrExt;
+    let c = CString::new(path.as_os_str().as_bytes()).ok()?;
+    let mut st: libc::statvfs = unsafe { std::mem::zeroed() };
+    if unsafe { libc::statvfs(c.as_ptr(), &mut st) } != 0 {
+        return None;
+    }
+    let total = st.f_blocks as f64 * st.f_frsize as f64;
+    if total <= 0.0 {
+        return None;
+    }
+    Some(st.f_bavail as f64 * st.f_frsize as f64 / total)
+}
+
+#[cfg(windows)]
+pub fn free_fraction(path: &Path) -> Option<f64> {
+    use std::os::windows::ffi::OsStrExt;
+    #[link(name = "kernel32")]
+    extern "system" {
+        fn GetDiskFreeSpaceExW(dir: *const u16, avail: *mut u64, total: *mut u64, free: *mut u64) -> i32;
+    }
+    let wide: Vec<u16> = path.as_os_str().encode_wide().chain(std::iter::once(0)).collect();
+    let (mut avail, mut total, mut free) = (0u64, 0u64, 0u64);
+    if unsafe { GetDiskFreeSpaceExW(wide.as_ptr(), &mut avail, &mut total, &mut free) } == 0 || total == 0 {
+        return None;
+    }
+    Some(avail as f64 / total as f64)
+}
+
+/// The pre-export gate: the caps enforced, then the free-disk check. Err carries the line to log when the export
+/// must be skipped.
+pub fn before_export(dir: &Path) -> Result<(), String> {
+    let _ = std::fs::create_dir_all(dir);
+    let (n, freed) = enforce(dir);
+    if n > 0 {
+        eprintln!("prover: proving dir cap: {n} entries deleted, {} MB freed", freed / (1024 * 1024));
+    }
+    match free_fraction(dir) {
+        Some(f) if f < MIN_FREE_FRACTION => Err(format!("no export: {:.1}% of the disk is free, under the {:.0}% floor; the proving dir is {} MB after its cap; free space or lower IGNEUM_PROVING_DIR_MAX_GB", f * 100.0, MIN_FREE_FRACTION * 100.0, dir_bytes(dir) / (1024 * 1024))),
+        _ => Ok(()),
+    }
+}
+
+/// A segment's directory goes the moment its record is submitted or paid.
+pub fn remove_segment(dir: &Path, first: u64) {
+    let _ = std::fs::remove_dir_all(dir.join(format!("seg-{first}")));
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn e(name: &str, mb: u64, days: u64) -> Entry {
+        Entry { path: PathBuf::from(name), bytes: mb * 1024 * 1024, age: Duration::from_secs(days * 24 * 3600) }
+    }
+
+    #[test]
+    fn the_cap_deletes_the_old_then_the_oldest_until_it_fits() {
+        let entries = vec![e("seg-1", 400, 9), e("seg-2", 300, 3), e("seg-3", 300, 2), e("seg-4", 200, 1), e("block-5.json", 1, 0)];
+        // the age cap takes seg-1; the size cap (600 MB) then takes seg-2 (oldest kept), leaving 501 MB
+        let plan = prune_plan(&entries, 600 * 1024 * 1024, Duration::from_secs(7 * 24 * 3600));
+        assert_eq!(plan, vec![PathBuf::from("seg-1"), PathBuf::from("seg-2")]);
+        // under both caps: nothing
+        assert!(prune_plan(&entries[1..], 2 * 1024 * 1024 * 1024, Duration::from_secs(30 * 24 * 3600)).is_empty());
+        // a 100 GB box for a week: the default caps leave 80 GB to the node and the system
+        assert_eq!(DEFAULT_MAX_BYTES, 20 * 1024 * 1024 * 1024);
+        assert_eq!(DEFAULT_MAX_AGE, Duration::from_secs(7 * 24 * 3600));
+    }
+
+    #[test]
+    fn the_free_check_answers_on_this_machine() {
+        let f = free_fraction(Path::new(".")).expect("statvfs");
+        assert!((0.0..=1.0).contains(&f));
+    }
+}