igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

Igneum Miner jobs: WSL as root by default, the shipped wsl2\bin prover run fixture by fixture (no cargo, no setup), account context logged at every job start, dashboard note when the app runs elevated or as another account

Browse source 
PC 2 on 0.3.3: 'wsl -d Ubuntu-24.04 -u [user]' from the app fails with getpwnam([user]) and the default user has no
cargo, while the project lead's own session has both in a distro of the same name: WSL distros belong to the Windows account,
and the engine runs under a different context than the interactive session. So the prover path is self-sufficient
inside the Ubuntu the app sees: the wsl-prover probe and the shard-benchmark job run as root (the job's wsl_user or
IGNEUM_APP_WSL_USER first, root second, the distro default last), and the shard job runs the Linux host the
payload ships next to the app (wsl2\bin\igneum-prove-host, cuda feature) directly for each fixture (shard 0 in
shard mode, the blocks in block mode, results under <app data>\prove\igneum-prove-wsl2\results); params.build
= true keeps the package's prove-shard.sh path. Every job logs the account context first (Windows user, SID,
elevated, the signed-in console user, then 'wsl user <id> uid <n> home <h>' and nvidia-smi inside the distro),
the engine logs it once at start, and the dashboard (Settings and the job strip) says 'The app runs as X
(elevated). The signed-in user is Y; WSL and its tools belong to that account.' when they differ.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs 2026-10-04 17:14:00 +00:00
parent b0c1e1a3d6
commit e52d74988f
3 changed files with 119 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

132
app/igneum-app/src/jobrun.rs
View file

@ -32,6 +32,9 @@ const PROGRESS_REPORT_EVERY_S: u64 = 300;
const GPU_IDLE_PCT: f64 = 5.0;
const GPU_IDLE_WAIT_S: u64 = 180;
const DEFAULT_DISTRO: &str = "Ubuntu-24.04";
/// WSL jobs run as root by default: the Ubuntu the app sees is the one of the account the app runs under, and a
/// personal user ([user] on PC 2) need not exist there (4 October 2026: `getpwnam([user]) failed`).
const DEFAULT_WSL_USER: &str = "root";
const DEFAULT_FIXTURES: &[&str] = &["block-338-shard1", "block-341-shards2", "block-344-shards4"];
const HISTORY_SHOWN: usize = 20;
@ -141,6 +144,16 @@ impl Jobs {
fingerprint: manifest::fingerprint(manifest::OTA_PUBLIC_KEY_HEX),
};
j.publish(shared);
let sh = shared.clone();
std::thread::spawn(move || {
let ctx = account_context();
sh.log(&format!("account: {ctx}"));
let w = account_warning(&ctx);
if !w.is_empty() {
sh.log(&format!("account: {w}"));
sh.state.lock().unwrap().jobs.account = w;
}
});
j
}
@ -445,6 +458,12 @@ impl Jobs {
std::thread::spawn(move || {
let sink = Sink::new(&shared2, &job, &dir);
sink.line(&format!("job {} ({}) on {} machine {} run {run_id}, started {}", job.id, job.kind, shared2.runtime.host, shared2.runtime.machine_id, jobs::format_time(started)));
let ctx = account_context();
sink.line(&format!("account: {ctx}"));
let warn = account_warning(&ctx);
if !warn.is_empty() {
sink.line(&format!("account: {warn}"));
}
let r = match job.kind.as_str() {
"run" => run_script(&shared2, &job, &sink, &dir, &data_root, &ctl, started),
"fetch" => run_fetch(&job, &sink, &dir, &data_root, &shared2.runtime.app_dir, &ctl),
@ -616,12 +635,13 @@ fn probe(req: &str, wsl_user: &str) -> Result<String, String> {
if !cfg!(windows) {
return Err("not Windows".into());
}
// the configured user first (the job's wsl_user or IGNEUM_APP_WSL_USER), then the distro's default user
// the configured user first (the job's wsl_user or IGNEUM_APP_WSL_USER), then root, then the distro's default user
let mut tried = Vec::new();
let mut users: Vec<&str> = Vec::new();
if !wsl_user.is_empty() {
if !wsl_user.is_empty() && wsl_user != DEFAULT_WSL_USER {
users.push(wsl_user);
}
users.push(DEFAULT_WSL_USER);
users.push("");
for u in users {
let mut c = Command::new(&wsl);
@ -1051,7 +1071,7 @@ fn wait_gpu_idle(sink: &Sink, ctl: &Ctl) {
fn run_shard_benchmark(shared: &Arc<Shared>, job: &Job, sink: &Sink, data_root: &Path, jobs_url: &str, ctl: &Ctl, started: u64) -> Result<Done, String> {
let distro = { let d = job.str_param("distro"); if d.is_empty() { DEFAULT_DISTRO.to_string() } else { d } };
let user = job.str_param("wsl_user");
let user = { let u = job.str_param("wsl_user"); if !u.is_empty() { u } else { std::env::var("IGNEUM_APP_WSL_USER").ok().filter(|v| !v.is_empty()).unwrap_or_else(|| DEFAULT_WSL_USER.to_string()) } };
let mut fixtures = job.list_param("fixtures");
if fixtures.is_empty() {
fixtures = DEFAULT_FIXTURES.iter().map(|s| s.to_string()).collect();
@ -1083,22 +1103,57 @@ fn run_shard_benchmark(shared: &Arc<Shared>, job: &Job, sink: &Sink, data_root:
if !script.is_file() {
return Err(format!("{} is missing after the extract", script.display()));
}
let wsl_script = jobs::to_wsl_path(&script.display().to_string()).ok_or("the package path has no drive letter; WSL cannot see it")?;
let shard = fixtures[0].clone();
let blocks = fixtures[1..].join(" ");
sink.stage(&format!("proving {shard} then {} inside {distro} (cap {cap_min} min)", if blocks.is_empty() { "nothing else".to_string() } else { blocks.clone() }));
let mut cmd = Command::new(crate::platform::tool("wsl"));
cmd.args(["-d", &distro]);
if !user.is_empty() {
cmd.args(["-u", &user]);
}
cmd.args(["--", "bash", &wsl_script, &shard, &blocks]);
cmd.current_dir(&pkg);
let remaining = overall.saturating_duration_since(Instant::now()).max(Duration::from_secs(60));
let ran = run_streamed(&mut cmd, sink, ctl, remaining, shared, job, started, "shard benchmark running")?;
// what this run sees inside WSL: the account's own Ubuntu, so say who we are there
let (ccode, cout) = run_capture(Command::new(crate::platform::tool("wsl")).args(["-d", &distro, "-u", &user, "--", "bash", "-c", "echo \"wsl user $(id -un) uid $(id -u) home $HOME\"; nvidia-smi -L 2>&1 | head -1; ls -d \"$HOME/.sp1\" \"$HOME/igneum-prove\" 2>&1 | head -2"]), Duration::from_secs(60));
sink.line(&format!("wsl context (-u {user}): exit {ccode:?}: {}", short_out(&cout)));
// the payload ships the Linux host next to the app (wsl2\bin): no cargo, no toolchain, run it fixture by fixture;
// params.build = true forces the package's prove-shard.sh (cargo build inside the distro) instead
let shipped = std::env::current_exe().ok().and_then(|p| p.parent().map(|d| d.join("wsl2").join("bin").join("igneum-prove-host"))).filter(|p| p.is_file());
let (ran, what) = match shipped {
Some(host) if !job.bool_param("build") => {
let host_wsl = jobs::to_wsl_path(&host.display().to_string()).ok_or("the shipped prover path has no drive letter")?;
let fixdir = { let a = pkg.join("package").join("proving").join("fixtures"); if a.is_dir() { a } else { host.parent().and_then(|b| b.parent()).map(|w| w.join("fixtures")).unwrap_or(a) } };
let fixdir_wsl = jobs::to_wsl_path(&fixdir.display().to_string()).ok_or("the fixtures path has no drive letter")?;
let results = pkg.join("results");
let _ = std::fs::create_dir_all(&results);
let results_wsl = jobs::to_wsl_path(&results.display().to_string()).ok_or("the results path has no drive letter")?;
sink.stage(&format!("proving with the shipped prover as {user}: {shard} (shard 0: execute, core, compressed), then {} (cap {cap_min} min)", if blocks.is_empty() { "nothing else".to_string() } else { format!("block mode on {blocks}") }));
let mut last = Ran { code: Some(0), timed_out: false };
for (i, f) in fixtures.iter().enumerate() {
let left = overall.saturating_duration_since(Instant::now());
if left < Duration::from_secs(60) {
sink.line(&format!("time cap reached before {f}"));
last = Ran { code: None, timed_out: true };
break;
}
let mode = if i == 0 { "shard --shard 0" } else { "block" };
let line = format!("export PATH=\"/usr/local/cuda/bin:$PATH\"; CUDA_DIR=$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1); export LD_LIBRARY_PATH=\"/usr/lib/wsl/lib:${{CUDA_DIR:+$CUDA_DIR/lib64:}}${{LD_LIBRARY_PATH:-}}\"; echo \"=== GPU run: {f} --mode {mode} (SP1_PROVER=cuda) ===\"; SP1_PROVER=cuda RUST_LOG=info '{host_wsl}' '{fixdir_wsl}/{f}.json' --mode {mode} --out '{results_wsl}/{f}-cuda-{started}.json'; rc=$?; echo \"run exit $rc at $(date -u +%FT%TZ)\"; exit $rc");
let mut cmd = Command::new(crate::platform::tool("wsl"));
cmd.args(["-d", &distro, "-u", &user, "--", "bash", "-c", &line]);
cmd.current_dir(&pkg);
let r = run_streamed(&mut cmd, sink, ctl, left, shared, job, started, "shard benchmark running")?;
if r.code != Some(0) {
last = r;
break;
}
}
(last, "shipped prover")
}
_ => {
let wsl_script = jobs::to_wsl_path(&script.display().to_string()).ok_or("the package path has no drive letter; WSL cannot see it")?;
sink.stage(&format!("proving with prove-shard.sh as {user} (cargo build inside {distro}): {shard} then {} (cap {cap_min} min)", if blocks.is_empty() { "nothing else".to_string() } else { blocks.clone() }));
let mut cmd = Command::new(crate::platform::tool("wsl"));
cmd.args(["-d", &distro, "-u", &user, "--", "bash", &wsl_script, &shard, &blocks]);
cmd.current_dir(&pkg);
let remaining = overall.saturating_duration_since(Instant::now()).max(Duration::from_secs(60));
(run_streamed(&mut cmd, sink, ctl, remaining, shared, job, started, "shard benchmark running")?, "prove-shard.sh")
}
};
if ran.code.is_none() {
// the Linux side outlives wsl.exe: end the prover there too
let _ = run_capture(Command::new(crate::platform::tool("wsl")).args(["-d", &distro, "--", "bash", "-c", "pkill -f igneum-prove-host; pkill -f prove-shard.sh; true"]), Duration::from_secs(30));
let _ = run_capture(Command::new(crate::platform::tool("wsl")).args(["-d", &distro, "-u", &user, "--", "bash", "-c", "pkill -f igneum-prove-host; pkill -f prove-shard.sh; true"]), Duration::from_secs(30));
}
sink.stage("uploading the results");
let mut uploaded = Vec::new();
@ -1120,9 +1175,9 @@ fn run_shard_benchmark(shared: &Arc<Shared>, job: &Job, sink: &Sink, data_root:
}
let results = sink.results.lock().unwrap().clone();
let n_results = results.iter().filter(|r| r.starts_with("RESULT")).count();
let mut done = finish_ran(ran, "prove-shard.sh")?;
let mut done = finish_ran(ran, what)?;
done.summary = format!("{}; {n_results} RESULT line{}, {} result file{} uploaded", done.summary, if n_results == 1 { "" } else { "s" }, uploaded.len(), if uploaded.len() == 1 { "" } else { "s" });
done.extra = json!({ "fixtures": fixtures, "distro": distro, "result_files": uploaded, "package": pkg.display().to_string() });
done.extra = json!({ "fixtures": fixtures, "distro": distro, "wsl_user": user, "prover": what, "result_files": uploaded, "package": pkg.display().to_string() });
Ok(done)
}
@ -1166,6 +1221,49 @@ fn spawn_relaunch_helper(shared: &Arc<Shared>) -> Result<(), String> {
c.spawn().map(|_| ()).map_err(|e| e.to_string())
}
static ACCOUNT: std::sync::OnceLock<String> = std::sync::OnceLock::new();
/// Who the engine runs as: Windows "user=<domain\\name> sid=<S-1-...> elevated=<True|False> console=<the signed-in
/// user>" from one PowerShell call (cached), "user=<name>" elsewhere. WSL distros belong to the Windows account, so
/// this decides which Ubuntu a job sees (PC 2, 4 October 2026).
fn account_context() -> String {
ACCOUNT
.get_or_init(|| {
#[cfg(windows)]
{
let ps = "$i = [Security.Principal.WindowsIdentity]::GetCurrent(); $p = New-Object Security.Principal.WindowsPrincipal($i); $e = $p.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator); $c = ''; try { $c = (Get-CimInstance Win32_ComputerSystem).UserName } catch { }; Write-Output ('user=' + $i.Name + ' sid=' + $i.User.Value + ' elevated=' + $e + ' console=' + $c)";
let (code, out) = run_capture(Command::new(crate::platform::tool("powershell")).args(["-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", ps]), Duration::from_secs(40));
let t = short_out(&out);
if code == Some(0) && t.contains("user=") { t } else { format!("user={} (powershell exit {code:?}: {t})", std::env::var("USERNAME").unwrap_or_default()) }
}
#[cfg(not(windows))]
{
format!("user={}", std::env::var("USER").unwrap_or_default())
}
})
.clone()
}
/// The dashboard note when the app's account is elevated or not the signed-in user's; empty otherwise.
fn account_warning(ctx: &str) -> String {
let get = |k: &str| ctx.split_whitespace().find_map(|p| p.strip_prefix(&format!("{k}="))).unwrap_or("").to_string();
let user = get("user");
let console = get("console");
let elevated = get("elevated").eq_ignore_ascii_case("true");
let other = !console.is_empty() && !user.is_empty() && !user.eq_ignore_ascii_case(&console);
if !elevated && !other {
return String::new();
}
let mut s = format!("The app runs as {user}{}.", if elevated { " (elevated)" } else { "" });
if other {
s.push_str(&format!(" The signed-in user is {console}; WSL and its tools belong to that account."));
} else {
s.push_str(" WSL tools set up without elevation belong to the signed-in user.");
}
s.push_str(" Jobs run WSL as root inside the Ubuntu this account sees.");
s
}
fn short(s: &str, n: usize) -> String {
if s.chars().count() <= n { s.to_string() } else { format!("{}...", s.chars().take(n).collect::<String>()) }
}

2
app/igneum-app/src/state.rs
View file

@ -184,6 +184,8 @@ pub struct JobsState {
pub checked_at: f64,
pub error: String,
pub queued: u32,
/// set when the app runs elevated or under another account than the signed-in user (WSL belongs to the account)
pub account: String,
pub active: bool,
pub id: String,
pub kind: String,

3
app/igneum-app/ui/app.js
View file

@ -486,7 +486,7 @@
function jobLine(j, now) {
if (j.active) {
var m = Math.max(0, Math.floor((now - j.started_at) / 60));
return { text: 'Job: ' + jobTitle(j) + ' running, ' + m + ' min' + (j.stage ? '. ' + cap(j.stage) + '.' : '.') + (j.message ? ' ' + j.message : ''), results: j.results || [] };
return { text: 'Job: ' + jobTitle(j) + ' running, ' + m + ' min' + (j.stage ? '. ' + cap(j.stage) + '.' : '.') + (j.message ? ' ' + j.message : '') + (j.account ? ' ' + j.account : ''), results: j.results || [] };
}
var l = j.last;
if (!l || !l.id) return null;
@ -514,6 +514,7 @@
$('s-prove').checked = !!(state.settings && state.settings.prove);
$('s-jobs-key').textContent = j.key_fingerprint ? 'signing key sha256:' + j.key_fingerprint : '';
var parts = [];
if (j.account) parts.push(j.account);
if (!j.allowed) parts.push('Off: nothing runs here until it is switched on.');
else if (!j.url_set) parts.push('No jobs address in this build.');
else if (j.error) parts.push(j.error + '.');