diff --git a/app/igneum-app/src/jobbuild.rs b/app/igneum-app/src/jobbuild.rs new file mode 100644 index 000000000..08d4cd11b --- /dev/null +++ b/app/igneum-app/src/jobbuild.rs @@ -0,0 +1,574 @@ +//! The `build` job (the model is src/jobs.rs, the runner src/jobrun.rs): a Windows PC builds the node and the app +//! engine for Linux and Windows inside its WSL2 Ubuntu, as root, with nothing from Josh. Josh's ask, 4 October 2026 +//! evening ("efficiency"): every Windows build went through a GitHub runner at 15 to 25 minutes a round and every +//! Linux binary was cross-compiled on the Mac under the build lock; the two RTX 5090 PCs sit idle on the CPU side. +//! +//! What this module holds is the pure part, so it is unit-tested on the Mac: the job's parameters, the plan read +//! from the inputs manifest (what to build, what to test), the bash stage scripts that run inside the distro, the +//! per-stage time caps, and the parsers for what comes back (free space, the pack stage's outputs file, the relay's +//! JSON replies). The runner (jobrun.rs, `run_build`) does the process work: fetch, wsl.exe per stage, upload. +//! +//! Stages, in order (each a RESULT or STAGE line with a UTC time in the report, each under its own cap): +//! fetch the build-inputs zip by https, sha256 checked (src/jobrun.rs fetch_file), the manifest read out of it +//! setup apt packages (mingw, clang for bindgen, protoc, zstd), rustup target x86_64-pc-windows-gnu; idempotent +//! extract /root/igneum-build/src replaced by the zip's sources (the target dir /root/igneum-build/target persists) +//! linux cargo build --release per unit, native +//! windows cargo build --release --target x86_64-pc-windows-gnu per unit, mingw-w64 (posix threads), static libgcc +//! test cargo test --release for the packages the manifest names (Linux, native) +//! pack zstd per binary, sha256 of both forms, build-outputs.json, copied to the job folder on the Windows side +//! upload every .zst and the outputs file to the relay (fn=upload client token, PUT to Blob, fn=drop); 50 MB each +//! Mining is never stopped: the build is CPU work under `nice`; the app's job runner is serial, so a build never +//! overlaps a shard benchmark (src/jobrun.rs: one `Active` at a time, queued jobs wait). +//! +//! The zip's layout (packaging/windows/push-build-inputs.sh): igneum-build-inputs/{manifest.json, node/ (the fork +//! worktree without target dirs), app/igneum-app/, brand/icons/, proto-cuda/ (without nvrtc/redist)}. The manifest: +//! { "created_at", "node": {"branch","commit","dirty","source"}, "repo": {...}, "app_version", +//! "builds": [{"dir":"node","packages":["kaspad","igneum-miner"],"features":["kaspad/igneum-pow"],"bins":["igneumd","igneum-miner"],"targets":["linux","windows"]}, +//! {"dir":"app/igneum-app","packages":["igneum-app"],"bins":["igneum-app"],"targets":["linux","windows"],"optional_on":["linux"]}], +//! "tests": [{"dir":"app/igneum-app","packages":["igneum-app"]}, {"dir":"node","packages":["igneum-miner"]}] } + +#![allow(dead_code)] + +use crate::jobs::{self, Job}; +use serde_json::Value; +use std::collections::BTreeMap; +use std::time::Duration; + +pub const RELAY_URL_DEFAULT: &str = "https://relay.igneum.network"; +/// Everything of the build lives here inside the distro: target/ (persists), src/ (per job), out// (outputs). +pub const WSL_BASE: &str = "/root/igneum-build"; +pub const ZIP_ROOT: &str = "igneum-build-inputs"; +pub const OUTPUTS_FILE: &str = "build-outputs.json"; +pub const DEFAULT_DISTRO: &str = "Ubuntu-24.04"; +pub const DEFAULT_WSL_USER: &str = "root"; +pub const DEFAULT_NICE: u64 = 19; +/// The relay's cap for one Blob upload (relay/lib/relay.mjs MAX_BLOB). +pub const MAX_UPLOAD_BYTES: u64 = 50 * 1024 * 1024; +/// Debian packages the build needs; installed only when `dpkg -s` says they are missing. +pub const APT_COMMON: &[&str] = &["build-essential", "pkg-config", "libssl-dev", "clang", "libclang-dev", "cmake", "unzip", "zstd", "protobuf-compiler", "ca-certificates", "curl", "git"]; +pub const APT_WINDOWS: &[&str] = &["gcc-mingw-w64-x86-64", "g++-mingw-w64-x86-64", "binutils-mingw-w64-x86-64", "mingw-w64-x86-64-dev"]; + +#[derive(Clone, Debug, PartialEq)] +pub struct BuildParams { + pub targets: Vec, + pub budget_min: u64, + pub stage_min: BTreeMap, + pub min_free_gb: u64, + pub tests: bool, + pub relay_url: String, + pub distro: String, + pub wsl_user: String, + pub nice: u64, + /// cargo -j; 0 = cargo's default (every core) + pub cargo_jobs: u64, +} + +impl BuildParams { + pub fn from_job(job: &Job) -> BuildParams { + let mut targets = job.list_param("targets"); + if targets.is_empty() { + targets = jobs::BUILD_TARGETS.iter().map(|s| s.to_string()).collect(); + } + targets.dedup(); + let mut stage_min = BTreeMap::new(); + if let Some(o) = job.params.get("stage_minutes").and_then(|v| v.as_object()) { + for (k, v) in o { + if let Some(n) = v.as_u64() { + stage_min.insert(k.clone(), n.max(1)); + } + } + } + let relay = job.str_param("relay_url"); + let distro = job.str_param("distro"); + let user = job.str_param("wsl_user"); + BuildParams { + targets, + budget_min: job.timeout_minutes(), + stage_min, + min_free_gb: job.u64_param("min_free_gb").unwrap_or(jobs::DEFAULT_BUILD_MIN_FREE_GB), + tests: job.params.get("tests").and_then(|v| v.as_bool()).unwrap_or(true), + relay_url: if relay.is_empty() { RELAY_URL_DEFAULT.to_string() } else { relay.trim_end_matches('/').to_string() }, + distro: if distro.is_empty() { DEFAULT_DISTRO.to_string() } else { distro }, + wsl_user: if user.is_empty() { DEFAULT_WSL_USER.to_string() } else { user }, + nice: job.u64_param("nice").unwrap_or(DEFAULT_NICE).min(19), + cargo_jobs: job.u64_param("cargo_jobs").unwrap_or(0), + } + } + pub fn wants(&self, target: &str) -> bool { + self.targets.iter().any(|t| t == target) + } +} + +/// A stage's cap: its own minutes when the job names them, else what is left of the total; never over what is left. +pub fn stage_cap(p: &BuildParams, stage: &str, remaining: Duration) -> Duration { + match p.stage_min.get(stage) { + Some(m) => Duration::from_secs(m * 60).min(remaining), + None => remaining, + } +} + +// ---- the plan from the inputs manifest ------------------------------------------------------------------------------ + +#[derive(Clone, Debug, PartialEq, Default)] +pub struct Unit { + /// path inside the zip root ("node", "app/igneum-app") + pub dir: String, + pub packages: Vec, + pub features: Vec, + /// the binaries cargo leaves in /release (and /x86_64-pc-windows-gnu/release with .exe) + pub bins: Vec, + pub targets: Vec, + /// targets where a failure is reported but does not fail the job (the engine on Linux) + pub optional_on: Vec, +} + +#[derive(Clone, Debug, PartialEq, Default)] +pub struct TestUnit { + pub dir: String, + pub packages: Vec, +} + +#[derive(Clone, Debug, PartialEq, Default)] +pub struct Manifest { + pub created_at: String, + pub node_branch: String, + pub node_commit: String, + pub node_dirty: bool, + pub app_version: String, + pub builds: Vec, + pub tests: Vec, +} + +fn strings(v: Option<&Value>) -> Vec { + v.and_then(|a| a.as_array()).map(|a| a.iter().filter_map(|x| x.as_str()).map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect()).unwrap_or_default() +} + +/// A plain relative directory inside the zip root: no "..", no leading slash, no drive, one or more components. +fn plain_dir(s: &str) -> bool { + jobs::safe_rel_path(s).is_some() +} + +fn plain_name(s: &str) -> bool { + !s.is_empty() && s.len() <= 80 && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' || c == '/') && !s.starts_with('.') && !s.contains("..") +} + +/// Reads manifest.json out of the zip. Refuses anything that would not be a plain cargo invocation inside the +/// extracted tree (a dir with "..", a package name with spaces), because these strings go into a shell script. +pub fn parse_manifest(text: &str) -> Result { + let v: Value = serde_json::from_str(text).map_err(|e| format!("manifest.json is not JSON: {e}"))?; + let s = |k: &str| v.get(k).and_then(|x| x.as_str()).unwrap_or("").trim().to_string(); + let node = v.get("node").cloned().unwrap_or(Value::Null); + let ns = |k: &str| node.get(k).and_then(|x| x.as_str()).unwrap_or("").trim().to_string(); + let mut m = Manifest { created_at: s("created_at"), node_branch: ns("branch"), node_commit: ns("commit"), node_dirty: node.get("dirty").and_then(|x| x.as_bool()).unwrap_or(false), app_version: s("app_version"), ..Default::default() }; + let builds = v.get("builds").and_then(|b| b.as_array()).ok_or("manifest.json has no \"builds\" list")?; + for (i, b) in builds.iter().enumerate() { + let dir = b.get("dir").and_then(|x| x.as_str()).unwrap_or("").trim().to_string(); + if !plain_dir(&dir) { + return Err(format!("builds[{i}].dir '{dir}' is not a plain relative path")); + } + let u = Unit { dir, packages: strings(b.get("packages")), features: strings(b.get("features")), bins: strings(b.get("bins")), targets: { let t = strings(b.get("targets")); if t.is_empty() { jobs::BUILD_TARGETS.iter().map(|s| s.to_string()).collect() } else { t } }, optional_on: strings(b.get("optional_on")) }; + if u.packages.is_empty() { + return Err(format!("builds[{i}] ({}) names no packages", u.dir)); + } + if u.bins.is_empty() { + return Err(format!("builds[{i}] ({}) names no bins", u.dir)); + } + for x in u.packages.iter().chain(u.features.iter()).chain(u.bins.iter()) { + if !plain_name(x) { + return Err(format!("builds[{i}] ({}): '{x}' is not a plain name", u.dir)); + } + } + for t in &u.targets { + if !jobs::BUILD_TARGETS.contains(&t.as_str()) { + return Err(format!("builds[{i}] ({}): target '{t}' is unknown", u.dir)); + } + } + m.builds.push(u); + } + if m.builds.is_empty() { + return Err("manifest.json names nothing to build".into()); + } + for (i, t) in v.get("tests").and_then(|b| b.as_array()).map(|a| a.to_vec()).unwrap_or_default().iter().enumerate() { + let dir = t.get("dir").and_then(|x| x.as_str()).unwrap_or("").trim().to_string(); + if !plain_dir(&dir) { + return Err(format!("tests[{i}].dir '{dir}' is not a plain relative path")); + } + let packages = strings(t.get("packages")); + if packages.iter().any(|p| !plain_name(p)) { + return Err(format!("tests[{i}] ({dir}): a package name is not plain")); + } + if !packages.is_empty() { + m.tests.push(TestUnit { dir, packages }); + } + } + Ok(m) +} + +// ---- the stage scripts (bash, run as `wsl -d -u root -- bash