111 lines
9 KiB
Bash
Executable file
111 lines
9 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Igneum GPU fleet, on-box setup (6 October 2026). Runs as root inside a rented Vast or RunPod container built from
|
|
# nvidia/cuda:12.8.1-devel-ubuntu24.04 (or RunPod's CUDA 12.8 image). Idempotent; every step prints a RESULT line to
|
|
# /root/fleet/setup.log and the last line is "RESULT setup_done" or "RESULT setup_failed <step>".
|
|
#
|
|
# What it does, in order: tools (apt), the 0.3.12 Linux package (sha256 checked) unpacked to /opt/igneum/pkg, the node
|
|
# started at once on the devnet with the ten-field override (so it syncs while the builds run), rustup, Go 1.27.1 (pinned
|
|
# sha256), SP1 v6.8.1 cloned and patched with proving/prover-floor/sp1-gpu-6.8.1-floor.patch, the patched server built
|
|
# with CUDA_ARCHS=$ARCHS into /opt/igneum-floor (the SDK's stock server stays under /root/.sp1 for the stock rows), and
|
|
# igneum-prove-host + igneum-prove-export built with the cuda feature from the prover-floor host bundle.
|
|
#
|
|
# Inputs in /root/fleet/in: igneum-prove-wsl2-floor.zip (the host sources, pinned ELFs, fixtures), floor.patch,
|
|
# override.json. Env: ARCHS (CUDA arch list, default 86,89,120), LABEL (this box's name), WALLET (0x payout, throwaway).
|
|
set -uo pipefail
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
F=/root/fleet; IN=$F/in; OUT=$F/out; LOG=$F/setup.log
|
|
mkdir -p $F $IN $OUT /opt/igneum /opt/igneum-floor/bin /opt/igneum-floor/home/.sp1/bin /opt/igneum-floor/logs
|
|
exec > >(tee -a $LOG) 2>&1
|
|
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
|
say() { echo "$(stamp) $*"; }
|
|
fail() { echo "RESULT setup_failed $1 $(stamp)"; exit 2; }
|
|
ARCHS="${ARCHS:-86,89,120}"; LABEL="${LABEL:-box}"; WALLET="${WALLET:-}"
|
|
PKG_URL=https://dl.igneum.network/dl/public/igneum-hive-0.3.12.tar.gz
|
|
PKG_SHA=7972af92e7cd9a032303eca4d95b533f53e0e68d1b9cae5bfe406a5b7c30a454
|
|
PATCH_SHA=e81cb0d03b291f9fd4bf0a109d6da2d7c897795c9ffd7f797c0ddce723eee2b1
|
|
SEED=188.245.5.161:26611
|
|
echo "RESULT start $(stamp) label=$LABEL archs=$ARCHS host=$(hostname) nproc=$(nproc) ram_gb=$(( $(awk '/MemTotal/{print $2}' /proc/meminfo) / 1048576 )) disk_avail=$(df -BG /root | awk 'NR==2{print $4}')"
|
|
echo "RESULT gpu $(nvidia-smi --query-gpu=name,memory.total,driver_version,pci.bus_id,power.limit,power.min_limit,power.max_limit,clocks.max.sm --format=csv,noheader 2>&1 | tr '\n' ';')"
|
|
echo "RESULT os $(. /etc/os-release; echo "$ID $VERSION_ID") glibc $(ldd --version | head -1 | awk '{print $NF}') nvcc $(nvcc --version 2>/dev/null | grep -o 'release [0-9.]*' || echo none)"
|
|
|
|
# 1. tools
|
|
if ! command -v protoc >/dev/null 2>&1 || ! command -v cmake >/dev/null 2>&1 || ! command -v clang >/dev/null 2>&1; then
|
|
say "apt"
|
|
apt-get update -qq >/dev/null 2>&1
|
|
apt-get install -y -qq build-essential clang cmake pkg-config libssl-dev git curl ca-certificates python3 wget unzip protobuf-compiler jq rsync bc pciutils >/dev/null 2>&1 || fail apt
|
|
fi
|
|
echo "RESULT tools protoc=$(protoc --version 2>/dev/null) cmake=$(cmake --version | head -1) clang=$(clang --version | head -1 | cut -c1-40)"
|
|
|
|
# 2. the 0.3.12 Linux package and the node
|
|
if [ ! -x /opt/igneum/pkg/bin/igneumd ]; then
|
|
say "package"
|
|
curl -fsSL -o $F/pkg.tgz "$PKG_URL" || fail package_download
|
|
echo "$PKG_SHA $F/pkg.tgz" | sha256sum -c - >/dev/null || fail package_sha256
|
|
rm -rf /opt/igneum/pkg.new && mkdir -p /opt/igneum/pkg.new && tar -C /opt/igneum/pkg.new --strip-components=1 -xzf $F/pkg.tgz && mv /opt/igneum/pkg.new /opt/igneum/pkg
|
|
fi
|
|
B=/opt/igneum/pkg/bin
|
|
echo "RESULT package igneumd=$(sha256sum $B/igneumd | cut -c1-16) miner=$(sha256sum $B/igneum-miner | cut -c1-16) cuda_worker=$(sha256sum $B/igneum-worker-cuda | cut -c1-16) version=$($B/igneumd --version 2>&1 | head -1)"
|
|
cp $IN/override.json $F/override.json
|
|
if ! pgrep -x igneumd >/dev/null; then
|
|
say "node"
|
|
mkdir -p $F/node
|
|
nohup $B/igneumd --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 \
|
|
--addpeer=$SEED --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 &
|
|
sleep 8
|
|
fi
|
|
echo "RESULT node pid=$(pgrep -x igneumd | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | head -1 | awk '{print $2}') fresh=$(grep -c 'fresh-record rule' $F/node.log)"
|
|
|
|
# 3. rust and go
|
|
export PATH="$HOME/.cargo/bin:/opt/igneum-floor/go/bin:$PATH" RUSTUP_TOOLCHAIN=stable # the SP1 repo's rust-toolchain.toml would pull llvm-tools, rustc-dev and clippy from a CDN some hosts reach badly (the 3060 box, 12:00Z); plain stable builds the server
|
|
if [ ! -x "$HOME/.cargo/bin/cargo" ]; then say "rustup"; curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal >/dev/null 2>&1 || fail rustup; fi
|
|
if [ ! -x /opt/igneum-floor/go/bin/go ]; then
|
|
say "go"
|
|
curl -sSL -o /opt/igneum-floor/go.tgz https://go.dev/dl/go1.27.1.linux-amd64.tar.gz || fail go_download
|
|
echo "63d339f0da5ab53635a56f2490a7984dfe12dfcff22ad749f63edaf590168445 /opt/igneum-floor/go.tgz" | sha256sum -c - >/dev/null || fail go_sha256
|
|
tar -xzf /opt/igneum-floor/go.tgz -C /opt/igneum-floor && rm -f /opt/igneum-floor/go.tgz
|
|
fi
|
|
export GOPATH=/opt/igneum-floor/gopath GOCACHE=/opt/igneum-floor/gocache GOFLAGS=-mod=mod
|
|
echo "RESULT toolchain cargo=$(cargo --version) go=$(go version | awk '{print $3}')"
|
|
|
|
# 4. the patched SP1 GPU server
|
|
SRC=/opt/igneum-floor/sp1
|
|
echo "RESULT patch_sha256 $(sha256sum $IN/floor.patch | cut -c1-64) expected $PATCH_SHA"
|
|
[ "$(sha256sum $IN/floor.patch | cut -c1-64)" = "$PATCH_SHA" ] || fail patch_sha256
|
|
if [ ! -x /opt/igneum-floor/bin/sp1-gpu-server ]; then
|
|
if [ ! -d "$SRC/.git" ]; then say "clone sp1"; git clone -q --depth 1 --branch v6.8.1 https://github.com/succinctlabs/sp1 "$SRC" || fail clone; fi
|
|
cd "$SRC" && git checkout -q -- . && git clean -qfd sp1-gpu/crates >/dev/null 2>&1
|
|
echo "RESULT source $(git describe --tags --always) $(git rev-parse HEAD)"
|
|
git apply $IN/floor.patch || fail patch_apply
|
|
touch sp1-gpu/crates/prover_components/src/builder.rs sp1-gpu/crates/jagged_tracegen/src/lib.rs sp1-gpu/crates/server/src/server.rs sp1-gpu/crates/cuda/src/task.rs
|
|
echo "RESULT patched $(git diff --stat | tail -1)"
|
|
export CUDA_ARCHS="$ARCHS" CARGO_TARGET_DIR=/opt/igneum-floor/target CUDA_PATH=/usr/local/cuda CUDACXX=/usr/local/cuda/bin/nvcc
|
|
J=$(nproc); [ "$J" -gt 16 ] && J=16
|
|
say "build server jobs=$J archs=$ARCHS"; t0=$(date +%s)
|
|
cargo build --release --bin sp1-gpu-server -j $J > /opt/igneum-floor/logs/build-server.log 2>&1 &
|
|
BP=$!; while kill -0 $BP 2>/dev/null; do sleep 60; echo "STAGE server $(stamp) $(( ($(date +%s) - t0) / 60 )) min $(grep -c '^ Compiling' /opt/igneum-floor/logs/build-server.log) crates"; done
|
|
wait $BP; rc=$?
|
|
echo "RESULT server_build_exit $rc time_s=$(( $(date +%s) - t0 ))"
|
|
[ $rc -eq 0 ] || { grep -n -A8 '^error' /opt/igneum-floor/logs/build-server.log | head -60; fail server_build; }
|
|
cp /opt/igneum-floor/target/release/sp1-gpu-server /opt/igneum-floor/bin/ && cp /opt/igneum-floor/bin/sp1-gpu-server /opt/igneum-floor/home/.sp1/bin/ && chmod +x /opt/igneum-floor/bin/sp1-gpu-server /opt/igneum-floor/home/.sp1/bin/sp1-gpu-server
|
|
fi
|
|
echo "RESULT server bytes=$(stat -c %s /opt/igneum-floor/bin/sp1-gpu-server) sha256=$(sha256sum /opt/igneum-floor/bin/sp1-gpu-server | cut -c1-64) version=$(/opt/igneum-floor/bin/sp1-gpu-server --version 2>&1 | head -1) elf=$(cuobjdump --list-elf /opt/igneum-floor/bin/sp1-gpu-server 2>/dev/null | grep -o 'sm_[0-9]*' | sort -u | tr '\n' ' ')"
|
|
|
|
# 5. the host and the exporter (cuda feature), from the prover-floor bundle
|
|
H=/opt/igneum-floor/prove
|
|
if [ ! -x $H/proving/igneum-prove/target/release/igneum-prove-host ]; then
|
|
say "host"
|
|
rm -rf $H && mkdir -p $H && unzip -q -o $IN/igneum-prove-wsl2-floor.zip -d $F/unz && cp -r $F/unz/igneum-prove-wsl2/package/. $H/ && find $H -type f -exec touch {} +
|
|
cd $H/proving/igneum-prove || fail host_src
|
|
t0=$(date +%s)
|
|
cargo build --release -p igneum-prove-export -p igneum-prove-host --features igneum-prove-host/cuda -j $(( $(nproc) > 16 ? 16 : $(nproc) )) > /opt/igneum-floor/logs/build-host.log 2>&1 &
|
|
BP=$!; while kill -0 $BP 2>/dev/null; do sleep 60; echo "STAGE host $(stamp) $(( ($(date +%s) - t0) / 60 )) min $(grep -c '^ Compiling' /opt/igneum-floor/logs/build-host.log) crates"; done
|
|
wait $BP; rc=$?
|
|
echo "RESULT host_build_exit $rc time_s=$(( $(date +%s) - t0 ))"
|
|
[ $rc -eq 0 ] || { grep -n -A8 '^error' /opt/igneum-floor/logs/build-host.log | head -60; fail host_build; }
|
|
fi
|
|
HOST=$H/proving/igneum-prove/target/release/igneum-prove-host; EXPORT=$H/proving/igneum-prove/target/release/igneum-prove-export
|
|
ln -sfn $HOST /opt/igneum-floor/bin/igneum-prove-host; ln -sfn $EXPORT /opt/igneum-floor/bin/igneum-prove-export
|
|
echo "RESULT host sha256=$(sha256sum $HOST | cut -c1-16) export=$(sha256sum $EXPORT | cut -c1-16) ids=$($HOST --mode id 2>&1 | grep -o '0x[0-9a-f]*' | tr '\n' ' ')"
|
|
echo "RESULT fixtures $(ls $H/proving/fixtures | wc -l) files, v1=$(sha256sum $H/proving/fixtures/fees-v1-shards2.json | cut -c1-16) empty=$(sha256sum $H/proving/fixtures/block-72854-empty-block-first.json | cut -c1-16)"
|
|
echo "RESULT node_now $($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"
|
|
echo "RESULT setup_done $(stamp)"
|