diff --git a/tools/ci/canary-check.sh b/tools/ci/canary-check.sh index 85d315279..4fbdf7a39 100755 --- a/tools/ci/canary-check.sh +++ b/tools/ci/canary-check.sh @@ -19,9 +19,12 @@ # box {host, isa_line}: a non-AVX-512 box; isa_line is the kit-isa clean line read on it (tools/ci/kit-isa-check.sh) # or the host's cpu flags line showing no avx512 # datadir {path, empty_at_start: true, read_back}: the empty datadir before the install -# sync {genesis_height: 0, tip_height, seconds, read_back}: genesis to the network's tip +# sync {genesis_height: 0, tip_height, seconds, read_back, feed: {height, age_s, blocks_per_min}} (the feed block is +# optional and not in the form; add it when the canary read a feed line): genesis to the +# network's tip; with a feed line, tip_height must be within ten blocks of height + age_s * blocks_per_min / 60 # mining {minutes >= 5, refusals: 0, accepted_blocks >= 1, read_back} -# shard {claimed: true, proved: true, paid_or_queued: "paid" | "queued", read_back} +# shard {claimed: true, proved: true, paid_or_queued: "paid" | "queued", read_back}; for kind mac only +# {not_applicable: "no prover on Apple silicon", read_back} passes (Apple silicon has no prover) # quit {bounded: true, seconds, read_back}: the process ends on its own quit inside the bound # lines_read_back a list of the eight line names, each with an evidence path on a box # verdict "PASS" (anything else is not a canary record for publishing) @@ -100,6 +103,12 @@ for r_ in blocks: s = need(r, 'sync', dict) if s.get('genesis_height') != 0: red("sync.genesis_height is not 0 (the sync starts at genesis)") if not (isinstance(s.get('tip_height'), int) and s['tip_height'] > 0): red("sync.tip_height is not a positive height") + fd = s.get('feed') # the age-adjusted tip (9 October 2026, 01:1x UK): the feed line is written once a minute, so the node reads ahead of it by the minute's blocks + if isinstance(fd, dict): + try: fh, age, rate = int(fd['height']), float(fd.get('age_s', 0)), float(fd.get('blocks_per_min', 0)) + except Exception: red("sync.feed needs height, age_s and blocks_per_min") + adj = fh + age * rate / 60.0 + if abs(s['tip_height'] - adj) > 10: red(f"sync.tip_height {s['tip_height']} is not within ten blocks of the feed line {fh} adjusted by its age ({age:.0f} s at {rate:.0f} a minute = {adj:.0f})") if not box_path(s.get('read_back')): red("sync.read_back is not a box path") m = need(r, 'mining', dict) if not (isinstance(m.get('minutes'), (int, float)) and m['minutes'] >= 5): red("mining.minutes is under 5") @@ -107,9 +116,14 @@ for r_ in blocks: if not (isinstance(m.get('accepted_blocks'), int) and m['accepted_blocks'] >= 1): red("mining.accepted_blocks is under 1") if not box_path(m.get('read_back')): red("mining.read_back is not a box path") h = need(r, 'shard', dict) - if h.get('claimed') is not True or h.get('proved') is not True: red("shard.claimed and shard.proved must both be true") - if h.get('paid_or_queued') not in ('paid', 'queued'): red("shard.paid_or_queued must be 'paid' or 'queued'") - if not box_path(h.get('read_back')): red("shard.read_back is not a box path") + kind = str(r.get('kind', '')).lower() + if kind == 'mac' and h.get('not_applicable'): # Apple silicon has no prover (9 October 2026, 01:1x UK): the mac block's shard line is n/a, read back like any line + if not box_path(h.get('read_back')): red("shard.read_back is not a box path (the n/a line is read back too)") + else: + if h.get('not_applicable'): red(f"shard.not_applicable is accepted for kind mac only (this block is '{kind or 'artefact'}')") + if h.get('claimed') is not True or h.get('proved') is not True: red("shard.claimed and shard.proved must both be true") + if h.get('paid_or_queued') not in ('paid', 'queued'): red("shard.paid_or_queued must be 'paid' or 'queued'") + if not box_path(h.get('read_back')): red("shard.read_back is not a box path") q = need(r, 'quit', dict) if q.get('bounded') is not True: red("quit.bounded is not true") if not (isinstance(q.get('seconds'), (int, float)) and q['seconds'] >= 0): red("quit.seconds is not a number") @@ -125,7 +139,7 @@ for r_ in blocks: v = r.get('verdict', top.get('verdict')) if v != 'PASS': red(f"verdict is {v!r}, not PASS") if not (r.get('recorded_at') or top.get('recorded_at')) or not (r.get('recorded_by') or top.get('recorded_by')): red("recorded_at or recorded_by is empty") - lines_out.append(f"{r.get('kind', 'artefact')} on {b['host']} ({str(a.get('sha256'))[:12]}…): genesis to {s['tip_height']} in {s.get('seconds')} s, {m['minutes']} min mining, 0 refusals, {m['accepted_blocks']} accepted, one shard {h['paid_or_queued']}, quit in {q['seconds']} s; eight lines read back") + lines_out.append(f"{r.get('kind', 'artefact')} on {b['host']} ({str(a.get('sha256'))[:12]}…): genesis to {s['tip_height']} in {s.get('seconds')} s, {m['minutes']} min mining, 0 refusals, {m['accepted_blocks']} accepted, {('shard n/a (' + str(h.get('not_applicable')) + ')') if h.get('not_applicable') else ('one shard ' + str(h.get('paid_or_queued')))}, quit in {q['seconds']} s; eight lines read back") print(f"canary: PASS: {sha[:12]} fresh-install canary: " + '; '.join(lines_out) + f"; recorded {blocks[0].get('recorded_at') or json.load(open(f)).get('recorded_at')} by {blocks[0].get('recorded_by') or json.load(open(f)).get('recorded_by')}") PY } @@ -163,9 +177,22 @@ m={'sha':r['sha'],'artefacts':arts,'verdict':'PASS','recorded_at':r['recorded_at out=$(bash "$ME" "$SHA" --artefact windows 2>&1) && { echo "self-test failed: a failing windows block passed under --artefact windows"; fails=1; } bash "$ME" "$SHA" --artefact mac >/dev/null 2>&1 || { echo "self-test failed: the mac block was refused because the windows block fails (each entry publishes on its own canary)"; fails=1; } out=$(bash "$ME" "$SHA" 2>&1) && { echo "self-test failed: the whole record passed with one failing block"; fails=1; } + # the mac shard n/a and the age-adjusted tip + cp "$d/keep.json" "$d/$SHA.json" + python3 -c " +import json,copy; p='$d/$SHA.json'; r=json.load(open(p)); blk={k:r[k] for k in ('artefact','box','datadir','sync','mining','shard','quit','lines_read_back')} +m=copy.deepcopy(blk); m['kind']='mac'; m['shard']={'not_applicable':'no prover on Apple silicon','read_back':'build-1:/srv/canary/x/04-shard.txt'} +f=copy.deepcopy(blk); f['kind']='fleet'; f['shard']={'not_applicable':'no prover','read_back':'build-1:/srv/canary/x/04-shard.txt'} +t=copy.deepcopy(blk); t['kind']='windows'; t['sync']={'genesis_height':0,'tip_height':14189,'seconds':1036,'feed':{'height':14155,'age_s':60,'blocks_per_min':30},'read_back':'build-1:/srv/canary/x/02-sync.txt'} +b=copy.deepcopy(blk); b['kind']='hive'; b['sync']={'genesis_height':0,'tip_height':14100,'seconds':1036,'feed':{'height':14155,'age_s':60,'blocks_per_min':30},'read_back':'build-1:/srv/canary/x/02-sync.txt'} +json.dump({'sha':r['sha'],'artefacts':[m,f,t,b],'verdict':'PASS','recorded_at':'t','recorded_by':'x'}, open(p,'w'))" + bash "$ME" "$SHA" --artefact mac >/dev/null 2>&1 || { echo "self-test failed: a mac block with the shard n/a was refused: $(bash "$ME" "$SHA" --artefact mac 2>&1)"; fails=1; } + out=$(bash "$ME" "$SHA" --artefact fleet 2>&1) && { echo "self-test failed: a fleet block with the shard n/a passed"; fails=1; }; case "$out" in *"kind mac only"*) ;; *) echo "self-test failed: the n/a refusal was not named: $out"; fails=1 ;; esac + bash "$ME" "$SHA" --artefact windows >/dev/null 2>&1 || { echo "self-test failed: a tip within ten blocks of the age-adjusted feed was refused: $(bash "$ME" "$SHA" --artefact windows 2>&1)"; fails=1; } + out=$(bash "$ME" "$SHA" --artefact hive 2>&1) && { echo "self-test failed: a tip 85 blocks under the adjusted feed passed"; fails=1; }; case "$out" in *"not within ten blocks"*) ;; *) echo "self-test failed: the tip refusal was not named: $out"; fails=1 ;; esac echo "not json" > "$d/$SHA.json"; bash "$ME" "$SHA" >/dev/null 2>&1 && { echo "self-test failed: a non-JSON record passed"; fails=1; } out=$(bash "$ME" "not-a-sha" 2>&1) && { echo "self-test failed: a non-sha argument passed"; fails=1; } - [ "$fails" = 0 ] && echo "self-test passed: a complete fresh-install canary record is PASS and found by its short sha; no record, a refusal, under five minutes, no accepted block, an unproved or lost shard, an unbounded quit, a non-empty datadir, a sync not from genesis, an AVX-512 box, a missing read-back line, a non-box evidence path, a non-PASS verdict or a bad artefact hash is refused and named; a one-file-per-sha record with an artefacts list (fleet, windows, mac, hive) passes whole or per --artefact kind, and a failing block fails its own kind and the whole, never another kind" + [ "$fails" = 0 ] && echo "self-test passed: a complete fresh-install canary record is PASS and found by its short sha; no record, a refusal, under five minutes, no accepted block, an unproved or lost shard, an unbounded quit, a non-empty datadir, a sync not from genesis, an AVX-512 box, a missing read-back line, a non-box evidence path, a non-PASS verdict or a bad artefact hash is refused and named; a one-file-per-sha record with an artefacts list (fleet, windows, mac, hive) passes whole or per --artefact kind, and a failing block fails its own kind and the whole, never another kind; a mac block's shard may read n/a (no prover on Apple silicon) and no other kind's; a tip within ten blocks of the feed line adjusted by its age passes and one further off is refused" exit $fails fi KIND=""; SHA_ARG=""