oxedyne/fe2o3/fe2o3_austenite/tools/bench/corpus/sweep.sh
9.1 KiB, 1 run, executable
created by r1870400018:72522, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #!/usr/bin/env bash |
| 2 | # The whole-corpus sweep (D-20261009-10): every document in real_corpus.txt is compiled by Austenite and by |
| 3 | # typst 0.15.1, one at a time, each run under its own wall-time and memory cap, and one JSONL record is |
| 4 | # written per document (rec.py). A crash, hang or out-of-memory is recorded and the sweep moves on. Each |
| 5 | # document's outputs are deleted once measured; only the JSONL is kept. The sweep resumes: a document already |
| 6 | # in the JSONL is skipped, so a killed sweep is simply started again. |
| 7 | # |
| 8 | # Run it as ONE heavy job, never beside another document run: |
| 9 | # ~/usr/code/bash/rc-build 3G --io -- tools/bench/corpus/sweep.sh |
| 10 | # |
| 11 | # Per document: the first run of each engine is a warm-up of the disk cache and, for a Universe template, |
| 12 | # its download (`typst init`, then one typst compile that fetches its imports). Then RUNS alternating |
| 13 | # Austenite/typst runs (typst `-j 1 --ignore-system-fonts`, the same fonts on both sides), one typst run under |
| 14 | # `--timings` for its stage split (skipped above TRACE_MAX_S), and for a document (not a fixture) the warm |
| 15 | # one-letter edit through watch_edit.sh. Fixtures and the crate's own tree files get one run each and no edit. |
| 16 | # Before each document the 1-minute load average must be under LOADMAX; the sweep waits up to LOADWAIT seconds |
| 17 | # for it, then measures anyway and records the load, so report.py can name the runs to repeat. |
| 18 | # |
| 19 | # Environment (defaults): W (~/.cache/austenite/claude-rc-2/spd/corp), BIN (~/.local/bin/austenite), TYPST |
| 20 | # (~/bin/typst), LIST (real_corpus.txt beside this script), OUT ($W/sweep.jsonl), RUNS (3), TMO (120 s per |
| 21 | # run), MEM (3G per run), LOADMAX (6), LOADWAIT (120), TRACE_MAX_S (20), TIERS (all; e.g. "universe own"), |
| 22 | # ONLY (a file of ids to run, one per line; such a run ignores the resume skip), AUST_INV_ROOT |
| 23 | # (~/usr/books/ontheism), AUST_HEM_DIR (~/usr/complement/projects/oxedyne/doc/Hematite). |
| 24 | set -uo pipefail |
| 25 | |
| 26 | HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 27 | CRATE="$(cd "$HERE/../../.." && pwd)" |
| 28 | REPO="$(cd "$CRATE/.." && pwd)" |
| 29 | W="${W:-$HOME/.cache/austenite/claude-rc-2/spd/corp}" |
| 30 | BIN="${BIN:-$HOME/.local/bin/austenite}" |
| 31 | TYPST="${TYPST:-$HOME/bin/typst}" |
| 32 | LIST="${LIST:-$HERE/real_corpus.txt}" |
| 33 | OUT="${OUT:-$W/sweep.jsonl}" |
| 34 | RUNS="${RUNS:-3}" |
| 35 | TMO="${TMO:-120}" |
| 36 | MEM="${MEM:-3G}" |
| 37 | LOADMAX="${LOADMAX:-6}" |
| 38 | LOADWAIT="${LOADWAIT:-120}" |
| 39 | TRACE_MAX_S="${TRACE_MAX_S:-20}" |
| 40 | TIERS="${TIERS:-universe own syn tree fixture}" |
| 41 | ONLY="${ONLY:-}" |
| 42 | AUST_INV_ROOT="${AUST_INV_ROOT:-$HOME/usr/books/ontheism}" |
| 43 | AUST_HEM_DIR="${AUST_HEM_DIR:-$HOME/usr/complement/projects/oxedyne/doc/Hematite}" |
| 44 | export TYPST_PACKAGE_CACHE_PATH="${TYPST_PACKAGE_CACHE_PATH:-$W/pkgs}" |
| 45 | |
| 46 | mkdir -p "$W/src" "$W/run" "$TYPST_PACKAGE_CACHE_PATH" |
| 47 | touch "$OUT" |
| 48 | LOG="$W/sweep.log" |
| 49 | |
| 50 | log() { printf '%s %s\n' "$(date +%T)" "$*" >> "$LOG"; } |
| 51 | |
| 52 | load1() { cut -d' ' -f1 /proc/loadavg; } |
| 53 | |
| 54 | wait_load() { |
| 55 | local t=0 |
| 56 | while awk -v l="$(load1)" -v m="$LOADMAX" 'BEGIN { exit !(l > m) }'; do |
| 57 | [ "$t" -ge "$LOADWAIT" ] && return 0 |
| 58 | sleep 10 |
| 59 | t=$((t + 10)) |
| 60 | done |
| 61 | } |
| 62 | |
| 63 | # capped <prefix> <command...>: one run under its own memory scope and wall-time cap. Writes <prefix>.rc, |
| 64 | # .wall (seconds, from tick.sh), .time (wall user sys maxrss_kb; /usr/bin/time wraps the timeout, inside the memory cap), .out, .err and .load. |
| 65 | capped() { |
| 66 | local pfx="$1" |
| 67 | shift |
| 68 | load1 > "$pfx.load" |
| 69 | systemd-run --user --scope --quiet -p MemoryMax="$MEM" -p MemorySwapMax=0 --slice=claude-rc.slice \ |
| 70 | /usr/bin/time -o "$pfx.time" -f '%e %U %S %M' timeout -k 5 "$TMO" "$HERE/tick.sh" "$pfx.wall" "$@" > "$pfx.out" 2> "$pfx.err" |
| 71 | echo $? > "$pfx.rc" |
| 72 | } |
| 73 | |
| 74 | has_fonts() { [ -n "$(find "$1" -maxdepth 3 \( -iname '*.ttf' -o -iname '*.otf' -o -iname '*.ttc' \) -print -quit 2>/dev/null)" ]; } |
| 75 | |
| 76 | # stage <tier> <id> <spec> <extra>: sets SRC, ROOT, FONTS (may be empty) and EDIT (1 when the warm edit |
| 77 | # runs, on a copy this script owns). Returns non-zero, with STAGE_ERR, when the document cannot be had. |
| 78 | stage() { |
| 79 | local tier="$1" id="$2" spec="$3" extra="$4" d |
| 80 | SRC="" ROOT="" FONTS="" EDIT=0 STAGE_ERR="" |
| 81 | case "$tier" in |
| 82 | universe) |
| 83 | d="$W/src/universe/${id/:/-}" |
| 84 | if [ ! -f "$d/.staged" ]; then |
| 85 | rm -rf "${d:?}" |
| 86 | mkdir -p "$W/src/universe" |
| 87 | if ! timeout 120 "$TYPST" init "@preview/$id" "$d" > /dev/null 2> "$R/fetch.err"; then |
| 88 | STAGE_ERR="typst init: $(tail -c 300 "$R/fetch.err" | tr '\n' ' ')" |
| 89 | rm -rf "${d:?}" |
| 90 | return 1 |
| 91 | fi |
| 92 | touch "$d/.staged" |
| 93 | fi |
| 94 | SRC="$d/$spec" ROOT="$d" EDIT=1 |
| 95 | has_fonts "$d" && FONTS="$d" |
| 96 | ;; |
| 97 | syn) |
| 98 | d="$W/src/syn" |
| 99 | mkdir -p "$d" |
| 100 | SRC="$d/$id.typ" ROOT="$d" EDIT=1 |
| 101 | if [ ! -s "$SRC" ]; then |
| 102 | case "$spec" in |
| 103 | gen:*) python3 -I "$CRATE/tools/bench/gen_synthetic.py" "${spec#gen:}" "$SRC" > /dev/null ;; |
| 104 | heap:*) node "$CRATE/tools/heap_probe.mjs" --dump "${spec#heap:}" > "$SRC" ;; |
| 105 | esac |
| 106 | fi |
| 107 | [ -s "$SRC" ] || { STAGE_ERR="cannot generate $spec"; return 1; } |
| 108 | ;; |
| 109 | own) |
| 110 | d="$W/src/own" |
| 111 | case "$spec" in |
| 112 | invitation) |
| 113 | [ -d "$AUST_INV_ROOT/TheOrder/Invitation" ] || { STAGE_ERR="no $AUST_INV_ROOT"; return 1; } |
| 114 | if [ ! -f "$d/ontheism/.staged" ]; then |
| 115 | mkdir -p "$d/ontheism/TheOrder" "$d/ontheism/assets" |
| 116 | rsync -a --exclude '*.pdf' "$AUST_INV_ROOT/TheOrder/Invitation" "$d/ontheism/TheOrder/" |
| 117 | rsync -a "$AUST_INV_ROOT/assets/fonts" "$d/ontheism/assets/" |
| 118 | touch "$d/ontheism/.staged" |
| 119 | fi |
| 120 | SRC="$d/ontheism/TheOrder/Invitation/invitation.typ" ROOT="$d/ontheism" |
| 121 | FONTS="$d/ontheism/assets/fonts" EDIT=1 |
| 122 | ;; |
| 123 | hematite) |
| 124 | [ -d "$AUST_HEM_DIR" ] || { STAGE_ERR="no $AUST_HEM_DIR"; return 1; } |
| 125 | if [ ! -f "$d/Hematite/.staged" ]; then |
| 126 | mkdir -p "$d" |
| 127 | rsync -aL --exclude '*.pdf' --exclude archive --exclude dev "$AUST_HEM_DIR" "$d/" |
| 128 | touch "$d/Hematite/.staged" |
| 129 | fi |
| 130 | SRC="$d/Hematite/hematite.typ" ROOT="$d/Hematite" EDIT=1 |
| 131 | ;; |
| 132 | esac |
| 133 | ;; |
| 134 | tree|fixture) |
| 135 | SRC="$REPO/$spec" ROOT="$(dirname "$SRC")" |
| 136 | ;; |
| 137 | esac |
| 138 | [ -f "$SRC" ] || { STAGE_ERR="no source $SRC"; return 1; } |
| 139 | } |
| 140 | |
| 141 | aus_cmd() { |
| 142 | local o="$1" j="$2" |
| 143 | AUS=("$BIN" --eval --timings "$j" --root "$ROOT") |
| 144 | [ -n "$FONTS" ] && AUS+=(--font-path "$FONTS") |
| 145 | AUS+=("$SRC" "$o") |
| 146 | } |
| 147 | |
| 148 | typ_cmd() { |
| 149 | local o="$1" |
| 150 | shift |
| 151 | TYP=("$TYPST" compile -j 1 --ignore-system-fonts --root "$ROOT") |
| 152 | [ -n "$FONTS" ] && TYP+=(--font-path "$FONTS") |
| 153 | TYP+=("$@" "$SRC" "$o") |
| 154 | } |
| 155 | |
| 156 | measure() { |
| 157 | local tier="$1" id="$2" spec="$3" extra="$4" runs="$RUNS" i |
| 158 | case "$tier" in tree|fixture) runs=1 ;; esac |
| 159 | if [ "$tier" = universe ]; then |
| 160 | # The warm-up compile fetches the template's imports; typst may need the network for it. |
| 161 | typ_cmd "$R/warm.pdf" |
| 162 | capped "$R/fetch" "${TYP[@]}" |
| 163 | rm -f "$R/warm.pdf" |
| 164 | fi |
| 165 | # Warm the disk cache for both engines (not recorded). |
| 166 | aus_cmd "$R/aw" "$R/aw.json" |
| 167 | capped "$R/aw" "${AUS[@]}" |
| 168 | rm -rf "${R:?}/aw" "$R/aw.json" |
| 169 | for i in $(seq 1 "$runs"); do |
| 170 | aus_cmd "$R/ao" "$R/a$i.json" |
| 171 | capped "$R/a$i" "${AUS[@]}" |
| 172 | rm -rf "${R:?}/ao" |
| 173 | typ_cmd "$R/t.pdf" |
| 174 | capped "$R/t$i" "${TYP[@]}" |
| 175 | if [ "$i" = "$runs" ] && [ -f "$R/t.pdf" ]; then |
| 176 | pdfinfo "$R/t.pdf" 2>/dev/null | awk '/^Pages:/ { print $2 }' > "$R/t.pages" |
| 177 | fi |
| 178 | rm -f "$R/t.pdf" |
| 179 | done |
| 180 | case "$tier" in tree|fixture) return 0 ;; esac |
| 181 | if [ "$(cat "$R/t1.rc" 2>/dev/null)" = 0 ] && awk -v w="$(cut -d' ' -f1 "$R/t1.time")" -v m="$TRACE_MAX_S" 'BEGIN { exit !(w <= m) }'; then |
| 182 | typ_cmd "$R/t.pdf" --timings "$R/tt.json" |
| 183 | capped "$R/tt" "${TYP[@]}" |
| 184 | rm -f "$R/t.pdf" |
| 185 | fi |
| 186 | if [ "$EDIT" = 1 ] && [ "$(cat "$R/a1.rc" 2>/dev/null)" = 0 ]; then |
| 187 | local set=(--set "root=$ROOT") |
| 188 | [ -n "$FONTS" ] && set+=(--set "fonts=$FONTS") |
| 189 | # Three waits of WAIT_S inside one cap, so the watch is never cut off with its child alive. |
| 190 | SCRATCH="$R/w" WAIT_S=60 TMO=220 capped "$R/w" "$CRATE/tools/bench/watch_edit.sh" "$BIN" "$SRC" "${set[@]}" |
| 191 | rm -rf "${R:?}/w" |
| 192 | fi |
| 193 | } |
| 194 | |
| 195 | done_ids() { python3 -I -c ' |
| 196 | import json, sys |
| 197 | for l in open(sys.argv[1]): |
| 198 | try: print(json.loads(l)["id"]) |
| 199 | except Exception: pass' "$OUT"; } |
| 200 | |
| 201 | declare -A SKIP=() |
| 202 | declare -A WANT=() |
| 203 | if [ -n "$ONLY" ]; then |
| 204 | while read -r x; do [ -n "$x" ] && WANT["$x"]=1; done < "$ONLY" |
| 205 | else |
| 206 | while read -r x; do SKIP["$x"]=1; done < <(done_ids) |
| 207 | fi |
| 208 | |
| 209 | log "sweep start: list $LIST, out $OUT, runs $RUNS, bin $BIN ($(sha256sum "$BIN" | cut -c1-12)), $(${TYPST} --version)" |
| 210 | n=0 |
| 211 | while IFS=$'\t' read -r tier id spec extra rest; do |
| 212 | case "$tier" in ''|'#'*) continue ;; esac |
| 213 | case " $TIERS " in *" $tier "*) ;; *) continue ;; esac |
| 214 | if [ -n "$ONLY" ]; then |
| 215 | [ -n "${WANT[$id]:-}" ] || continue |
| 216 | else |
| 217 | [ -n "${SKIP[$id]:-}" ] && continue |
| 218 | fi |
| 219 | n=$((n + 1)) |
| 220 | R="$W/run/cur" |
| 221 | rm -rf "${R:?}" |
| 222 | mkdir -p "$R" |
| 223 | wait_load |
| 224 | t0=$EPOCHREALTIME |
| 225 | meta="$(python3 -I -c 'import json,sys; print(json.dumps({"id":sys.argv[1],"tier":sys.argv[2],"spec":sys.argv[3],"extra":sys.argv[4],"licence":sys.argv[5]}))' "$id" "$tier" "$spec" "${extra:-}" "${rest:-}")" |
| 226 | if stage "$tier" "$id" "$spec" "${extra:-}"; then |
| 227 | measure "$tier" "$id" "$spec" "${extra:-}" |
| 228 | runs="$RUNS" |
| 229 | case "$tier" in tree|fixture) runs=1 ;; esac |
| 230 | python3 -I "$HERE/rec.py" "$R" "$runs" "$meta" >> "$OUT" || log "rec failed: $id" |
| 231 | else |
| 232 | python3 -I -c 'import json,sys; m=json.loads(sys.argv[1]); m["stage_err"]=sys.argv[2]; print(json.dumps(m))' "$meta" "$STAGE_ERR" >> "$OUT" |
| 233 | fi |
| 234 | log "$n $tier $id $(awk -v s="$t0" -v e="$EPOCHREALTIME" 'BEGIN { printf "%.1fs", e - s }')" |
| 235 | rm -rf "${R:?}" |
| 236 | done < "$LIST" |
| 237 | log "sweep end: $n documents" |