oxedyne/fe2o3/fe2o3_austenite/tools/bench/cost_runs.sh
3.9 KiB, 1 run, executable
created by r1870400018:66223, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #!/usr/bin/env bash |
| 2 | # Cold and unchanged wasm compiles (G11 (a) and (b)), the engines interleaved run for run: ROUNDS |
| 3 | # rounds, each one wasm_bench.mjs process per engine, the engine that goes first alternating from |
| 4 | # round to round (round 0 Austenite first). One process is one measured run: |
| 5 | # cold a fresh wasm instance in a fresh process, no warm-up (the instance is the cold part); |
| 6 | # unchanged one warm-up compile of the project, then one measured compile of the same project. |
| 7 | # Each process holds both Austenite build-slot locks while it runs (lib/quiet_host.sh) and |
| 8 | # releases them, with a pause, before the next. cpu PSI avg10 and the 1-minute load are read at |
| 9 | # each process's start and end and kept in the JSONL; a run over PSI_LIMIT at either end is |
| 10 | # marked flagged. bench_verdict.mjs cost takes the medians. |
| 11 | # |
| 12 | # Usage: AUST_VENDOR=DIR TYPST_VENDOR=DIR OUT=FILE.jsonl [FMT=pdf] [ROUNDS=5] [MODES="cold unchanged"] \ |
| 13 | # cost_runs.sh <doc.typ> |
| 14 | set -uo pipefail |
| 15 | |
| 16 | HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 17 | # shellcheck source=lib/host.sh |
| 18 | source "$HERE/lib/host.sh" |
| 19 | # shellcheck source=lib/quiet_host.sh |
| 20 | source "$HERE/lib/quiet_host.sh" |
| 21 | |
| 22 | AUST_VENDOR="${AUST_VENDOR:?set AUST_VENDOR to the Austenite package dir}" |
| 23 | TYPST_VENDOR="${TYPST_VENDOR:?set TYPST_VENDOR to the typst.ts vendor dir}" |
| 24 | OUT="${OUT:?set OUT to the JSONL file}" |
| 25 | FMT="${FMT:-pdf}" |
| 26 | ROUNDS="${ROUNDS:-5}" |
| 27 | MODES="${MODES:-cold unchanged}" |
| 28 | PSI_LIMIT="${PSI_LIMIT:-20}" |
| 29 | GAP_S="${GAP_S:-12}" |
| 30 | DOC="${1:?usage: cost_runs.sh <doc.typ>}" |
| 31 | |
| 32 | DOCNAME="$(basename "$DOC" .typ)" |
| 33 | DOC_SHA="$(sha256sum "$DOC" | awk '{print $1}')" |
| 34 | mkdir -p "$(dirname "$OUT")" |
| 35 | |
| 36 | # One process under the locks. Globals: S_ENGINE, S_VENDOR, S_MODE, S_ROUND. |
| 37 | run_body() { |
| 38 | local stdout_f time_f warm=1 |
| 39 | [[ "$S_MODE" == cold ]] && warm=0 |
| 40 | stdout_f="$(mktemp)"; time_f="$(mktemp)" |
| 41 | local load_before psi_before load_after psi_after |
| 42 | load_before="$(loadavg1)"; psi_before="$(psi_avg10 cpu)" |
| 43 | systemd-run --user --scope --quiet -p MemoryMax=3G --slice=claude-rc.slice -- \ |
| 44 | /usr/bin/time -v -o "$time_f" -- \ |
| 45 | node "$HERE/wasm_bench.mjs" "--engine=$S_ENGINE" "--vendor=$S_VENDOR" "--doc=$DOC" \ |
| 46 | "--mode=$S_MODE" "--warmups=$warm" "--runs=1" "--fmt=$FMT" \ |
| 47 | > "$stdout_f" 2> /dev/null |
| 48 | local exit_code=$? |
| 49 | load_after="$(loadavg1)"; psi_after="$(psi_avg10 cpu)" |
| 50 | local rss_kb=0 |
| 51 | [[ -f "$time_f" ]] && rss_kb="$(awk -F': ' '/Maximum resident set size/{print $2}' "$time_f")" |
| 52 | [[ -z "$rss_kb" ]] && rss_kb=0 |
| 53 | local inner; inner="$(tail -n1 "$stdout_f" 2>/dev/null)" |
| 54 | [[ -z "$inner" ]] && inner='{"ok":false,"error":"no stdout from wasm_bench.mjs"}' |
| 55 | local flagged=false |
| 56 | if awk -v a="$psi_before" -v b="$psi_after" -v t="$PSI_LIMIT" 'BEGIN{exit !(a>t || b>t)}'; then |
| 57 | flagged=true |
| 58 | fi |
| 59 | jq -nc \ |
| 60 | --arg doc "$DOCNAME" --arg sha "$DOC_SHA" --arg engine "$S_ENGINE" --argjson round "$S_ROUND" \ |
| 61 | --argjson rss_kb "${rss_kb:-0}" --argjson exit_code "$exit_code" \ |
| 62 | --argjson load_before "$load_before" --argjson load_after "$load_after" \ |
| 63 | --argjson psi_before "$psi_before" --argjson psi_after "$psi_after" \ |
| 64 | --argjson flagged "$flagged" --arg ts "$(date -u +%FT%TZ)" --argjson inner "$inner" \ |
| 65 | '$inner * {ts:$ts, doc:$doc, doc_sha256:$sha, engine:$engine, round:$round, rss_kb:$rss_kb, |
| 66 | exit_code:$exit_code, load_before:$load_before, load_after:$load_after, |
| 67 | psi_before:$psi_before, psi_after:$psi_after, flagged:$flagged}' \ |
| 68 | >> "$OUT" |
| 69 | rm -f "$stdout_f" "$time_f" |
| 70 | } |
| 71 | |
| 72 | run_one() { |
| 73 | S_ENGINE="$1"; S_VENDOR="$2"; S_MODE="$3"; S_ROUND="$4" |
| 74 | with_quiet_host run_body |
| 75 | echo "[cost] $DOCNAME $FMT $S_MODE round $S_ROUND $S_ENGINE: $(tail -n1 "$OUT" | jq -c '{ok,runs,load_s,psi_before,psi_after,flagged}')" >&2 |
| 76 | sleep "$GAP_S" |
| 77 | } |
| 78 | |
| 79 | for mode in $MODES; do |
| 80 | for ((r = 0; r < ROUNDS; r++)); do |
| 81 | if ((r % 2 == 0)); then |
| 82 | run_one austenite "$AUST_VENDOR" "$mode" "$r" |
| 83 | run_one typstts "$TYPST_VENDOR" "$mode" "$r" |
| 84 | else |
| 85 | run_one typstts "$TYPST_VENDOR" "$mode" "$r" |
| 86 | run_one austenite "$AUST_VENDOR" "$mode" "$r" |
| 87 | fi |
| 88 | done |
| 89 | done |
| 90 | echo "$OUT" |