oxedyne/fe2o3/fe2o3_austenite/tools/bench/run_bench.sh
4.0 KiB, 1 run, executable
created by r1870400018:60948, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #!/usr/bin/env bash |
| 2 | # S0 bench harness top-level driver. Runs the native comparison |
| 3 | # (typst -j16 / -j1 vs the austenite binary) and, when the vendored wasm is |
| 4 | # present, the wasm comparison (Austenite wasm vs typst.ts wasm in node) and |
| 5 | # the incremental edit-latency bench, on the sample docs plus a generated |
| 6 | # 300-page synthetic document, then aggregates everything into one JSON |
| 7 | # report and one markdown summary. |
| 8 | # |
| 9 | # Defaults to SMOKE mode: 1 warm-up, 3 measured runs, a small synthetic doc, |
| 10 | # few edits. Pass --full for the real S0 protocol (2 warm-ups, 7 runs, the |
| 11 | # 300-page doc, 50 edits at 5 positions) -- run that only when the host is |
| 12 | # idle; see aggregate.py for the load gate this cannot be talked past. |
| 13 | # |
| 14 | # Usage: run_bench.sh --austenite-bin PATH [--full] [--force] |
| 15 | # [--out DIR] [--docs 'glob'] [--skip-wasm] |
| 16 | set -uo pipefail |
| 17 | |
| 18 | HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 19 | AUSTENITE_ROOT="$(cd "$HERE/../.." && pwd)" # fe2o3_austenite/ |
| 20 | # shellcheck source=lib/host.sh |
| 21 | source "$HERE/lib/host.sh" |
| 22 | |
| 23 | AUSTENITE_BIN="" |
| 24 | FULL=false |
| 25 | FORCE=false |
| 26 | OUT_DIR="$HERE/out/$(date -u +%Y%m%dT%H%M%SZ)" |
| 27 | SKIP_WASM=false |
| 28 | DOCS_GLOB="" |
| 29 | |
| 30 | # Daimond's vendored wasm -- see tools/bench/README.md for how these paths |
| 31 | # were found; both are hand-shipped, not part of any build, so their absence |
| 32 | # is normal on a fresh checkout and just means the wasm legs are skipped. |
| 33 | DAIMOND_WWW="$HOME/usr/code/web/apps/oxedyne/daimond/www" |
| 34 | AUST_VENDOR="$DAIMOND_WWW/vendor/austenite" |
| 35 | TYPST_VENDOR="$DAIMOND_WWW/vendor/typst" |
| 36 | |
| 37 | while [[ $# -gt 0 ]]; do |
| 38 | case "$1" in |
| 39 | --austenite-bin) AUSTENITE_BIN="$2"; shift 2 ;; |
| 40 | --full) FULL=true; shift ;; |
| 41 | --force) FORCE=true; shift ;; |
| 42 | --out) OUT_DIR="$2"; shift 2 ;; |
| 43 | --docs) DOCS_GLOB="$2"; shift 2 ;; |
| 44 | --skip-wasm) SKIP_WASM=true; shift ;; |
| 45 | *) echo "run_bench.sh: unknown argument $1" >&2; exit 2 ;; |
| 46 | esac |
| 47 | done |
| 48 | |
| 49 | if [[ -z "$AUSTENITE_BIN" || ! -x "$AUSTENITE_BIN" ]]; then |
| 50 | echo "run_bench.sh: --austenite-bin must point at a built, executable austenite binary" >&2 |
| 51 | exit 2 |
| 52 | fi |
| 53 | |
| 54 | mkdir -p "$OUT_DIR" |
| 55 | export OUT_DIR AUSTENITE_BIN AUST_VENDOR TYPST_VENDOR |
| 56 | |
| 57 | if $FULL; then |
| 58 | export WARMUPS=2 RUNS=7 EDITS=50 POSITIONS=5 |
| 59 | SYN_PAGES=300 |
| 60 | else |
| 61 | export WARMUPS=1 RUNS=3 EDITS=6 POSITIONS=2 |
| 62 | SYN_PAGES=20 |
| 63 | fi |
| 64 | |
| 65 | # Under-load notice up front -- the harness still runs (so a smoke test can |
| 66 | # prove the plumbing works), but aggregate.py refuses to print the numbers |
| 67 | # as a baseline unless --force is given, and every markdown/json output says so. |
| 68 | if host_is_loaded; then |
| 69 | echo "[run_bench] HOST IS LOADED (1-min load average $(loadavg1) > $BENCH_LOAD_THRESHOLD)." >&2 |
| 70 | echo "[run_bench] proceeding (this run is for plumbing, not a baseline) -- runs will be flagged." >&2 |
| 71 | fi |
| 72 | |
| 73 | # Docs: the fixed samples plus one generated synthetic doc. |
| 74 | DOCS=() |
| 75 | if [[ -n "$DOCS_GLOB" ]]; then |
| 76 | # shellcheck disable=SC2086 |
| 77 | for f in $DOCS_GLOB; do DOCS+=("$f"); done |
| 78 | else |
| 79 | for f in "$AUSTENITE_ROOT"/samples/*.typ; do DOCS+=("$f"); done |
| 80 | SYN="$OUT_DIR/synthetic_${SYN_PAGES}.typ" |
| 81 | python3 "$HERE/gen_synthetic.py" "$SYN_PAGES" "$SYN" |
| 82 | DOCS+=("$SYN") |
| 83 | fi |
| 84 | echo "[run_bench] docs: ${DOCS[*]}" >&2 |
| 85 | |
| 86 | # Native leg. |
| 87 | bash "$HERE/speed_bench.sh" "${DOCS[@]}" |
| 88 | |
| 89 | # Wasm legs, only where the vendored copies exist (they are hand-shipped, not |
| 90 | # built by this repo -- see tools/bench/README.md). |
| 91 | if ! $SKIP_WASM && [[ -f "$AUST_VENDOR/oxedyne_fe2o3_austenite_bg.wasm" && \ |
| 92 | -f "$TYPST_VENDOR/typst_ts_web_compiler_bg.wasm" ]]; then |
| 93 | bash "$HERE/wasm_bench_runner.sh" "${DOCS[@]}" |
| 94 | bash "$HERE/edit_latency_runner.sh" "${DOCS[@]}" |
| 95 | else |
| 96 | echo "[run_bench] skipping wasm legs: vendored wasm not found under $DAIMOND_WWW/vendor" >&2 |
| 97 | fi |
| 98 | |
| 99 | LABEL="mode=$($FULL && echo full || echo smoke); host load 1-min before finish: $(loadavg1); $(date -u +%FT%TZ)" |
| 100 | FORCE_FLAG=() |
| 101 | $FORCE && FORCE_FLAG=(--force) |
| 102 | python3 "$HERE/aggregate.py" --in "$OUT_DIR" \ |
| 103 | --out-json "$OUT_DIR/bench_report.json" --out-md "$OUT_DIR/bench_report.md" \ |
| 104 | --label "$LABEL" "${FORCE_FLAG[@]}" |
| 105 | |
| 106 | echo "[run_bench] report: $OUT_DIR/bench_report.md" >&2 |
| 107 | echo "$OUT_DIR" |