oxedyne/fe2o3/fe2o3_austenite/tools/bench/typing_sessions.sh
4.7 KiB, 1 run, executable
created by r1870400018:66254, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #!/usr/bin/env bash |
| 2 | # The typing test's sessions (D-20261001-08): ROUNDS rounds, each one session per engine, the |
| 3 | # engine that starts alternating from round to round (round 0 Austenite first). A session is one |
| 4 | # fresh node process (edit_latency.mjs), capped like every other run, holding both Austenite |
| 5 | # build-slot locks while it runs (lib/quiet_host.sh) and releasing them, with a pause, before the |
| 6 | # next, so the lane's builds run in the gaps. |
| 7 | # |
| 8 | # At each session's start and end the cpu PSI `some avg10` and the 1-minute load are read. A |
| 9 | # session with avg10 above PSI_LIMIT at either end is run once more, and a second flagged run |
| 10 | # is reported flagged. A session whose result is invalid (an error, a changed page count) is not |
| 11 | # repeated. Every attempt is appended to the JSONL, with its round and attempt number; the verdict |
| 12 | # script (bench_verdict.mjs typing) takes each round's last attempt. |
| 13 | # |
| 14 | # Usage: AUST_VENDOR=DIR TYPST_VENDOR=DIR OUT=FILE.jsonl typing_sessions.sh <doc.typ> [rounds] |
| 15 | set -uo pipefail |
| 16 | |
| 17 | HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 18 | # shellcheck source=lib/host.sh |
| 19 | source "$HERE/lib/host.sh" |
| 20 | # shellcheck source=lib/quiet_host.sh |
| 21 | source "$HERE/lib/quiet_host.sh" |
| 22 | |
| 23 | AUST_VENDOR="${AUST_VENDOR:?set AUST_VENDOR to the Austenite package dir}" |
| 24 | TYPST_VENDOR="${TYPST_VENDOR:?set TYPST_VENDOR to the typst.ts vendor dir}" |
| 25 | OUT="${OUT:?set OUT to the JSONL file}" |
| 26 | EDITS="${EDITS:-50}" |
| 27 | POSITIONS="${POSITIONS:-5}" |
| 28 | PSI_LIMIT="${PSI_LIMIT:-20}" |
| 29 | GAP_S="${GAP_S:-25}" # a gap longer than the slot helper's 10 s poll, so a waiting build gets in |
| 30 | DOC="${1:?usage: typing_sessions.sh <doc.typ> [rounds]}" |
| 31 | ROUNDS="${2:-3}" |
| 32 | |
| 33 | DOCNAME="$(basename "$DOC" .typ)" |
| 34 | DOC_SHA="$(sha256sum "$DOC" | awk '{print $1}')" |
| 35 | mkdir -p "$(dirname "$OUT")" |
| 36 | |
| 37 | # One attempt at one session, run inside the lock-holding subshell. Globals set by the caller: |
| 38 | # S_ENGINE, S_VENDOR, S_ROUND, S_ATTEMPT, S_T0 (epoch seconds when the lock was asked for). |
| 39 | session_body() { |
| 40 | local locked_s stdout_f time_f |
| 41 | locked_s="$(awk -v a="$S_T0" -v b="$(date +%s.%N)" 'BEGIN{printf "%.2f", b-a}')" |
| 42 | stdout_f="$(mktemp)"; time_f="$(mktemp)" |
| 43 | |
| 44 | local load_before psi_before load_after psi_after |
| 45 | load_before="$(loadavg1)"; psi_before="$(psi_avg10 cpu)" |
| 46 | local t0 t1 |
| 47 | t0=$(date +%s.%N) |
| 48 | systemd-run --user --scope --quiet -p MemoryMax=3G --slice=claude-rc.slice -- \ |
| 49 | /usr/bin/time -v -o "$time_f" -- \ |
| 50 | node "$HERE/edit_latency.mjs" "--engine=$S_ENGINE" "--vendor=$S_VENDOR" "--doc=$DOC" \ |
| 51 | "--edits=$EDITS" "--positions=$POSITIONS" \ |
| 52 | > "$stdout_f" 2> "$stdout_f.err" |
| 53 | local exit_code=$? |
| 54 | t1=$(date +%s.%N) |
| 55 | load_after="$(loadavg1)"; psi_after="$(psi_avg10 cpu)" |
| 56 | |
| 57 | local rss_kb=0 |
| 58 | [[ -f "$time_f" ]] && rss_kb="$(awk -F': ' '/Maximum resident set size/{print $2}' "$time_f")" |
| 59 | [[ -z "$rss_kb" ]] && rss_kb=0 |
| 60 | local inner; inner="$(tail -n1 "$stdout_f" 2>/dev/null)" |
| 61 | [[ -z "$inner" ]] && inner='{"ok":false,"error":"no stdout from edit_latency.mjs"}' |
| 62 | |
| 63 | local flagged=false |
| 64 | if awk -v a="$psi_before" -v b="$psi_after" -v t="$PSI_LIMIT" 'BEGIN{exit !(a>t || b>t)}'; then |
| 65 | flagged=true |
| 66 | fi |
| 67 | |
| 68 | jq -nc \ |
| 69 | --arg doc "$DOCNAME" --arg sha "$DOC_SHA" --arg engine "$S_ENGINE" \ |
| 70 | --argjson round "$S_ROUND" --argjson attempt "$S_ATTEMPT" \ |
| 71 | --argjson rss_kb "${rss_kb:-0}" --argjson exit_code "$exit_code" \ |
| 72 | --argjson load_before "$load_before" --argjson load_after "$load_after" \ |
| 73 | --argjson psi_before "$psi_before" --argjson psi_after "$psi_after" \ |
| 74 | --argjson flagged "$flagged" --argjson lock_wait_s "$locked_s" \ |
| 75 | --argjson wall_s "$(awk -v a="$t0" -v b="$t1" 'BEGIN{printf "%.2f", b-a}')" \ |
| 76 | --arg ts "$(date -u +%FT%TZ)" --argjson inner "$inner" \ |
| 77 | '$inner * {ts:$ts, doc:$doc, doc_sha256:$sha, engine:$engine, round:$round, attempt:$attempt, |
| 78 | rss_kb:$rss_kb, exit_code:$exit_code, load_before:$load_before, load_after:$load_after, |
| 79 | psi_before:$psi_before, psi_after:$psi_after, flagged:$flagged, |
| 80 | lock_wait_s:$lock_wait_s, wall_s:$wall_s}' \ |
| 81 | >> "$OUT" |
| 82 | rm -f "$stdout_f" "$stdout_f.err" "$time_f" |
| 83 | [[ "$flagged" == true ]] && return 3 |
| 84 | return 0 |
| 85 | } |
| 86 | |
| 87 | run_session() { |
| 88 | S_ENGINE="$1"; S_VENDOR="$2"; S_ROUND="$3" |
| 89 | S_ATTEMPT=1 |
| 90 | local rc |
| 91 | while :; do |
| 92 | S_T0="$(date +%s.%N)" |
| 93 | with_quiet_host session_body |
| 94 | rc=$? |
| 95 | echo "[typing] $DOCNAME round $S_ROUND $S_ENGINE attempt $S_ATTEMPT: $(tail -n1 "$OUT" | jq -c '{ok,p50_s,p95_s,psi_before,psi_after,load_before,load_after,flagged,lock_wait_s,wall_s}')" >&2 |
| 96 | if [[ $rc -eq 3 && $S_ATTEMPT -lt 2 ]]; then |
| 97 | S_ATTEMPT=$((S_ATTEMPT + 1)) |
| 98 | sleep "$GAP_S" |
| 99 | continue |
| 100 | fi |
| 101 | break |
| 102 | done |
| 103 | sleep "$GAP_S" |
| 104 | } |
| 105 | |
| 106 | for ((r = 0; r < ROUNDS; r++)); do |
| 107 | if ((r % 2 == 0)); then |
| 108 | run_session austenite "$AUST_VENDOR" "$r" |
| 109 | run_session typstts "$TYPST_VENDOR" "$r" |
| 110 | else |
| 111 | run_session typstts "$TYPST_VENDOR" "$r" |
| 112 | run_session austenite "$AUST_VENDOR" "$r" |
| 113 | fi |
| 114 | done |
| 115 | echo "$OUT" |