Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tools/bench/typing_sessions.sh

4.7 KiB, 1 run, executable

created by r1870400018:66254, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1#!/usr/bin/env bash
2# The typing test's sessions (D-20261001-08): ROUNDS rounds, each one session per engine, the
3# engine that starts alternating from round to round (round 0 Austenite first). A session is one
4# fresh node process (edit_latency.mjs), capped like every other run, holding both Austenite
5# build-slot locks while it runs (lib/quiet_host.sh) and releasing them, with a pause, before the
6# next, so the lane's builds run in the gaps.
7#
8# At each session's start and end the cpu PSI `some avg10` and the 1-minute load are read. A
9# session with avg10 above PSI_LIMIT at either end is run once more, and a second flagged run
10# is reported flagged. A session whose result is invalid (an error, a changed page count) is not
11# repeated. Every attempt is appended to the JSONL, with its round and attempt number; the verdict
12# script (bench_verdict.mjs typing) takes each round's last attempt.
13#
14# Usage: AUST_VENDOR=DIR TYPST_VENDOR=DIR OUT=FILE.jsonl typing_sessions.sh <doc.typ> [rounds]
15set -uo pipefail
16
17HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
18# shellcheck source=lib/host.sh
19source "$HERE/lib/host.sh"
20# shellcheck source=lib/quiet_host.sh
21source "$HERE/lib/quiet_host.sh"
22
23AUST_VENDOR="${AUST_VENDOR:?set AUST_VENDOR to the Austenite package dir}"
24TYPST_VENDOR="${TYPST_VENDOR:?set TYPST_VENDOR to the typst.ts vendor dir}"
25OUT="${OUT:?set OUT to the JSONL file}"
26EDITS="${EDITS:-50}"
27POSITIONS="${POSITIONS:-5}"
28PSI_LIMIT="${PSI_LIMIT:-20}"
29GAP_S="${GAP_S:-25}" # a gap longer than the slot helper's 10 s poll, so a waiting build gets in
30DOC="${1:?usage: typing_sessions.sh <doc.typ> [rounds]}"
31ROUNDS="${2:-3}"
32
33DOCNAME="$(basename "$DOC" .typ)"
34DOC_SHA="$(sha256sum "$DOC" | awk '{print $1}')"
35mkdir -p "$(dirname "$OUT")"
36
37# One attempt at one session, run inside the lock-holding subshell. Globals set by the caller:
38# S_ENGINE, S_VENDOR, S_ROUND, S_ATTEMPT, S_T0 (epoch seconds when the lock was asked for).
39session_body() {
40 local locked_s stdout_f time_f
41 locked_s="$(awk -v a="$S_T0" -v b="$(date +%s.%N)" 'BEGIN{printf "%.2f", b-a}')"
42 stdout_f="$(mktemp)"; time_f="$(mktemp)"
43
44 local load_before psi_before load_after psi_after
45 load_before="$(loadavg1)"; psi_before="$(psi_avg10 cpu)"
46 local t0 t1
47 t0=$(date +%s.%N)
48 systemd-run --user --scope --quiet -p MemoryMax=3G --slice=claude-rc.slice -- \
49 /usr/bin/time -v -o "$time_f" -- \
50 node "$HERE/edit_latency.mjs" "--engine=$S_ENGINE" "--vendor=$S_VENDOR" "--doc=$DOC" \
51 "--edits=$EDITS" "--positions=$POSITIONS" \
52 > "$stdout_f" 2> "$stdout_f.err"
53 local exit_code=$?
54 t1=$(date +%s.%N)
55 load_after="$(loadavg1)"; psi_after="$(psi_avg10 cpu)"
56
57 local rss_kb=0
58 [[ -f "$time_f" ]] && rss_kb="$(awk -F': ' '/Maximum resident set size/{print $2}' "$time_f")"
59 [[ -z "$rss_kb" ]] && rss_kb=0
60 local inner; inner="$(tail -n1 "$stdout_f" 2>/dev/null)"
61 [[ -z "$inner" ]] && inner='{"ok":false,"error":"no stdout from edit_latency.mjs"}'
62
63 local flagged=false
64 if awk -v a="$psi_before" -v b="$psi_after" -v t="$PSI_LIMIT" 'BEGIN{exit !(a>t || b>t)}'; then
65 flagged=true
66 fi
67
68 jq -nc \
69 --arg doc "$DOCNAME" --arg sha "$DOC_SHA" --arg engine "$S_ENGINE" \
70 --argjson round "$S_ROUND" --argjson attempt "$S_ATTEMPT" \
71 --argjson rss_kb "${rss_kb:-0}" --argjson exit_code "$exit_code" \
72 --argjson load_before "$load_before" --argjson load_after "$load_after" \
73 --argjson psi_before "$psi_before" --argjson psi_after "$psi_after" \
74 --argjson flagged "$flagged" --argjson lock_wait_s "$locked_s" \
75 --argjson wall_s "$(awk -v a="$t0" -v b="$t1" 'BEGIN{printf "%.2f", b-a}')" \
76 --arg ts "$(date -u +%FT%TZ)" --argjson inner "$inner" \
77 '$inner * {ts:$ts, doc:$doc, doc_sha256:$sha, engine:$engine, round:$round, attempt:$attempt,
78 rss_kb:$rss_kb, exit_code:$exit_code, load_before:$load_before, load_after:$load_after,
79 psi_before:$psi_before, psi_after:$psi_after, flagged:$flagged,
80 lock_wait_s:$lock_wait_s, wall_s:$wall_s}' \
81 >> "$OUT"
82 rm -f "$stdout_f" "$stdout_f.err" "$time_f"
83 [[ "$flagged" == true ]] && return 3
84 return 0
85}
86
87run_session() {
88 S_ENGINE="$1"; S_VENDOR="$2"; S_ROUND="$3"
89 S_ATTEMPT=1
90 local rc
91 while :; do
92 S_T0="$(date +%s.%N)"
93 with_quiet_host session_body
94 rc=$?
95 echo "[typing] $DOCNAME round $S_ROUND $S_ENGINE attempt $S_ATTEMPT: $(tail -n1 "$OUT" | jq -c '{ok,p50_s,p95_s,psi_before,psi_after,load_before,load_after,flagged,lock_wait_s,wall_s}')" >&2
96 if [[ $rc -eq 3 && $S_ATTEMPT -lt 2 ]]; then
97 S_ATTEMPT=$((S_ATTEMPT + 1))
98 sleep "$GAP_S"
99 continue
100 fi
101 break
102 done
103 sleep "$GAP_S"
104}
105
106for ((r = 0; r < ROUNDS; r++)); do
107 if ((r % 2 == 0)); then
108 run_session austenite "$AUST_VENDOR" "$r"
109 run_session typstts "$TYPST_VENDOR" "$r"
110 else
111 run_session typstts "$TYPST_VENDOR" "$r"
112 run_session austenite "$AUST_VENDOR" "$r"
113 fi
114done
115echo "$OUT"