oxedyne/fe2o3/fe2o3_austenite/tools/bench/corpus/report.py
9.4 KiB, 1 run, executable
created by r1870400018:72517, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #!/usr/bin/env python3 |
| 2 | """Summarises the sweep's JSONL (sweep.sh) as markdown: robustness, speed ratios, the worst documents, the |
| 3 | stage ranking and the warm edit. A later file's record for an id replaces an earlier one, so a rerun file |
| 4 | goes last. |
| 5 | |
| 6 | Usage: report.py [--rerun-ids FILE] [--fail-tsv FILE] <sweep.jsonl> [rerun.jsonl]... |
| 7 | --rerun-ids writes the ids to measure again with RUNS=4: the slowest 10 by ratio, every document over 3x, |
| 8 | and every document with a run at a load above 6. |
| 9 | --fail-tsv writes every Austenite failure: id, tier, status, typst status, message. |
| 10 | """ |
| 11 | |
| 12 | import json |
| 13 | import statistics |
| 14 | import sys |
| 15 | |
| 16 | STAGES = ("load", "eval", "realise", "layout", "emit") |
| 17 | DOCS = ("universe", "syn", "own", "tree") # the headline set; fixtures are reported apart |
| 18 | |
| 19 | |
| 20 | def pct(xs, p): |
| 21 | xs = sorted(xs) |
| 22 | if not xs: |
| 23 | return float("nan") |
| 24 | k = (len(xs) - 1) * p |
| 25 | f = int(k) |
| 26 | c = min(f + 1, len(xs) - 1) |
| 27 | return xs[f] + (xs[c] - xs[f]) * (k - f) |
| 28 | |
| 29 | |
| 30 | def load_max(r): |
| 31 | m = 0.0 |
| 32 | for e in ("aus", "typ"): |
| 33 | for run in (r.get(e) or {}).get("runs", []): |
| 34 | try: |
| 35 | m = max(m, float(run.get("load") or 0)) |
| 36 | except ValueError: |
| 37 | pass |
| 38 | return m |
| 39 | |
| 40 | |
| 41 | def excess(r): |
| 42 | a = (r["aus"].get("stages") or {}) |
| 43 | t = (r["typ"].get("stages") or {}) |
| 44 | if not a or not t: |
| 45 | return None |
| 46 | return {s: a.get(s, 0.0) - t.get(s, 0.0) for s in STAGES} |
| 47 | |
| 48 | |
| 49 | def dominant(r): |
| 50 | x = excess(r) |
| 51 | if x: |
| 52 | s = max(x, key=x.get) |
| 53 | return s, x[s] |
| 54 | a = r["aus"].get("stages") or {} |
| 55 | if a: |
| 56 | s = max(STAGES, key=lambda k: a.get(k, 0.0)) |
| 57 | return s + " (share)", a.get(s, 0.0) |
| 58 | return "-", 0.0 |
| 59 | |
| 60 | |
| 61 | def ratio_stats(rs): |
| 62 | xs = [r["ratio"] for r in rs] |
| 63 | if not xs: |
| 64 | return "n=0" |
| 65 | w = max(rs, key=lambda r: r["ratio"]) |
| 66 | return (f"n={len(xs)}, median {statistics.median(xs):.2f}x, p90 {pct(xs, 0.9):.2f}x, " |
| 67 | f"worst {w['ratio']:.2f}x ({w['id']}), geomean {statistics.geometric_mean(xs):.2f}x") |
| 68 | |
| 69 | |
| 70 | def main(): |
| 71 | args = sys.argv[1:] |
| 72 | rerun_out = fail_out = None |
| 73 | while args and args[0].startswith("--"): |
| 74 | flag = args.pop(0) |
| 75 | if flag == "--rerun-ids": |
| 76 | rerun_out = args.pop(0) |
| 77 | elif flag == "--fail-tsv": |
| 78 | fail_out = args.pop(0) |
| 79 | recs = {} |
| 80 | for path in args: |
| 81 | for line in open(path): |
| 82 | try: |
| 83 | r = json.loads(line) |
| 84 | except ValueError: |
| 85 | continue |
| 86 | recs[r["id"]] = r |
| 87 | allr = list(recs.values()) |
| 88 | staged = [r for r in allr if "aus" in r] |
| 89 | print(f"# Sweep report: {len(allr)} records ({len(allr) - len(staged)} could not be staged)\n") |
| 90 | |
| 91 | # Robustness. |
| 92 | print("## Robustness\n") |
| 93 | print("| tier | docs | aus ok | aus error | panic | timeout | killed/OOM | typst ok | both ok | pages differ |") |
| 94 | print("|---|---|---|---|---|---|---|---|---|---|") |
| 95 | tiers = sorted({r["tier"] for r in allr}) |
| 96 | for tier in tiers: |
| 97 | rs = [r for r in staged if r["tier"] == tier] |
| 98 | st = lambda r: r["aus"]["status"] |
| 99 | cnt = lambda s: sum(1 for r in rs if st(r) == s) |
| 100 | both = [r for r in rs if st(r) == "ok" and r["typ"]["status"] == "ok"] |
| 101 | diff = sum(1 for r in both if r.get("pages") and r.get("typst_pages") and r["pages"] != r["typst_pages"]) |
| 102 | nstage = sum(1 for r in allr if r["tier"] == tier and "aus" not in r) |
| 103 | sig = sum(1 for r in rs if st(r).startswith("signal")) |
| 104 | print(f"| {tier} | {len(rs)}{f' (+{nstage} unstaged)' if nstage else ''} | {cnt('ok')} | {cnt('error')} | " |
| 105 | f"{cnt('panic') + sig} | {cnt('timeout')} | {cnt('killed')} | " |
| 106 | f"{sum(1 for r in rs if r['typ']['status'] == 'ok')} | {len(both)} | {diff} |") |
| 107 | crash = [r for r in staged if r["aus"]["status"] in ("panic", "timeout", "killed") or r["aus"]["status"].startswith("signal")] |
| 108 | print(f"\n### Every Austenite crash, hang or kill ({len(crash)})\n") |
| 109 | for r in sorted(crash, key=lambda r: (r["aus"]["status"], r["id"])): |
| 110 | print(f"- `{r['id']}` ({r['tier']}): **{r['aus']['status']}**, typst {r['typ']['status']}: {r['aus'].get('msg', '')}") |
| 111 | wfail = [r for r in staged if r.get("warm", {}).get("status") == "fail"] |
| 112 | print(f"\n### Warm-edit failures ({len(wfail)})\n") |
| 113 | for r in wfail[:40]: |
| 114 | print(f"- `{r['id']}`: rc {r['warm'].get('rc')}: {r['warm'].get('msg', '')[:200]}") |
| 115 | # Errors where typst succeeds, grouped by message head. |
| 116 | refused = [r for r in staged if r["aus"]["status"] == "error" and r["typ"]["status"] == "ok"] |
| 117 | groups = {} |
| 118 | for r in refused: |
| 119 | m = r["aus"].get("msg", "") |
| 120 | key = m.split(" | ")[-1][:110] |
| 121 | groups.setdefault(key, []).append(r["id"]) |
| 122 | print(f"\n### Austenite refuses, typst compiles ({len(refused)}), top messages\n") |
| 123 | for k, ids in sorted(groups.items(), key=lambda kv: -len(kv[1]))[:25]: |
| 124 | print(f"- {len(ids)} x `{k}` (e.g. {', '.join(ids[:3])})") |
| 125 | fx = [r for r in staged if r["tier"] == "fixture"] |
| 126 | agree = sum(1 for r in fx if (r["aus"]["status"] == "ok") == (r["typ"]["status"] == "ok")) |
| 127 | print(f"\nFixtures: Austenite and typst agree on accept/reject for {agree} of {len(fx)}.") |
| 128 | |
| 129 | # Speed. |
| 130 | print("\n## Speed (cold, Austenite wall / typst -j1 wall, median of runs)\n") |
| 131 | both = [r for r in staged if r.get("ratio")] |
| 132 | docs = [r for r in both if r["tier"] in DOCS] |
| 133 | print(f"- **Documents (universe, syn, own, tree): {ratio_stats(docs)}**") |
| 134 | big = [r for r in docs if r["typ"]["wall"] >= 0.5] |
| 135 | print(f"- Documents where typst takes >= 0.5 s: {ratio_stats(big)}") |
| 136 | for tier in tiers: |
| 137 | print(f"- {tier}: {ratio_stats([r for r in both if r['tier'] == tier])}") |
| 138 | tot_a = sum(r["aus"]["wall"] for r in docs) |
| 139 | tot_t = sum(r["typ"]["wall"] for r in docs) |
| 140 | print(f"- Summed wall over documents: Austenite {tot_a:.1f} s, typst {tot_t:.1f} s ({tot_a / max(tot_t, 1e-9):.2f}x)") |
| 141 | over = [r for r in docs if r["ratio"] > 3] |
| 142 | print(f"- Documents over 3x: {len(over)}; over 1.5x: {sum(1 for r in docs if r['ratio'] > 1.5)}") |
| 143 | |
| 144 | print("\n### The 20 worst documents by ratio\n") |
| 145 | print("| doc | tier | pages | typst s | Austenite s | ratio | passes | dominant stage (excess s) | warm s / passes / view s | load max |") |
| 146 | print("|---|---|---|---|---|---|---|---|---|---|") |
| 147 | worst = sorted(docs, key=lambda r: -r["ratio"])[:20] |
| 148 | for r in worst: |
| 149 | dom, ex = dominant(r) |
| 150 | w = r.get("warm") or {} |
| 151 | ws = f"{w.get('compile_s', '-')} / {w.get('passes', '-')} / {w.get('save_to_view_s', '-')}" if "mode" in w else "-" |
| 152 | st = r["aus"].get("stages") or {} |
| 153 | print(f"| `{r['id']}` | {r['tier']} | {r.get('pages', '-')} | {r['typ']['wall']:.3f} | {r['aus']['wall']:.3f} | " |
| 154 | f"{r['ratio']:.2f}x | {st.get('passes', '-')} | {dom} ({ex:+.3f}) | {ws} | {load_max(r):.1f} |") |
| 155 | |
| 156 | print("\n### The 10 slowest documents by Austenite wall\n") |
| 157 | for r in sorted(docs, key=lambda r: -r["aus"]["wall"])[:10]: |
| 158 | dom, ex = dominant(r) |
| 159 | print(f"- `{r['id']}`: {r['aus']['wall']:.2f} s against {r['typ']['wall']:.2f} s ({r['ratio']:.2f}x), " |
| 160 | f"{r.get('pages', '-')} pages, dominant {dom} ({ex:+.2f} s)") |
| 161 | |
| 162 | # Stage ranking. |
| 163 | print("\n## Stages ranked by total excess over typst (documents with both stage splits)\n") |
| 164 | tot = dict.fromkeys(STAGES, 0.0) |
| 165 | pos = dict.fromkeys(STAGES, 0.0) |
| 166 | dom_n = dict.fromkeys(STAGES, 0) |
| 167 | a_tot = dict.fromkeys(STAGES, 0.0) |
| 168 | t_tot = dict.fromkeys(STAGES, 0.0) |
| 169 | n = 0 |
| 170 | for r in docs: |
| 171 | x = excess(r) |
| 172 | if not x: |
| 173 | continue |
| 174 | n += 1 |
| 175 | for s in STAGES: |
| 176 | tot[s] += x[s] |
| 177 | pos[s] += max(x[s], 0.0) |
| 178 | a_tot[s] += r["aus"]["stages"].get(s, 0.0) |
| 179 | t_tot[s] += r["typ"]["stages"].get(s, 0.0) |
| 180 | if r["ratio"] > 1: |
| 181 | dom_n[max(x, key=x.get)] += 1 |
| 182 | print(f"n={n}. Excess = Austenite stage time less typst's, summed over documents.\n") |
| 183 | print("| stage | Austenite s | typst s | net excess s | positive excess s | dominant in (docs slower than typst) |") |
| 184 | print("|---|---|---|---|---|---|") |
| 185 | for s in sorted(STAGES, key=lambda s: -pos[s]): |
| 186 | print(f"| {s} | {a_tot[s]:.1f} | {t_tot[s]:.1f} | {tot[s]:+.1f} | {pos[s]:.1f} | {dom_n[s]} |") |
| 187 | flow = sum((r["aus"].get("stages") or {}).get("flow", 0.0) for r in docs if excess(r)) |
| 188 | print(f"\nOf Austenite's layout, flow (which holds line breaking) is {flow:.1f} s.") |
| 189 | |
| 190 | # Warm edit. |
| 191 | ws = [r for r in docs if "mode" in (r.get("warm") or {})] |
| 192 | if ws: |
| 193 | cs = [r["warm"]["compile_s"] for r in ws] |
| 194 | vs = [r["warm"]["save_to_view_s"] for r in ws] |
| 195 | passes = {} |
| 196 | for r in ws: |
| 197 | passes[r["warm"]["passes"]] = passes.get(r["warm"]["passes"], 0) + 1 |
| 198 | cold = sum(1 for r in ws if r["warm"]["mode"] != "warm") |
| 199 | print(f"\n## Warm one-letter edit (n={len(ws)}; {cold} ran cold)\n") |
| 200 | print(f"- compile: median {statistics.median(cs):.3f} s, p90 {pct(cs, 0.9):.3f} s, worst {max(cs):.3f} s") |
| 201 | print(f"- save-to-view: median {statistics.median(vs):.3f} s, p90 {pct(vs, 0.9):.3f} s, worst {max(vs):.3f} s " |
| 202 | f"({max(ws, key=lambda r: r['warm']['save_to_view_s'])['id']})") |
| 203 | print(f"- passes: {dict(sorted(passes.items(), key=lambda kv: (kv[0] is None, kv[0] or 0)))}") |
| 204 | for r in sorted(ws, key=lambda r: -r["warm"]["save_to_view_s"])[:5]: |
| 205 | print(f" - `{r['id']}`: view {r['warm']['save_to_view_s']} s, compile {r['warm']['compile_s']} s, " |
| 206 | f"{r['warm']['passes']} passes, cold {r['aus']['wall']:.2f} s") |
| 207 | |
| 208 | hi = [r for r in both if load_max(r) > 6] |
| 209 | print(f"\nRuns measured at a load above 6: {len(hi)} documents.") |
| 210 | |
| 211 | if rerun_out: |
| 212 | ids = {r["id"] for r in sorted(docs, key=lambda r: -r["ratio"])[:10]} |
| 213 | ids |= {r["id"] for r in docs if r["ratio"] > 3} |
| 214 | ids |= {r["id"] for r in hi if r["tier"] in DOCS} |
| 215 | with open(rerun_out, "w") as f: |
| 216 | for i in sorted(ids): |
| 217 | f.write(i + "\n") |
| 218 | if fail_out: |
| 219 | with open(fail_out, "w") as f: |
| 220 | for r in staged: |
| 221 | if r["aus"]["status"] != "ok": |
| 222 | f.write("\t".join([r["id"], r["tier"], r["aus"]["status"], r["typ"]["status"], |
| 223 | r["aus"].get("msg", "").replace("\t", " ")]) + "\n") |
| 224 | for r in allr: |
| 225 | if "aus" not in r: |
| 226 | f.write("\t".join([r["id"], r["tier"], "unstaged", "-", r.get("stage_err", "")]) + "\n") |
| 227 | |
| 228 | |
| 229 | if __name__ == "__main__": |
| 230 | main() |