oxedyne/fe2o3/fe2o3_austenite/tools/bench/corpus/report_parity.py
3.0 KiB, 1 run
created by r1870400018:72520, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #!/usr/bin/env python3 |
| 2 | # Summarises parity.py's JSONL: counts and shares by tier and verdict, the aus_fails causes ranked by the |
| 3 | # number of documents each breaks, and a sample of differs rows. Usage: report_parity.py PARITY.jsonl [N]. |
| 4 | import collections |
| 5 | import json |
| 6 | import sys |
| 7 | |
| 8 | rows = {} |
| 9 | for l in open(sys.argv[1]): |
| 10 | try: |
| 11 | r = json.loads(l) |
| 12 | except ValueError: |
| 13 | continue |
| 14 | rows[r["id"]] = r |
| 15 | top = int(sys.argv[2]) if len(sys.argv) > 2 else 25 |
| 16 | V = ["equivalent", "near", "differs", "aus_fails", "typst_fails", "harness_error"] |
| 17 | |
| 18 | |
| 19 | def table(sel, name): |
| 20 | c = collections.Counter(r["verdict"] for r in sel) |
| 21 | base = sum(c[v] for v in V[:4]) |
| 22 | comp = c["equivalent"] + c["near"] + c["differs"] |
| 23 | print("\n### %s (%d documents; %d where typst compiles)\n" % (name, len(sel), base)) |
| 24 | print("| Measure | Count | Share of typst-compiling |") |
| 25 | print("|---|---:|---:|") |
| 26 | pct = lambda n: "%.1f%%" % (100.0 * n / base) if base else "-" |
| 27 | print("| Austenite compiles | %d | %s |" % (comp, pct(comp))) |
| 28 | for v in V[:4]: |
| 29 | print("| %s | %d | %s |" % (v, c[v], pct(c[v]))) |
| 30 | print("| typst_fails (apart) | %d | - |" % c["typst_fails"]) |
| 31 | if c["harness_error"]: |
| 32 | print("| harness_error (apart) | %d | - |" % c["harness_error"]) |
| 33 | |
| 34 | |
| 35 | allr = list(rows.values()) |
| 36 | table(allr, "All tiers") |
| 37 | for t in ("universe", "fixture", "tree"): |
| 38 | sel = [r for r in allr if r["tier"] == t] |
| 39 | if sel: |
| 40 | table(sel, "Tier " + t) |
| 41 | |
| 42 | print("\n### aus_fails causes, ranked by documents broken\n") |
| 43 | print("| Rank | Cause | All | Universe | Where (top sites) |") |
| 44 | print("|---:|---|---:|---:|---|") |
| 45 | cz = collections.defaultdict(list) |
| 46 | for r in allr: |
| 47 | if r["verdict"] == "aus_fails": |
| 48 | cz[r.get("cause") or "?"].append(r) |
| 49 | for i, (k, rs) in enumerate(sorted(cz.items(), key=lambda kv: -len(kv[1]))[:top], 1): |
| 50 | u = sum(1 for r in rs if r["tier"] == "universe") |
| 51 | sites = collections.Counter(r.get("cause_site") or "?" for r in rs).most_common(3) |
| 52 | where = ", ".join("%s %d" % s for s in sites) |
| 53 | print("| %d | `%s` | %d | %d | %s |" % (i, k.replace("|", "/"), len(rs), u, where)) |
| 54 | |
| 55 | print("\n### differs: what differs\n") |
| 56 | d = [r for r in allr if r["verdict"] == "differs"] |
| 57 | pg = sum(1 for r in d if r.get("typst_pages") != r.get("aus_pages")) |
| 58 | tx = sum(1 for r in d if r.get("text_diff")) |
| 59 | print("%d rows: %d with a different page count, %d with text lines differing.\n" % (len(d), pg, tx)) |
| 60 | print("| Document | Pages typst/aus | Text lines differing / typst lines | Worst raster | First text difference (typst / aus) |") |
| 61 | print("|---|---|---|---:|---|") |
| 62 | step = max(1, len(d) // 10) |
| 63 | def at_split(a, b): |
| 64 | # The two lines from just before where they first part. |
| 65 | i = 0 |
| 66 | while i < min(len(a), len(b)) and a[i] == b[i]: |
| 67 | i += 1 |
| 68 | s = max(0, i - 25) |
| 69 | return a[s:s + 60], b[s:s + 60] |
| 70 | |
| 71 | |
| 72 | for r in d[::step][:10]: |
| 73 | tf = dict(r.get("text_first") or {}) |
| 74 | tf["typst"], tf["aus"] = at_split(tf.get("typst") or "", tf.get("aus") or "") |
| 75 | print("| %s | %s/%s | %s/%s | %s | %s / %s |" % (r["id"], r.get("typst_pages"), r.get("aus_pages"), |
| 76 | r.get("text_diff"), r.get("text_lines"), r.get("raster_worst"), |
| 77 | tf["typst"].replace("|", "/"), tf["aus"].replace("|", "/"))) |