oxedyne/fe2o3/fe2o3_austenite/tools/bench/corpus/parity.py
14.2 KiB, 1 run
created by r1870400018:72510, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #!/usr/bin/env python3 |
| 2 | # Render parity over the whole corpus (FIRE-3, 2026-10-11): every document in real_corpus.txt is copied once |
| 3 | # to a work directory and compiled from that one copy by Austenite's evaluator (`--eval`) and by typst |
| 4 | # 0.15.1, each under its own memory scope and wall-time cap. The two PDFs are compared by page count, by |
| 5 | # `pdftotext -layout` text (runs of spaces collapsed, then the count of differing lines) and by raster (every |
| 6 | # page up to MAXPAGES at DPI, `compare -metric AE -fuzz 10%`, the worst page's fraction). One JSONL row per |
| 7 | # document. A crash, hang or out-of-memory is recorded and the run moves on. A document already in the JSONL |
| 8 | # is skipped, so a killed run is simply started again. |
| 9 | # |
| 10 | # Verdicts: equivalent (same pages, no text line differs, worst raster <= RASTER_EQ), near (same pages, text |
| 11 | # equal, raster above it), differs, aus_fails, typst_fails (counted apart from the shares). `cause` keys an |
| 12 | # aus_fails row by the construct its diagnostic names, so failures can be ranked (report_parity.py). |
| 13 | # |
| 14 | # Run it detached as one job: |
| 15 | # systemd-run --user --scope --quiet -p MemoryMax=4G --slice=claude-rc.slice --unit=<name> \ |
| 16 | # python3 -I tools/bench/corpus/parity.py |
| 17 | # |
| 18 | # Environment (defaults): W (~/.cache/austenite/claude-rc-7/parity), BIN (~/.local/bin/austenite), TYPST |
| 19 | # (~/bin/typst), LIST (real_corpus.txt beside this script), OUT ($W/parity.jsonl), TMO (120 s per engine), |
| 20 | # MEM (3G per engine), TIERS ("universe fixture tree"), ONLY (a file of ids, one per line), LIMIT (0: all), |
| 21 | # DPI (50), MAXPAGES (20), RASTER_EQ (0.001). |
| 22 | import difflib |
| 23 | import json |
| 24 | import os |
| 25 | import re |
| 26 | import shutil |
| 27 | import subprocess |
| 28 | import sys |
| 29 | import time |
| 30 | import unicodedata |
| 31 | |
| 32 | HERE = os.path.dirname(os.path.abspath(__file__)) |
| 33 | CRATE = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) |
| 34 | REPO = os.path.dirname(CRATE) |
| 35 | HOME = os.path.expanduser("~") |
| 36 | W = os.environ.get("W", os.path.join(HOME, ".cache/austenite/claude-rc-7/parity")) |
| 37 | BIN = os.environ.get("BIN", os.path.join(HOME, ".local/bin/austenite")) |
| 38 | TYPST = os.environ.get("TYPST", os.path.join(HOME, "bin/typst")) |
| 39 | LIST = os.environ.get("LIST", os.path.join(HERE, "real_corpus.txt")) |
| 40 | OUT = os.environ.get("OUT", os.path.join(W, "parity.jsonl")) |
| 41 | TMO = int(os.environ.get("TMO", "120")) |
| 42 | MEM = os.environ.get("MEM", "3G") |
| 43 | TIERS = os.environ.get("TIERS", "universe fixture tree").split() |
| 44 | ONLY = os.environ.get("ONLY", "") |
| 45 | LIMIT = int(os.environ.get("LIMIT", "0")) |
| 46 | DPI = os.environ.get("DPI", "50") |
| 47 | MAXP = int(os.environ.get("MAXPAGES", "20")) |
| 48 | RASTER_EQ = float(os.environ.get("RASTER_EQ", "0.001")) |
| 49 | os.environ.setdefault("TYPST_PACKAGE_CACHE_PATH", os.path.join(W, "pkgs")) |
| 50 | |
| 51 | LOG = os.path.join(W, "parity.log") |
| 52 | |
| 53 | |
| 54 | def log(msg): |
| 55 | with open(LOG, "a") as f: |
| 56 | f.write("%s %s\n" % (time.strftime("%T"), msg)) |
| 57 | |
| 58 | |
| 59 | def capped(cmd, cwd, out_prefix): |
| 60 | # One engine run under its own memory scope and wall-time cap; returns (exit code, seconds). |
| 61 | t0 = time.time() |
| 62 | with open(out_prefix + ".out", "w") as o, open(out_prefix + ".err", "w") as e: |
| 63 | rc = subprocess.call( |
| 64 | ["systemd-run", "--user", "--scope", "--quiet", "-p", "MemoryMax=" + MEM, "-p", "MemorySwapMax=0", |
| 65 | "--slice=claude-rc.slice", "timeout", "-k", "5", str(TMO)] + cmd, |
| 66 | cwd=cwd, stdout=o, stderr=e) |
| 67 | return rc, round(time.time() - t0, 2) |
| 68 | |
| 69 | |
| 70 | def read(p, limit=None): |
| 71 | try: |
| 72 | with open(p, errors="replace") as f: |
| 73 | return f.read() if limit is None else f.read(limit) |
| 74 | except OSError: |
| 75 | return "" |
| 76 | |
| 77 | |
| 78 | ANSI = re.compile(r"\x1b\[[0-9;]*m") |
| 79 | |
| 80 | |
| 81 | def first_error(prefix, rc): |
| 82 | # The first line that names an error, else the last non-empty line, of stderr then stdout. |
| 83 | if rc == 124 or rc == 137: |
| 84 | return "timeout after %d s" % TMO if rc == 124 else "killed (memory cap %s or timeout)" % MEM |
| 85 | text = ANSI.sub("", read(prefix + ".err") + "\n" + read(prefix + ".out")) |
| 86 | lines = [l.strip() for l in text.splitlines() if l.strip()] |
| 87 | for l in lines: |
| 88 | if re.search(r"\berror\b|panicked|Error|ERR", l): |
| 89 | return l[:400] |
| 90 | return (lines[-1] if lines else "")[:400] |
| 91 | |
| 92 | |
| 93 | def pages(pdf): |
| 94 | try: |
| 95 | info = subprocess.run(["pdfinfo", pdf], capture_output=True, text=True, timeout=60).stdout |
| 96 | except (subprocess.TimeoutExpired, OSError): |
| 97 | return None |
| 98 | m = re.search(r"^Pages:\s+(\d+)", info, re.M) |
| 99 | return int(m.group(1)) if m else None |
| 100 | |
| 101 | |
| 102 | SPACES = re.compile(r" +") |
| 103 | |
| 104 | |
| 105 | def text_lines(pdf): |
| 106 | try: |
| 107 | t = subprocess.run(["pdftotext", "-layout", pdf, "-"], capture_output=True, text=True, |
| 108 | errors="replace", timeout=120).stdout |
| 109 | except (subprocess.TimeoutExpired, OSError): |
| 110 | return None |
| 111 | # A page break is a line break; runs of spaces collapse; trailing space is not a difference. |
| 112 | # Invisible format characters (word joiner, zero-width space, soft hyphen; Unicode category Cf) draw |
| 113 | # nothing, so one engine emitting them and the other not is not a text difference. |
| 114 | t = "".join(c for c in t.replace("\f", "\n") if unicodedata.category(c) != "Cf") |
| 115 | return [SPACES.sub(" ", l).strip() for l in t.split("\n")] |
| 116 | |
| 117 | |
| 118 | def text_diff(a, b): |
| 119 | if a is None or b is None: |
| 120 | return None |
| 121 | n = 0 |
| 122 | for tag, i1, i2, j1, j2 in difflib.SequenceMatcher(None, a, b, autojunk=False).get_opcodes(): |
| 123 | if tag != "equal": |
| 124 | n += max(i2 - i1, j2 - j1) |
| 125 | return n |
| 126 | |
| 127 | |
| 128 | def raster(pdf, pfx, npages): |
| 129 | subprocess.run(["pdftoppm", "-r", DPI, "-png", "-l", str(min(npages, MAXP)), pdf, pfx], |
| 130 | capture_output=True, timeout=300) |
| 131 | |
| 132 | |
| 133 | def page_pngs(d, pfx): |
| 134 | out = {} |
| 135 | for f in os.listdir(d): |
| 136 | m = re.match(re.escape(pfx) + r"-0*(\d+)\.png$", f) |
| 137 | if m: |
| 138 | out[int(m.group(1))] = os.path.join(d, f) |
| 139 | return out |
| 140 | |
| 141 | |
| 142 | def png_size(p): |
| 143 | with open(p, "rb") as f: |
| 144 | h = f.read(24) |
| 145 | return int.from_bytes(h[16:20], "big"), int.from_bytes(h[20:24], "big") |
| 146 | |
| 147 | |
| 148 | def page_diff(a, b): |
| 149 | # Fraction of pixels differing beyond 10% fuzz; 1.0 when the page sizes differ. |
| 150 | wa, ha = png_size(a) |
| 151 | wb, hb = png_size(b) |
| 152 | if (wa, ha) != (wb, hb): |
| 153 | return 1.0 |
| 154 | r = subprocess.run(["compare", "-metric", "AE", "-fuzz", "10%", a, b, "null:"], |
| 155 | capture_output=True, text=True, timeout=120) |
| 156 | m = re.match(r"\s*([0-9.e+]+)", r.stderr) |
| 157 | if not m: |
| 158 | return 1.0 |
| 159 | return float(m.group(1)) / float(wa * ha) |
| 160 | |
| 161 | |
| 162 | # Cause keys. The evaluator's diagnostic names the construct; the key is the construct, not the site. |
| 163 | CAUSES = [ |
| 164 | (r"bibliography style `?[^`]*`? is not supported", "unsupported: bibliography style (CSL)"), |
| 165 | (r"type `?([\w.-]+)`? has no method `?([\w.-]+)", "no method: {}.{}"), |
| 166 | (r"`?([\w.-]+)`? does not contain field `?([\w.-]+)", "no field: {}.{}"), |
| 167 | (r"(expected [^,]+(?:, [^,]+)*,? found [\w.-]+)", "type mismatch: {}"), |
| 168 | (r"unknown variable:? `?([\w.-]+)", "unknown variable: {}"), |
| 169 | (r"unknown function:? `?([\w.-]+)", "unknown function: {}"), |
| 170 | (r"unknown field:? `?([\w.-]+)`?", "unknown field: {}"), |
| 171 | (r"(?:type|value) `?([\w.-]+)`? (?:has|does not have) (?:no )?(?:field|method) `?([\w.-]+)", "no field: {}.{}"), |
| 172 | (r"(?:no|unknown) (?:field|method) `?([\w.-]+)`? (?:on|of|for) `?([\w.-]+)", "no field: {1}.{0}"), |
| 173 | (r"unsupported element:? `?([\w.-]+)", "unsupported element: {}"), |
| 174 | (r"element `?([\w.-]+)`? (?:is )?(?:not supported|unsupported)", "unsupported element: {}"), |
| 175 | (r"unknown (?:argument|parameter):? `?([\w.-]+)", "unknown argument: {}"), |
| 176 | (r"unexpected argument:? `?([\w.-]+)", "unexpected argument: {}"), |
| 177 | (r"package `?(@[\w/.:-]+)`?", "package: {}"), |
| 178 | (r"(?:cannot|could not|failed to) (?:read|open|find|load) (?:file )?`?([^`:\s]+)", "file: {}"), |
| 179 | ] |
| 180 | |
| 181 | |
| 182 | def norm_cause(err, rc): |
| 183 | if rc == 124: |
| 184 | return "timeout" |
| 185 | if rc == 137: |
| 186 | return "killed (memory or timeout)" |
| 187 | if "panicked" in err: |
| 188 | m = re.search(r"panicked at ([^:]+)", err) |
| 189 | return "panic: %s" % (m.group(1) if m else "?") |
| 190 | for pat, fmt in CAUSES: |
| 191 | m = re.search(pat, err, re.I) |
| 192 | if m: |
| 193 | g = m.groups() |
| 194 | name = fmt.format(*g) |
| 195 | if name.startswith("file: "): |
| 196 | name = "file: " + os.path.splitext(name[6:])[1] |
| 197 | return name |
| 198 | # The error class: the message with its names, numbers and quotes dropped. |
| 199 | e = re.sub(r"^.*?error:?\s*", "", err, flags=re.I) |
| 200 | e = re.sub(r"`[^`]*`|\"[^\"]*\"|'[^']*'", "_", e) |
| 201 | e = re.sub(r"\d+", "N", e) |
| 202 | e = re.sub(r"\s+at\s+\S+$", "", e) |
| 203 | return ("class: " + e.strip())[:120] |
| 204 | |
| 205 | |
| 206 | SITE = re.compile(r"(@[\w-]+/[\w.-]+:[\d.]+)/|(?:^|\s)(/\S+?\.typ):\d+") |
| 207 | |
| 208 | |
| 209 | def cause_site(err): |
| 210 | # Where the failing construct sits: a package (name:version) or "document" for the template's own files. |
| 211 | m = SITE.search(err or "") |
| 212 | if not m: |
| 213 | return None |
| 214 | return m.group(1) if m.group(1) else "document" |
| 215 | |
| 216 | |
| 217 | def stage(tier, ident, spec, work): |
| 218 | # Copies the document's tree into work/src; returns (root, source, fonts) or raises RuntimeError. |
| 219 | src_dir = os.path.join(work, "src") |
| 220 | if tier == "universe": |
| 221 | cache = os.path.join(W, "src", "universe", ident.replace(":", "-")) |
| 222 | if not os.path.isfile(os.path.join(cache, ".staged")): |
| 223 | shutil.rmtree(cache, ignore_errors=True) |
| 224 | os.makedirs(os.path.dirname(cache), exist_ok=True) |
| 225 | r = subprocess.run([TYPST, "init", "@preview/" + ident, cache], capture_output=True, text=True, |
| 226 | timeout=180) |
| 227 | if r.returncode != 0: |
| 228 | shutil.rmtree(cache, ignore_errors=True) |
| 229 | raise RuntimeError("typst init: " + " ".join(r.stderr.split())[:300]) |
| 230 | open(os.path.join(cache, ".staged"), "w").close() |
| 231 | shutil.copytree(cache, src_dir, symlinks=True, ignore=shutil.ignore_patterns(".staged")) |
| 232 | source = os.path.join(src_dir, spec) |
| 233 | else: |
| 234 | path = os.path.join(REPO, spec) |
| 235 | shutil.copytree(os.path.dirname(path), src_dir, symlinks=True, ignore=shutil.ignore_patterns("*.pdf")) |
| 236 | source = os.path.join(src_dir, os.path.basename(path)) |
| 237 | if not os.path.isfile(source): |
| 238 | raise RuntimeError("no source " + spec) |
| 239 | fonts = None |
| 240 | for dp, _, fs in os.walk(src_dir): |
| 241 | if any(f.lower().endswith((".ttf", ".otf", ".ttc")) for f in fs): |
| 242 | fonts = src_dir |
| 243 | break |
| 244 | return src_dir, source, fonts |
| 245 | |
| 246 | |
| 247 | def measure(tier, ident, spec, extra): |
| 248 | row = {"id": ident, "tier": tier, "spec": spec, "extra": extra} |
| 249 | work = os.path.join(W, "run", "cur") |
| 250 | shutil.rmtree(work, ignore_errors=True) |
| 251 | os.makedirs(work) |
| 252 | try: |
| 253 | root, source, fonts = stage(tier, ident, spec, work) |
| 254 | except (RuntimeError, OSError, subprocess.TimeoutExpired) as e: |
| 255 | row.update({"typst_ok": False, "typst_rc": None, "typst_err": "stage: " + str(e)[:300], |
| 256 | "aus_ok": None, "verdict": "typst_fails", "stage_fail": True}) |
| 257 | return row |
| 258 | fa = ["--font-path", fonts] if fonts else [] |
| 259 | tpdf = os.path.join(work, "t.pdf") |
| 260 | adir = os.path.join(work, "ao") |
| 261 | # typst first: it fetches the template's imports into the package cache Austenite then reads. |
| 262 | trc, tsec = capped([TYPST, "compile", "--ignore-system-fonts", "--root", root] + fa + [source, tpdf], |
| 263 | os.path.dirname(source), os.path.join(work, "t")) |
| 264 | arc, asec = capped([BIN, "--eval", "--root", root] + fa + [source, adir], os.path.dirname(source), |
| 265 | os.path.join(work, "a")) |
| 266 | apdf = os.path.join(adir, "document.pdf") |
| 267 | if not os.path.isfile(apdf): |
| 268 | cands = [f for f in os.listdir(adir) if f.endswith(".pdf")] if os.path.isdir(adir) else [] |
| 269 | apdf = os.path.join(adir, cands[0]) if cands else apdf |
| 270 | t_ok = trc == 0 and os.path.isfile(tpdf) |
| 271 | a_ok = arc == 0 and os.path.isfile(apdf) |
| 272 | row.update({ |
| 273 | "typst_ok": t_ok, "typst_rc": trc, "typst_s": tsec, "typst_err": None if t_ok else first_error(os.path.join(work, "t"), trc), |
| 274 | "aus_ok": a_ok, "aus_rc": arc, "aus_s": asec, "aus_err": None if a_ok else first_error(os.path.join(work, "a"), arc), |
| 275 | }) |
| 276 | if not t_ok: |
| 277 | row["verdict"] = "typst_fails" |
| 278 | return row |
| 279 | row["typst_pages"] = pages(tpdf) |
| 280 | if not a_ok: |
| 281 | row["verdict"] = "aus_fails" |
| 282 | row["cause"] = norm_cause(row["aus_err"], arc) |
| 283 | row["cause_site"] = cause_site(row["aus_err"]) |
| 284 | return row |
| 285 | row["aus_pages"] = pages(apdf) |
| 286 | tl = text_lines(tpdf) |
| 287 | al = text_lines(apdf) |
| 288 | row["text_lines"] = len(tl) if tl is not None else None |
| 289 | row["text_diff"] = text_diff(tl, al) |
| 290 | if tl is not None and al is not None and row["text_diff"]: |
| 291 | # The first differing pair, for the report's sample. |
| 292 | for tag, i1, i2, j1, j2 in difflib.SequenceMatcher(None, tl, al, autojunk=False).get_opcodes(): |
| 293 | if tag != "equal": |
| 294 | row["text_first"] = {"line": i1 + 1, "typst": " | ".join(tl[i1:i2][:2])[:200], |
| 295 | "aus": " | ".join(al[j1:j2][:2])[:200]} |
| 296 | break |
| 297 | worst, worst_p, npages = None, None, min(row["typst_pages"] or 0, row["aus_pages"] or 0) |
| 298 | if npages: |
| 299 | raster(tpdf, os.path.join(work, "t"), npages) |
| 300 | raster(apdf, os.path.join(work, "a"), npages) |
| 301 | tp = page_pngs(work, "t") |
| 302 | ap = page_pngs(work, "a") |
| 303 | for p in sorted(tp): |
| 304 | if p in ap: |
| 305 | try: |
| 306 | d = page_diff(tp[p], ap[p]) |
| 307 | except (OSError, subprocess.TimeoutExpired): |
| 308 | d = 1.0 |
| 309 | if worst is None or d > worst: |
| 310 | worst, worst_p = d, p |
| 311 | row["raster_worst"] = None if worst is None else round(worst, 6) |
| 312 | row["raster_page"] = worst_p |
| 313 | same = row["typst_pages"] == row["aus_pages"] and row["typst_pages"] is not None |
| 314 | if same and row["text_diff"] == 0 and worst is not None and worst <= RASTER_EQ: |
| 315 | row["verdict"] = "equivalent" |
| 316 | elif same and row["text_diff"] == 0: |
| 317 | row["verdict"] = "near" |
| 318 | else: |
| 319 | row["verdict"] = "differs" |
| 320 | if row["verdict"] == "differs" and worst_p is not None: |
| 321 | keep = os.path.join(W, "keep", re.sub(r"[^\w.-]", "_", ident)) |
| 322 | os.makedirs(keep, exist_ok=True) |
| 323 | for pfx, pp in (("t", tp), ("a", ap)): |
| 324 | shutil.copy(pp[worst_p], os.path.join(keep, "%s-%d.png" % (pfx, worst_p))) |
| 325 | return row |
| 326 | |
| 327 | |
| 328 | def main(): |
| 329 | os.makedirs(W, exist_ok=True) |
| 330 | os.makedirs(os.environ["TYPST_PACKAGE_CACHE_PATH"], exist_ok=True) |
| 331 | done = set() |
| 332 | if os.path.isfile(OUT): |
| 333 | for l in open(OUT): |
| 334 | try: |
| 335 | done.add(json.loads(l)["id"]) |
| 336 | except (ValueError, KeyError): |
| 337 | pass |
| 338 | want = None |
| 339 | if ONLY: |
| 340 | want = set(x.strip() for x in open(ONLY) if x.strip()) |
| 341 | tv = subprocess.run([TYPST, "--version"], capture_output=True, text=True).stdout.strip() |
| 342 | log("parity start: list %s, out %s, bin %s, %s" % (LIST, OUT, BIN, tv)) |
| 343 | n = 0 |
| 344 | for line in open(LIST): |
| 345 | f = line.rstrip("\n").split("\t") |
| 346 | if not f[0] or f[0].startswith("#") or len(f) < 3 or f[0] not in TIERS: |
| 347 | continue |
| 348 | tier, ident, spec = f[0], f[1], f[2] |
| 349 | extra = f[3] if len(f) > 3 else "" |
| 350 | if want is not None and ident not in want: |
| 351 | continue |
| 352 | if ident in done: |
| 353 | continue |
| 354 | if LIMIT and n >= LIMIT: |
| 355 | break |
| 356 | n += 1 |
| 357 | t0 = time.time() |
| 358 | try: |
| 359 | row = measure(tier, ident, spec, extra) |
| 360 | except Exception as e: |
| 361 | row = {"id": ident, "tier": tier, "spec": spec, "verdict": "harness_error", "harness_err": repr(e)[:300]} |
| 362 | with open(OUT, "a") as o: |
| 363 | o.write(json.dumps(row) + "\n") |
| 364 | log("%d %s %s %s %.1fs" % (n, tier, ident, row.get("verdict"), time.time() - t0)) |
| 365 | shutil.rmtree(os.path.join(W, "run", "cur"), ignore_errors=True) |
| 366 | log("parity end: %d documents" % n) |
| 367 | |
| 368 | |
| 369 | if __name__ == "__main__": |
| 370 | main() |