oxedyne/fe2o3/fe2o3_austenite/tools/bench/corpus/parity_diff.py
2.4 KiB, 1 run
created by r1870400018:73737, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #!/usr/bin/env python3 |
| 2 | # Compares two parity.py JSONL runs row by row: regressions (a row equivalent or near in the base that is |
| 3 | # now worse, equivalent to near included), gains, verdict moves and the compile, equivalent and near shares per tier. |
| 4 | # Usage: parity_diff.py BASE.jsonl NEW.jsonl [--moves]. Exits 1 on any regression. |
| 5 | import collections |
| 6 | import json |
| 7 | import sys |
| 8 | |
| 9 | RANK = {"equivalent": 0, "near": 1, "differs": 2, "aus_fails": 3} |
| 10 | KEEP = ("equivalent", "near") |
| 11 | |
| 12 | |
| 13 | def load(path): |
| 14 | rows = {} |
| 15 | for l in open(path): |
| 16 | try: |
| 17 | r = json.loads(l) |
| 18 | except ValueError: |
| 19 | continue |
| 20 | rows[r["id"]] = r |
| 21 | return rows |
| 22 | |
| 23 | |
| 24 | def shares(rows, tier): |
| 25 | c = collections.Counter(r["verdict"] for r in rows.values() if r["tier"] == tier) |
| 26 | base = sum(c[v] for v in RANK) |
| 27 | comp = c["equivalent"] + c["near"] + c["differs"] |
| 28 | return base, comp, c["equivalent"], c["near"] |
| 29 | |
| 30 | |
| 31 | def pct(n, d): |
| 32 | return "%.1f%%" % (100.0 * n / d) if d else "-" |
| 33 | |
| 34 | |
| 35 | def main(argv): |
| 36 | if len(argv) < 3: |
| 37 | print("usage: parity_diff.py BASE.jsonl NEW.jsonl [--moves]") |
| 38 | return 2 |
| 39 | a = load(argv[1]) |
| 40 | b = load(argv[2]) |
| 41 | moves = "--moves" in argv |
| 42 | tiers = sorted(set(r["tier"] for r in a.values()) | set(r["tier"] for r in b.values())) |
| 43 | print("| Tier | Measure | Base | New | Delta |") |
| 44 | print("|---|---|---:|---:|---:|") |
| 45 | for t in tiers: |
| 46 | ba, ca, ea, na = shares(a, t) |
| 47 | bb, cb, eb, nb = shares(b, t) |
| 48 | if not (ba or bb): |
| 49 | continue |
| 50 | for name, x, y in (("compile", ca, cb), ("equivalent", ea, eb), ("near", na, nb)): |
| 51 | print("| %s | %s | %d/%d %s | %d/%d %s | %+d |" % (t, name, x, ba, pct(x, ba), y, bb, pct(y, bb), y - x)) |
| 52 | regs = [] |
| 53 | gains = [] |
| 54 | mv = collections.Counter() |
| 55 | for k in sorted(set(a) & set(b)): |
| 56 | va = a[k]["verdict"] |
| 57 | vb = b[k]["verdict"] |
| 58 | if va == vb: |
| 59 | continue |
| 60 | mv[(va, vb)] += 1 |
| 61 | if va in KEEP and RANK.get(vb, 9) > RANK[va]: |
| 62 | regs.append((k, va, vb)) |
| 63 | elif vb in KEEP and va not in KEEP: |
| 64 | gains.append((k, va, vb)) |
| 65 | if moves: |
| 66 | print("move %s: %s -> %s" % (k, va, vb)) |
| 67 | print("\nVerdict moves:") |
| 68 | for (va, vb), n in sorted(mv.items(), key=lambda x: -x[1]): |
| 69 | print(" %s -> %s: %d" % (va, vb, n)) |
| 70 | print("\nGains into equivalent or near: %d" % len(gains)) |
| 71 | for k, va, vb in gains: |
| 72 | print(" + %s: %s -> %s" % (k, va, vb)) |
| 73 | print("Regressions out of equivalent or near: %d" % len(regs)) |
| 74 | for k, va, vb in regs: |
| 75 | print(" - %s: %s -> %s" % (k, va, vb)) |
| 76 | only = (set(a) ^ set(b)) |
| 77 | if only: |
| 78 | print("Rows in one run only: %d" % len(only)) |
| 79 | return 1 if regs else 0 |
| 80 | |
| 81 | |
| 82 | if __name__ == "__main__": |
| 83 | sys.exit(main(sys.argv)) |