oxedyne/fe2o3/fe2o3_austenite/tools/gate/g_checks.py
22.0 KiB, 1 run, executable
created by r1870400018:66260, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #!/usr/bin/env python3 |
| 2 | """The gate's checks: do two PDFs, one Typst's and one Austenite's, agree? |
| 3 | |
| 4 | g_checks.py compare <a.pdf> <b.pdf> G3, the text of every page (also `g3`) |
| 5 | g_checks.py pages <x.pdf> the per-page hashes of G3, for one PDF |
| 6 | g_checks.py g1 <a.pdf> <b.pdf> page count and each page's MediaBox |
| 7 | g_checks.py g2 <a.pdf> <b.pdf> the set of font base names, and that every font is embedded |
| 8 | and has a Unicode map |
| 9 | g_checks.py g4 <a.pdf> <b.pdf> outline count, level sequence and the titles |
| 10 | g_checks.py g5 <a.pdf> <b.pdf> Title, Author, Subject and Keywords |
| 11 | g_checks.py g7 <a.pdf> <b.pdf> pages rasterised at 50 dpi, the fraction of pixels that differ |
| 12 | (needs GATE_SCRATCH, a directory that exists) |
| 13 | g_checks.py g8 <a.pdf> <b.pdf> the character count of each page, within 1% |
| 14 | g_checks.py g12 <cli1> <cli2> <door1> <door2> <door3> |
| 15 | two CLI PDFs, two from one door instance, one from another |
| 16 | g_checks.py g9 <a.pdf> <b.pdf> where the text sits on each page: lines, blocks, words, extents, |
| 17 | line pitch, word height, images and links, then line by line |
| 18 | |
| 19 | `a` is the oracle's PDF (Typst's) and `b` the PDF under test. G9 is measured, with no verdict: it carries no word of |
| 20 | a page, only where the words sit, from `pdftotext -bbox-layout`. In `compare` and `pages` a page is its `pdftotext |
| 21 | -raw` text, normalised to NFKC and split on whitespace, with the tokens that are only punctuation dropped. Page i |
| 22 | of one PDF is compared with page i of the other by the 12-hex hash of its first token, the 12-hex hash of its |
| 23 | last, and the difflib ratio over its token lists. Text is read from a pipe and never written to disk, and |
| 24 | nothing of a document is printed: every line is built from numbers, hashes and the fixed words below, so the |
| 25 | output is safe for a document whose text must not be logged. A font name is printed only when it matches |
| 26 | `[A-Za-z0-9+-]+`; the pages of G7 are rasterised into a directory made under GATE_SCRATCH and removed at once. |
| 27 | |
| 28 | Every check ends in a `verdict` line and exits 0 for `same`, 1 for `differs`; on any failure it prints |
| 29 | `g<N> error <unreadable|usage|scratch>` and exits 2. |
| 30 | |
| 31 | g3 pages a <n> b <n> |
| 32 | g3 differ <n> pages <ranges|none> (pages whose first or last hash differs, 1-based, e.g. 3-5,9) |
| 33 | g3 min-similarity <d.ddd|none> (over the pages both have) |
| 34 | g3 verdict <same|differs> (differs on a count, a hash or a similarity below 1) |
| 35 | g3 page <i> first <hex12> last <hex12> tokens <n> (`pages`, one per page) |
| 36 | |
| 37 | g1 pages a <n> b <n> |
| 38 | g1 mediabox differ <n> pages <ranges|none> (a side is within 0.01 pt of the other on all four numbers) |
| 39 | g1 mediabox max-deviation <d.dddd> (pt, over the pages both have) |
| 40 | g1 verdict <same|differs> |
| 41 | |
| 42 | g2 fonts a <n> b <n> |
| 43 | g2 only-a <name> (a base name, subset tag and `-Identity-H` stripped, one per font) |
| 44 | g2 only-b <name> |
| 45 | g2 flags a <n> b <n> (fonts that are not `emb yes` and `uni yes`) |
| 46 | g2 verdict <same|differs> |
| 47 | |
| 48 | g4 outline a <n> b <n> (entries, every level) |
| 49 | g4 levels <equal|differs> first-diff <i|none> (the depth of each entry in document order) |
| 50 | g4 titles <equal|differs> unmatched a <n> b <n> (the multiset of title sha256) |
| 51 | g4 titles position differ <n> of <m> (title i against title i) |
| 52 | g4 titles prefix-only <n> (of those, one is the other after a leading number or label) |
| 53 | g4 verdict <same|differs> (the count, the levels and every title at its place) |
| 54 | |
| 55 | g5 <title|author|subject|keywords> <equal|differs|absent-both> (by sha256 of the Info entry) |
| 56 | g5 verdict <same|differs> |
| 57 | |
| 58 | g7 compared <n> of a <n> b <n> |
| 59 | g7 page <i> <d.dddd> (the fraction of pixels that differ by more than 16/255 in a channel) |
| 60 | g7 max <d.dddd> |
| 61 | g7 median <d.dddd> |
| 62 | |
| 63 | g8 pages a <n> b <n> |
| 64 | g8 out-of-tolerance <n> pages <ranges|none> |
| 65 | g8 page <i> a <n> b <n> (each page out of tolerance: its counts of non-space characters) |
| 66 | g8 max-deviation <d.dddd> (|b - a| / a, with 1.0000 where a is 0 and b is not) |
| 67 | g8 empty <n> pages <ranges|none> (b has no text where a has some) |
| 68 | g8 verdict <same|differs> |
| 69 | |
| 70 | g9 pages differ <n> pages <ranges|none> (a count or an extent below differs) |
| 71 | g9 page <i> <lines|blocks|words|sizes|images|links> a <n> b <n> (counts; sizes is the distinct word heights) |
| 72 | g9 page <i> <left|right|top|bottom|pitch|height> a <d.dd> b <d.dd> (pt; pitch is the median gap between lines |
| 73 | of a block, height the median word height; 0.5 pt, pitch 0.1 pt) |
| 74 | g9 line <i> <j> y a <d.dd> b <d.dd> x a <d.dd> b <d.dd> w a <d.dd> b <d.dd> |
| 75 | (line j of page i where its top, left or width differs by 0.5 pt; |
| 76 | the first 14 such lines of each of the first 5 pages that differ in |
| 77 | more than their links) |
| 78 | |
| 79 | g12 cli-twice <equal|differs|absent> |
| 80 | g12 door-one-instance <equal|differs|absent> |
| 81 | g12 door-two-instances <equal|differs|absent> |
| 82 | g12 cli-door <equal|differs|absent> (reported, not required) |
| 83 | g12 size cli <n> door <n> |
| 84 | g12 verdict <same|differs> (the first three; an absent file differs) |
| 85 | """ |
| 86 | |
| 87 | import difflib |
| 88 | import glob |
| 89 | import hashlib |
| 90 | import os |
| 91 | import re |
| 92 | import statistics |
| 93 | import subprocess |
| 94 | import sys |
| 95 | import tempfile |
| 96 | import unicodedata |
| 97 | |
| 98 | NONE = '0' * 12 # the hash of the first or last token of a page with no tokens |
| 99 | |
| 100 | |
| 101 | def page_texts(pdf): |
| 102 | """The text of each page of `pdf`, from pdftotext's pipe.""" |
| 103 | r = subprocess.run(['pdftotext', '-raw', pdf, '-'], capture_output=True) |
| 104 | if r.returncode != 0: |
| 105 | raise OSError('pdftotext') |
| 106 | pages = r.stdout.decode('utf-8', 'replace').split('\f') |
| 107 | # pdftotext ends every page, the last included, with a form feed, so one empty piece trails. |
| 108 | if pages and pages[-1] == '': |
| 109 | pages.pop() |
| 110 | return pages |
| 111 | |
| 112 | |
| 113 | def is_punct(tok): |
| 114 | return all(unicodedata.category(c).startswith('P') for c in tok) |
| 115 | |
| 116 | |
| 117 | def tokens(text): |
| 118 | """NFKC, split on whitespace, the tokens that are only punctuation dropped.""" |
| 119 | return [t for t in unicodedata.normalize('NFKC', text).split() if not is_punct(t)] |
| 120 | |
| 121 | |
| 122 | def h12(tok): |
| 123 | return hashlib.sha256(tok.encode('utf-8')).hexdigest()[:12] |
| 124 | |
| 125 | |
| 126 | def summary(toks): |
| 127 | """The hashes of a page's first and last token.""" |
| 128 | if not toks: |
| 129 | return NONE, NONE |
| 130 | return h12(toks[0]), h12(toks[-1]) |
| 131 | |
| 132 | |
| 133 | def ratio(ta, tb): |
| 134 | if not ta and not tb: |
| 135 | return 1.0 |
| 136 | return difflib.SequenceMatcher(None, ta, tb, autojunk=False).ratio() |
| 137 | |
| 138 | |
| 139 | def ranges(idx): |
| 140 | """1-based indices as `3-5,9`, or `none`.""" |
| 141 | if not idx: |
| 142 | return 'none' |
| 143 | out = [] |
| 144 | start = prev = idx[0] |
| 145 | for i in idx[1:] + [None]: |
| 146 | if i is not None and i == prev + 1: |
| 147 | prev = i |
| 148 | continue |
| 149 | out.append(str(start) if start == prev else '%d-%d' % (start, prev)) |
| 150 | if i is not None: |
| 151 | start = prev = i |
| 152 | return ','.join(out) |
| 153 | |
| 154 | |
| 155 | def compare_pages(pa, pb): |
| 156 | """The pages of `pa` and `pb`, each a list of tokens, compared page for page. |
| 157 | |
| 158 | Returns (differing 1-based page indices, minimum similarity or None). |
| 159 | """ |
| 160 | n = min(len(pa), len(pb)) |
| 161 | differ = [] |
| 162 | low = None |
| 163 | for i in range(n): |
| 164 | a, b = pa[i], pb[i] |
| 165 | if summary(a) != summary(b): |
| 166 | differ.append(i + 1) |
| 167 | r = ratio(a, b) |
| 168 | low = r if low is None else min(low, r) |
| 169 | return differ, low |
| 170 | |
| 171 | |
| 172 | def cmd_compare(a_pdf, b_pdf): |
| 173 | pa = [tokens(t) for t in page_texts(a_pdf)] |
| 174 | pb = [tokens(t) for t in page_texts(b_pdf)] |
| 175 | differ, low = compare_pages(pa, pb) |
| 176 | print('g3 pages a %d b %d' % (len(pa), len(pb))) |
| 177 | print('g3 differ %d pages %s' % (len(differ), ranges(differ))) |
| 178 | print('g3 min-similarity %s' % ('none' if low is None else '%.3f' % low)) |
| 179 | same = len(pa) == len(pb) and not differ and (low is None or low >= 1.0) |
| 180 | print('g3 verdict %s' % ('same' if same else 'differs')) |
| 181 | return 0 if same else 1 |
| 182 | |
| 183 | |
| 184 | def cmd_pages(pdf): |
| 185 | for i, t in enumerate(page_texts(pdf)): |
| 186 | toks = tokens(t) |
| 187 | first, last = summary(toks) |
| 188 | print('g3 page %d first %s last %s tokens %d' % (i + 1, first, last, len(toks))) |
| 189 | return 0 |
| 190 | |
| 191 | |
| 192 | # ── G1, G2, G4, G5, G7, G8 and G12 ─────────────────────────────────────────── |
| 193 | |
| 194 | def pdf_open(path): |
| 195 | import pikepdf |
| 196 | return pikepdf.open(path) |
| 197 | |
| 198 | |
| 199 | def mediaboxes(path): |
| 200 | """The MediaBox of each page, as four floats.""" |
| 201 | with pdf_open(path) as pdf: |
| 202 | return [[float(x) for x in page.mediabox] for page in pdf.pages] |
| 203 | |
| 204 | |
| 205 | def cmd_g1(a_pdf, b_pdf): |
| 206 | ma = mediaboxes(a_pdf) |
| 207 | mb = mediaboxes(b_pdf) |
| 208 | n = min(len(ma), len(mb)) |
| 209 | differ = [] |
| 210 | worst = 0.0 |
| 211 | for i in range(n): |
| 212 | d = max(abs(x - y) for x, y in zip(ma[i], mb[i])) |
| 213 | worst = max(worst, d) |
| 214 | if d >= 0.01: |
| 215 | differ.append(i + 1) |
| 216 | print('g1 pages a %d b %d' % (len(ma), len(mb))) |
| 217 | print('g1 mediabox differ %d pages %s' % (len(differ), ranges(differ))) |
| 218 | print('g1 mediabox max-deviation %.4f' % min(worst, 99999.9999)) |
| 219 | same = len(ma) == len(mb) and not differ |
| 220 | print('g1 verdict %s' % ('same' if same else 'differs')) |
| 221 | return 0 if same else 1 |
| 222 | |
| 223 | |
| 224 | FONT_TAG = re.compile(r'^[A-Z]{6}\+') |
| 225 | FONT_NAME = re.compile(r'^[A-Za-z0-9+-]{1,64}$') |
| 226 | |
| 227 | |
| 228 | def base_name(name): |
| 229 | """A font's name without its subset tag and without `-Identity-H`.""" |
| 230 | name = FONT_TAG.sub('', name) |
| 231 | if name.endswith('-Identity-H'): |
| 232 | name = name[:-len('-Identity-H')] |
| 233 | return name |
| 234 | |
| 235 | |
| 236 | def fonts_of(pdf): |
| 237 | """(base name, is it embedded and does it have a Unicode map) for each font `pdffonts` lists.""" |
| 238 | r = subprocess.run(['pdffonts', pdf], capture_output=True) |
| 239 | if r.returncode != 0: |
| 240 | raise OSError('pdffonts') |
| 241 | out = [] |
| 242 | for line in r.stdout.decode('utf-8', 'replace').splitlines()[2:]: |
| 243 | t = line.split() |
| 244 | # name, type (one to three words), encoding, emb, sub, uni, object number, generation. |
| 245 | if len(t) < 8: |
| 246 | continue |
| 247 | out.append((base_name(t[0]), t[-5] == 'yes' and t[-3] == 'yes')) |
| 248 | return out |
| 249 | |
| 250 | |
| 251 | def shown(name): |
| 252 | return name if FONT_NAME.match(name) else 'other' |
| 253 | |
| 254 | |
| 255 | def cmd_g2(a_pdf, b_pdf): |
| 256 | fa = fonts_of(a_pdf) |
| 257 | fb = fonts_of(b_pdf) |
| 258 | sa = {n for n, _ in fa} |
| 259 | sb = {n for n, _ in fb} |
| 260 | flags_a = sum(1 for _, ok in fa if not ok) |
| 261 | flags_b = sum(1 for _, ok in fb if not ok) |
| 262 | print('g2 fonts a %d b %d' % (len(sa), len(sb))) |
| 263 | for n in sorted(sa - sb)[:64]: |
| 264 | print('g2 only-a %s' % shown(n)) |
| 265 | for n in sorted(sb - sa)[:64]: |
| 266 | print('g2 only-b %s' % shown(n)) |
| 267 | print('g2 flags a %d b %d' % (flags_a, flags_b)) |
| 268 | same = sa == sb and flags_a == 0 and flags_b == 0 |
| 269 | print('g2 verdict %s' % ('same' if same else 'differs')) |
| 270 | return 0 if same else 1 |
| 271 | |
| 272 | |
| 273 | def outline_entries(path): |
| 274 | """(depth, title) for every outline entry in document order; depth 1 is the top.""" |
| 275 | out = [] |
| 276 | with pdf_open(path) as pdf: |
| 277 | top = pdf.Root.get('/Outlines') |
| 278 | if top is None: |
| 279 | return out |
| 280 | seen = set() |
| 281 | |
| 282 | def walk(item, depth): |
| 283 | while item is not None: |
| 284 | key = item.objgen |
| 285 | if key != (0, 0): |
| 286 | if key in seen: |
| 287 | return |
| 288 | seen.add(key) |
| 289 | title = item.get('/Title') |
| 290 | out.append((depth, '' if title is None else str(title))) |
| 291 | child = item.get('/First') |
| 292 | if child is not None: |
| 293 | walk(child, depth + 1) |
| 294 | item = item.get('/Next') |
| 295 | |
| 296 | walk(top.get('/First'), 1) |
| 297 | return out |
| 298 | |
| 299 | |
| 300 | def sha(text): |
| 301 | return hashlib.sha256(text.encode('utf-8')).hexdigest() |
| 302 | |
| 303 | |
| 304 | def prefixed(x, y): |
| 305 | """Is `x` just `y` after a leading number or label and a space?""" |
| 306 | if not y or x == y or not x.endswith(y): |
| 307 | return False |
| 308 | return x[:len(x) - len(y)][-1:].isspace() |
| 309 | |
| 310 | |
| 311 | def cmd_g4(a_pdf, b_pdf): |
| 312 | ea = outline_entries(a_pdf) |
| 313 | eb = outline_entries(b_pdf) |
| 314 | la = [d for d, _ in ea] |
| 315 | lb = [d for d, _ in eb] |
| 316 | first = None |
| 317 | for i in range(min(len(la), len(lb))): |
| 318 | if la[i] != lb[i]: |
| 319 | first = i + 1 |
| 320 | break |
| 321 | if first is None and len(la) != len(lb): |
| 322 | first = min(len(la), len(lb)) + 1 |
| 323 | ca = {} |
| 324 | cb = {} |
| 325 | for _, t in ea: |
| 326 | ca[sha(t)] = ca.get(sha(t), 0) + 1 |
| 327 | for _, t in eb: |
| 328 | cb[sha(t)] = cb.get(sha(t), 0) + 1 |
| 329 | miss_a = sum(max(0, n - cb.get(k, 0)) for k, n in ca.items()) |
| 330 | miss_b = sum(max(0, n - ca.get(k, 0)) for k, n in cb.items()) |
| 331 | n = min(len(ea), len(eb)) |
| 332 | pos = 0 |
| 333 | pre = 0 |
| 334 | for i in range(n): |
| 335 | ta, tb = ea[i][1], eb[i][1] |
| 336 | if ta != tb: |
| 337 | pos += 1 |
| 338 | if prefixed(ta, tb) or prefixed(tb, ta): |
| 339 | pre += 1 |
| 340 | print('g4 outline a %d b %d' % (len(ea), len(eb))) |
| 341 | print('g4 levels %s first-diff %s' % ('equal' if first is None else 'differs', 'none' if first is None else first)) |
| 342 | titles_equal = miss_a == 0 and miss_b == 0 |
| 343 | print('g4 titles %s unmatched a %d b %d' % ('equal' if titles_equal else 'differs', miss_a, miss_b)) |
| 344 | print('g4 titles position differ %d of %d' % (pos, n)) |
| 345 | print('g4 titles prefix-only %d' % pre) |
| 346 | same = len(ea) == len(eb) and first is None and titles_equal and pos == 0 |
| 347 | print('g4 verdict %s' % ('same' if same else 'differs')) |
| 348 | return 0 if same else 1 |
| 349 | |
| 350 | |
| 351 | INFO_KEYS = (('title', '/Title'), ('author', '/Author'), ('subject', '/Subject'), ('keywords', '/Keywords')) |
| 352 | |
| 353 | |
| 354 | def info_of(path): |
| 355 | """The sha256 of each Info entry among Title, Author, Subject and Keywords, or None where it is absent.""" |
| 356 | out = {} |
| 357 | with pdf_open(path) as pdf: |
| 358 | info = pdf.trailer.get('/Info') |
| 359 | for word, key in INFO_KEYS: |
| 360 | v = None if info is None else info.get(key) |
| 361 | out[word] = None if v is None else sha(str(v)) |
| 362 | return out |
| 363 | |
| 364 | |
| 365 | def cmd_g5(a_pdf, b_pdf): |
| 366 | ia = info_of(a_pdf) |
| 367 | ib = info_of(b_pdf) |
| 368 | same = True |
| 369 | for word, _ in INFO_KEYS: |
| 370 | x, y = ia[word], ib[word] |
| 371 | if x is None and y is None: |
| 372 | v = 'absent-both' |
| 373 | elif x == y: |
| 374 | v = 'equal' |
| 375 | else: |
| 376 | v = 'differs' |
| 377 | same = False |
| 378 | print('g5 %s %s' % (word, v)) |
| 379 | print('g5 verdict %s' % ('same' if same else 'differs')) |
| 380 | return 0 if same else 1 |
| 381 | |
| 382 | |
| 383 | def read_ppm(path): |
| 384 | import numpy |
| 385 | with open(path, 'rb') as f: |
| 386 | data = f.read() |
| 387 | m = re.match(rb'P6\s+(\d+)\s+(\d+)\s+(\d+)\s', data) |
| 388 | if not m: |
| 389 | raise OSError('ppm') |
| 390 | w, h = int(m.group(1)), int(m.group(2)) |
| 391 | return numpy.frombuffer(data, dtype=numpy.uint8, offset=m.end()).reshape(h, w, 3) |
| 392 | |
| 393 | |
| 394 | def rasterise(pdf, root): |
| 395 | """The paths of the pages of `pdf` at 50 dpi, in page order.""" |
| 396 | r = subprocess.run(['pdftoppm', '-r', '50', pdf, root], capture_output=True) |
| 397 | if r.returncode != 0: |
| 398 | raise OSError('pdftoppm') |
| 399 | found = {} |
| 400 | for f in glob.glob(glob.escape(root) + '-*.ppm'): |
| 401 | m = re.search(r'-(\d+)\.ppm$', f) |
| 402 | if m: |
| 403 | found[int(m.group(1))] = f |
| 404 | return [found[k] for k in sorted(found)] |
| 405 | |
| 406 | |
| 407 | def cmd_g7(a_pdf, b_pdf): |
| 408 | import numpy |
| 409 | base = os.environ.get('GATE_SCRATCH', '') |
| 410 | if not base or not os.path.isdir(base): |
| 411 | print('g7 error scratch') |
| 412 | return 2 |
| 413 | with tempfile.TemporaryDirectory(dir=base) as d: |
| 414 | pa = rasterise(a_pdf, os.path.join(d, 'a')) |
| 415 | pb = rasterise(b_pdf, os.path.join(d, 'b')) |
| 416 | n = min(len(pa), len(pb)) |
| 417 | fr = [] |
| 418 | for i in range(n): |
| 419 | ra, rb = read_ppm(pa[i]), read_ppm(pb[i]) |
| 420 | if ra.shape != rb.shape: |
| 421 | fr.append(1.0) |
| 422 | else: |
| 423 | diff = numpy.abs(ra.astype(numpy.int16) - rb.astype(numpy.int16)).max(axis=2) |
| 424 | fr.append(float((diff > 16).mean())) |
| 425 | os.remove(pa[i]) |
| 426 | os.remove(pb[i]) |
| 427 | print('g7 compared %d of a %d b %d' % (n, len(pa), len(pb))) |
| 428 | for i, f in enumerate(fr): |
| 429 | print('g7 page %d %.4f' % (i + 1, f)) |
| 430 | print('g7 max %s' % ('none' if not fr else '%.4f' % max(fr))) |
| 431 | print('g7 median %s' % ('none' if not fr else '%.4f' % statistics.median(fr))) |
| 432 | return 0 |
| 433 | |
| 434 | |
| 435 | def char_counts(pdf): |
| 436 | """The count of non-space characters on each page, after NFKC.""" |
| 437 | return [len(''.join(unicodedata.normalize('NFKC', t).split())) for t in page_texts(pdf)] |
| 438 | |
| 439 | |
| 440 | def cmd_g8(a_pdf, b_pdf): |
| 441 | ca = char_counts(a_pdf) |
| 442 | cb = char_counts(b_pdf) |
| 443 | n = min(len(ca), len(cb)) |
| 444 | out = [] |
| 445 | empty = [] |
| 446 | worst = 0.0 |
| 447 | for i in range(n): |
| 448 | a, b = ca[i], cb[i] |
| 449 | dev = 0.0 if a == b else (1.0 if a == 0 else abs(b - a) / a) |
| 450 | worst = max(worst, dev) |
| 451 | if dev > 0.01: |
| 452 | out.append(i + 1) |
| 453 | if a > 0 and b == 0: |
| 454 | empty.append(i + 1) |
| 455 | print('g8 pages a %d b %d' % (len(ca), len(cb))) |
| 456 | print('g8 out-of-tolerance %d pages %s' % (len(out), ranges(out))) |
| 457 | for i in out[:64]: |
| 458 | print('g8 page %d a %d b %d' % (i, ca[i - 1], cb[i - 1])) |
| 459 | print('g8 max-deviation %.4f' % min(worst, 9.9999)) |
| 460 | print('g8 empty %d pages %s' % (len(empty), ranges(empty))) |
| 461 | same = len(ca) == len(cb) and not out and not empty |
| 462 | print('g8 verdict %s' % ('same' if same else 'differs')) |
| 463 | return 0 if same else 1 |
| 464 | |
| 465 | |
| 466 | def digest(path): |
| 467 | """The sha256 of a file's bytes and its length, or None for a file that is absent or empty.""" |
| 468 | try: |
| 469 | with open(path, 'rb') as f: |
| 470 | data = f.read() |
| 471 | except OSError: |
| 472 | return None |
| 473 | return (hashlib.sha256(data).hexdigest(), len(data)) if data else None |
| 474 | |
| 475 | |
| 476 | def relation(x, y): |
| 477 | if x is None or y is None: |
| 478 | return 'absent' |
| 479 | return 'equal' if x[0] == y[0] else 'differs' |
| 480 | |
| 481 | |
| 482 | def cmd_g12(cli1, cli2, door1, door2, door3): |
| 483 | c1, c2, d1, d2, d3 = (digest(p) for p in (cli1, cli2, door1, door2, door3)) |
| 484 | rows = ( |
| 485 | ('cli-twice', relation(c1, c2)), |
| 486 | ('door-one-instance', relation(d1, d2)), |
| 487 | ('door-two-instances', relation(d1, d3)), |
| 488 | ) |
| 489 | for word, v in rows: |
| 490 | print('g12 %s %s' % (word, v)) |
| 491 | print('g12 cli-door %s' % relation(c1, d1)) |
| 492 | print('g12 size cli %s door %s' % ('none' if c1 is None else c1[1], 'none' if d1 is None else d1[1])) |
| 493 | same = all(v == 'equal' for _, v in rows) |
| 494 | print('g12 verdict %s' % ('same' if same else 'differs')) |
| 495 | return 0 if same else 1 |
| 496 | |
| 497 | |
| 498 | # ── G9 ────────────────────────────────────────────────────────────────────── |
| 499 | |
| 500 | TAG_PAGE = re.compile(rb'<page\b') |
| 501 | TAG_BLOCK = re.compile(rb'<block\b[^>]*>') |
| 502 | TAG_LINE = re.compile(rb'<line\b[^>]*>') |
| 503 | TAG_WORD = re.compile(rb'<word\b[^>]*>') |
| 504 | ATTR = re.compile(rb'(xMin|yMin|xMax|yMax)="([-0-9.]+)"') |
| 505 | |
| 506 | |
| 507 | def boxes(tag): |
| 508 | """(xMin, yMin, xMax, yMax) of one tag, from its own attributes.""" |
| 509 | a = {k: float(v) for k, v in ATTR.findall(tag)} |
| 510 | return a[b'xMin'], a[b'yMin'], a[b'xMax'], a[b'yMax'] |
| 511 | |
| 512 | |
| 513 | def layout_pages(pdf): |
| 514 | """Per page, a dict of counts and extents and the list of its lines, from `pdftotext -bbox-layout`.""" |
| 515 | r = subprocess.run(['pdftotext', '-bbox-layout', pdf, '-'], capture_output=True) |
| 516 | if r.returncode != 0: |
| 517 | raise OSError('pdftotext') |
| 518 | pages = [] |
| 519 | for chunk in TAG_PAGE.split(r.stdout)[1:]: |
| 520 | blocks = TAG_BLOCK.split(chunk) |
| 521 | lines = [boxes(t) for t in TAG_LINE.findall(chunk)] |
| 522 | heights = [b[3] - b[1] for b in (boxes(t) for t in TAG_WORD.findall(chunk))] |
| 523 | # The pitch: the median gap from one line to the next in reading order, leaving out the gaps of |
| 524 | # more than two and a half word heights, which are spaces between paragraphs and blocks. |
| 525 | hm = statistics.median(heights) if heights else 0.0 |
| 526 | gaps = [b[1] - a[1] for a, b in zip(lines, lines[1:]) if 0 < b[1] - a[1] <= 2.5 * hm] |
| 527 | pages.append({ |
| 528 | 'lines': len(lines), |
| 529 | 'blocks': len(blocks) - 1, |
| 530 | 'words': len(heights), |
| 531 | 'sizes': len({round(h * 2) for h in heights}), |
| 532 | 'left': min((b[0] for b in lines), default=0.0), |
| 533 | 'right': max((b[2] for b in lines), default=0.0), |
| 534 | 'top': min((b[1] for b in lines), default=0.0), |
| 535 | 'bottom': max((b[3] for b in lines), default=0.0), |
| 536 | 'pitch': statistics.median(gaps) if gaps else 0.0, |
| 537 | 'height': statistics.median(heights) if heights else 0.0, |
| 538 | 'rows': lines, |
| 539 | }) |
| 540 | return pages |
| 541 | |
| 542 | |
| 543 | def counts_per_page(pdf, n): |
| 544 | """(images, links) on each of the `n` pages.""" |
| 545 | r = subprocess.run(['pdfimages', '-list', pdf], capture_output=True) |
| 546 | imgs = [0] * n |
| 547 | if r.returncode == 0: |
| 548 | for line in r.stdout.decode('utf-8', 'replace').splitlines()[2:]: |
| 549 | t = line.split() |
| 550 | if t and t[0].isdigit() and 1 <= int(t[0]) <= n: |
| 551 | imgs[int(t[0]) - 1] += 1 |
| 552 | links = [] |
| 553 | with pdf_open(pdf) as doc: |
| 554 | for page in doc.pages: |
| 555 | ann = page.obj.get('/Annots') |
| 556 | links.append(0 if ann is None else sum(1 for x in ann if str(x.get('/Subtype')) == '/Link')) |
| 557 | return imgs, links |
| 558 | |
| 559 | |
| 560 | COUNTS = ('lines', 'blocks', 'words', 'sizes') |
| 561 | EXTENTS = ('left', 'right', 'top', 'bottom', 'height') |
| 562 | |
| 563 | |
| 564 | def cmd_g9(a_pdf, b_pdf): |
| 565 | pa = layout_pages(a_pdf) |
| 566 | pb = layout_pages(b_pdf) |
| 567 | ia, la = counts_per_page(a_pdf, len(pa)) |
| 568 | ib, lb = counts_per_page(b_pdf, len(pb)) |
| 569 | n = min(len(pa), len(pb)) |
| 570 | rows = [] # (page, [(metric, a, b)]) for each page that differs |
| 571 | for i in range(n): |
| 572 | a, b = pa[i], pb[i] |
| 573 | a['images'], a['links'], b['images'], b['links'] = ia[i], la[i], ib[i], lb[i] |
| 574 | d = [] |
| 575 | for m in COUNTS + ('images', 'links'): |
| 576 | if a[m] != b[m]: |
| 577 | d.append((m, a[m], b[m], '%d')) |
| 578 | for m in EXTENTS: |
| 579 | if abs(a[m] - b[m]) > 0.5: |
| 580 | d.append((m, a[m], b[m], '%.2f')) |
| 581 | if abs(a['pitch'] - b['pitch']) > 0.1: |
| 582 | d.append(('pitch', a['pitch'], b['pitch'], '%.2f')) |
| 583 | if d: |
| 584 | rows.append((i, d)) |
| 585 | idx = [i + 1 for i, _ in rows] |
| 586 | print('g9 pages differ %d pages %s' % (len(idx), ranges(idx))) |
| 587 | for i, d in rows[:12]: |
| 588 | for m, x, y, f in d: |
| 589 | print(('g9 page %d %s a ' + f + ' b ' + f) % (i + 1, m, x, y)) |
| 590 | # The pages whose line places are listed: those that differ in more than their links, then the rest. |
| 591 | more = [i for i, d in rows if any(m != 'links' for m, *_ in d)] |
| 592 | for i in (more or [i for i, _ in rows])[:5]: |
| 593 | shown = 0 |
| 594 | for j, (u, v) in enumerate(zip(pa[i]['rows'], pb[i]['rows'])): |
| 595 | if max(abs(u[1] - v[1]), abs(u[0] - v[0]), abs((u[2] - u[0]) - (v[2] - v[0]))) > 0.5: |
| 596 | print('g9 line %d %d y a %.2f b %.2f x a %.2f b %.2f w a %.2f b %.2f' |
| 597 | % (i + 1, j + 1, u[1], v[1], u[0], v[0], u[2] - u[0], v[2] - v[0])) |
| 598 | shown += 1 |
| 599 | if shown >= 14: |
| 600 | break |
| 601 | return 0 |
| 602 | |
| 603 | |
| 604 | CHECKS = { |
| 605 | 'compare': (2, cmd_compare, 'g3'), |
| 606 | 'g3': (2, cmd_compare, 'g3'), |
| 607 | 'pages': (1, cmd_pages, 'g3'), |
| 608 | 'g1': (2, cmd_g1, 'g1'), |
| 609 | 'g2': (2, cmd_g2, 'g2'), |
| 610 | 'g4': (2, cmd_g4, 'g4'), |
| 611 | 'g5': (2, cmd_g5, 'g5'), |
| 612 | 'g7': (2, cmd_g7, 'g7'), |
| 613 | 'g8': (2, cmd_g8, 'g8'), |
| 614 | 'g12': (5, cmd_g12, 'g12'), |
| 615 | 'g9': (2, cmd_g9, 'g9'), |
| 616 | } |
| 617 | |
| 618 | |
| 619 | def main(argv): |
| 620 | check = CHECKS.get(argv[1]) if len(argv) > 1 else None |
| 621 | if check is None: |
| 622 | print('g3 error usage') |
| 623 | return 2 |
| 624 | if len(argv) != 2 + check[0]: |
| 625 | print('%s error usage' % check[2]) |
| 626 | return 2 |
| 627 | try: |
| 628 | return check[1](*argv[2:]) |
| 629 | except Exception: |
| 630 | # No message and no traceback: either could carry a path or a line of the document. |
| 631 | print('%s error unreadable' % check[2]) |
| 632 | return 2 |
| 633 | |
| 634 | |
| 635 | if __name__ == '__main__': |
| 636 | sys.exit(main(sys.argv)) |