oxedyne/fe2o3/fe2o3_austenite/tools/wasm_measure.py
13.8 KiB, 15 runs, executable
created by r1870400018:65174, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #!/usr/bin/env python3 |
| 2 | """Size of a wasm binary, taken apart the way the Austenite size metric needs it. |
| 3 | |
| 4 | For each `.wasm` given: raw bytes, gzip -9 and brotli (quality 11); the byte split between the |
| 5 | code, data and custom sections; the bytes of every font embedded in the data segments, found by |
| 6 | walking each sfnt table directory rather than by guessing; and, when the binary keeps its `name` |
| 7 | section, code size attributed to Rust modules. |
| 8 | |
| 9 | tools/wasm_measure.py pkg/oxedyne_fe2o3_austenite_bg.wasm |
| 10 | tools/wasm_measure.py --compare ~/.../typst_ts_web_compiler_bg.wasm pkg/..._bg.wasm |
| 11 | tools/wasm_measure.py --limit-mb 10 --json out.json pkg/..._bg.wasm |
| 12 | |
| 13 | The comparison is like for like: typst.ts ships its fonts separately, so every row is also given |
| 14 | as total minus embedded fonts. `--limit-mb` is the proposed pass line (owner decision 1): the |
| 15 | first binary's raw size must not exceed it, and the exit status says whether it did. |
| 16 | |
| 17 | A release build strips the `name` section, so attribution needs a measurement build that keeps |
| 18 | it, for instance `wasm-pack build --profiling --target web --no-default-features --features wasm` or a raw |
| 19 | `cargo build --release --target wasm32-unknown-unknown --no-default-features --features wasm`. Without names the |
| 20 | attribution table is omitted and the report says so. |
| 21 | |
| 22 | Standard library only: no dependency is installed for this. Brotli is reported when the |
| 23 | `brotli` module is importable and marked unavailable otherwise, never estimated. |
| 24 | """ |
| 25 | |
| 26 | import argparse |
| 27 | import gzip |
| 28 | import json |
| 29 | import os |
| 30 | import sys |
| 31 | |
| 32 | try: |
| 33 | import brotli # optional; the size is reported as unavailable without it |
| 34 | except ImportError: |
| 35 | brotli = None |
| 36 | |
| 37 | SECTION_NAMES = { |
| 38 | 0: 'custom', 1: 'type', 2: 'import', 3: 'function', 4: 'table', 5: 'memory', 6: 'global', |
| 39 | 7: 'export', 8: 'start', 9: 'element', 10: 'code', 11: 'data', 12: 'datacount', 13: 'tag', |
| 40 | } |
| 41 | |
| 42 | SFNT_MAGICS = (b'\x00\x01\x00\x00', b'OTTO', b'true', b'typ1') |
| 43 | |
| 44 | |
| 45 | class Reader: |
| 46 | def __init__(self, buf, pos=0, end=None): |
| 47 | self.buf = buf |
| 48 | self.pos = pos |
| 49 | self.end = len(buf) if end is None else end |
| 50 | |
| 51 | def byte(self): |
| 52 | if self.pos >= self.end: |
| 53 | raise ValueError('read past the end at byte %d' % self.pos) |
| 54 | b = self.buf[self.pos] |
| 55 | self.pos += 1 |
| 56 | return b |
| 57 | |
| 58 | def uleb(self): |
| 59 | result = 0 |
| 60 | shift = 0 |
| 61 | while True: |
| 62 | b = self.byte() |
| 63 | result |= (b & 0x7f) << shift |
| 64 | if b & 0x80 == 0: |
| 65 | return result |
| 66 | shift += 7 |
| 67 | |
| 68 | def sleb(self): |
| 69 | result = 0 |
| 70 | shift = 0 |
| 71 | while True: |
| 72 | b = self.byte() |
| 73 | result |= (b & 0x7f) << shift |
| 74 | shift += 7 |
| 75 | if b & 0x80 == 0: |
| 76 | if b & 0x40: |
| 77 | result -= 1 << shift |
| 78 | return result |
| 79 | |
| 80 | def bytes(self, n): |
| 81 | if self.pos + n > self.end: |
| 82 | raise ValueError('a %d-byte field overruns its section at byte %d' % (n, self.pos)) |
| 83 | out = self.buf[self.pos:self.pos + n] |
| 84 | self.pos += n |
| 85 | return out |
| 86 | |
| 87 | def name(self): |
| 88 | n = self.uleb() |
| 89 | return self.bytes(n).decode('utf-8', 'replace') |
| 90 | |
| 91 | def const_expr(self): |
| 92 | # The offset expression of an active data segment: a constant or a global read, then `end`. |
| 93 | # Returns the constant, or None when the offset is not a plain constant. |
| 94 | value = None |
| 95 | while True: |
| 96 | op = self.byte() |
| 97 | if op == 0x0b: |
| 98 | return value |
| 99 | if op in (0x41, 0x42): |
| 100 | value = self.sleb() |
| 101 | elif op == 0x23: |
| 102 | self.uleb() |
| 103 | elif op == 0x44: |
| 104 | self.bytes(8) |
| 105 | elif op == 0x43: |
| 106 | self.bytes(4) |
| 107 | else: |
| 108 | raise ValueError('unexpected opcode 0x%02x in a constant expression' % op) |
| 109 | |
| 110 | |
| 111 | def sections(buf): |
| 112 | if buf[:4] != b'\x00asm': |
| 113 | raise ValueError('not a wasm binary (no \\0asm magic)') |
| 114 | r = Reader(buf, 8) |
| 115 | out = [] |
| 116 | while r.pos < r.end: |
| 117 | sid = r.byte() |
| 118 | size = r.uleb() |
| 119 | start = r.pos |
| 120 | name = SECTION_NAMES.get(sid, 'unknown%d' % sid) |
| 121 | if sid == 0: |
| 122 | sub = Reader(buf, start, start + size) |
| 123 | name = 'custom:' + sub.name() |
| 124 | out.append((sid, name, start, size)) |
| 125 | r.pos = start + size |
| 126 | return out |
| 127 | |
| 128 | |
| 129 | def imported_functions(buf, start, size): |
| 130 | r = Reader(buf, start, start + size) |
| 131 | count = 0 |
| 132 | for _ in range(r.uleb()): |
| 133 | r.name() |
| 134 | r.name() |
| 135 | kind = r.byte() |
| 136 | if kind == 0x00: |
| 137 | r.uleb() |
| 138 | count += 1 |
| 139 | elif kind == 0x01: |
| 140 | r.byte() |
| 141 | limits(r) |
| 142 | elif kind == 0x02: |
| 143 | limits(r) |
| 144 | elif kind == 0x03: |
| 145 | r.byte() |
| 146 | r.byte() |
| 147 | elif kind == 0x04: |
| 148 | r.byte() |
| 149 | r.uleb() |
| 150 | else: |
| 151 | raise ValueError('unknown import kind %d' % kind) |
| 152 | return count |
| 153 | |
| 154 | |
| 155 | def limits(r): |
| 156 | flags = r.byte() |
| 157 | r.uleb() |
| 158 | if flags & 0x01: |
| 159 | r.uleb() |
| 160 | |
| 161 | |
| 162 | def code_bodies(buf, start, size): |
| 163 | r = Reader(buf, start, start + size) |
| 164 | out = [] |
| 165 | for _ in range(r.uleb()): |
| 166 | n = r.uleb() |
| 167 | out.append(n) |
| 168 | r.pos += n |
| 169 | return out |
| 170 | |
| 171 | |
| 172 | def data_segments(buf, start, size): |
| 173 | r = Reader(buf, start, start + size) |
| 174 | out = [] |
| 175 | for _ in range(r.uleb()): |
| 176 | flags = r.uleb() |
| 177 | addr = None |
| 178 | if flags == 0: |
| 179 | addr = r.const_expr() |
| 180 | elif flags == 2: |
| 181 | r.uleb() |
| 182 | addr = r.const_expr() |
| 183 | n = r.uleb() |
| 184 | out.append((r.pos, n, addr)) |
| 185 | r.pos += n |
| 186 | return out |
| 187 | |
| 188 | |
| 189 | def function_names(buf, start, size): |
| 190 | r = Reader(buf, start, start + size) |
| 191 | r.name() |
| 192 | names = {} |
| 193 | while r.pos < r.end: |
| 194 | sub = r.byte() |
| 195 | n = r.uleb() |
| 196 | end = r.pos + n |
| 197 | if sub == 1: |
| 198 | for _ in range(r.uleb()): |
| 199 | idx = r.uleb() |
| 200 | names[idx] = r.name() |
| 201 | r.pos = end |
| 202 | return names |
| 203 | |
| 204 | |
| 205 | def sfnt_length(buf, at, end): |
| 206 | """The byte length of an sfnt font starting at `at`, or None when the bytes there are not one. |
| 207 | |
| 208 | The table directory is checked, not trusted: a table count in a sane range, printable tags, |
| 209 | and a `head` table present; the length is the furthest table end, padded to four bytes.""" |
| 210 | if at + 12 > end: |
| 211 | return None |
| 212 | num = int.from_bytes(buf[at + 4:at + 6], 'big') |
| 213 | if num < 4 or num > 64 or at + 12 + 16 * num > end: |
| 214 | return None |
| 215 | furthest = 0 |
| 216 | tags = set() |
| 217 | for i in range(num): |
| 218 | rec = at + 12 + 16 * i |
| 219 | tag = buf[rec:rec + 4] |
| 220 | if not all(0x20 <= c < 0x7f for c in tag): |
| 221 | return None |
| 222 | tags.add(bytes(tag)) |
| 223 | off = int.from_bytes(buf[rec + 8:rec + 12], 'big') |
| 224 | ln = int.from_bytes(buf[rec + 12:rec + 16], 'big') |
| 225 | if off < 12 + 16 * num or at + off + ln > end: |
| 226 | return None |
| 227 | furthest = max(furthest, off + ln) |
| 228 | if b'head' not in tags or not (b'cmap' in tags or b'CFF ' in tags or b'glyf' in tags): |
| 229 | return None |
| 230 | return (furthest + 3) & ~3 |
| 231 | |
| 232 | |
| 233 | def memory_image(buf, segments): |
| 234 | """The initial linear memory the active segments build. An optimiser splits the data at runs |
| 235 | of zeros, so a font spans many segments; it is only whole again in the image.""" |
| 236 | placed = [(addr, pos, n) for (pos, n, addr) in segments if addr is not None] |
| 237 | if not placed: |
| 238 | return b'' |
| 239 | lo = min(a for (a, _, _) in placed) |
| 240 | hi = max(a + n for (a, _, n) in placed) |
| 241 | img = bytearray(hi - lo) |
| 242 | for (a, pos, n) in placed: |
| 243 | img[a - lo:a - lo + n] = buf[pos:pos + n] |
| 244 | return bytes(img) |
| 245 | |
| 246 | |
| 247 | def fonts_in(img): |
| 248 | found = [] |
| 249 | end = len(img) |
| 250 | i = 0 |
| 251 | while True: |
| 252 | hits = [h for h in (img.find(m, i, end) for m in SFNT_MAGICS) if h >= 0] |
| 253 | if not hits: |
| 254 | return found |
| 255 | h = min(hits) |
| 256 | ln = sfnt_length(img, h, end) |
| 257 | if ln: |
| 258 | found.append((h, ln)) |
| 259 | i = h + ln |
| 260 | else: |
| 261 | i = h + 1 |
| 262 | |
| 263 | |
| 264 | def font_name(img, at): |
| 265 | """The face's own name, from its `name` table: the full name (ID 4), else family and style (IDs 1 |
| 266 | and 2), else `(unnamed)`. A Windows or Unicode record is UTF-16BE, a Macintosh one Mac Roman, read |
| 267 | as Latin-1 for the ASCII names a font carries.""" |
| 268 | num = int.from_bytes(img[at + 4:at + 6], 'big') |
| 269 | for i in range(num): |
| 270 | rec = at + 12 + 16 * i |
| 271 | if img[rec:rec + 4] != b'name': |
| 272 | continue |
| 273 | base = at + int.from_bytes(img[rec + 8:rec + 12], 'big') |
| 274 | count = int.from_bytes(img[base + 2:base + 4], 'big') |
| 275 | store = base + int.from_bytes(img[base + 4:base + 6], 'big') |
| 276 | found = {} |
| 277 | for k in range(count): |
| 278 | r = base + 6 + 12 * k |
| 279 | plat = int.from_bytes(img[r:r + 2], 'big') |
| 280 | nid = int.from_bytes(img[r + 6:r + 8], 'big') |
| 281 | n = int.from_bytes(img[r + 8:r + 10], 'big') |
| 282 | o = int.from_bytes(img[r + 10:r + 12], 'big') |
| 283 | raw = img[store + o:store + o + n] |
| 284 | text = raw.decode('utf-16-be', 'replace') if plat in (0, 3) else raw.decode('latin-1') |
| 285 | if plat in (0, 3) or nid not in found: |
| 286 | found[nid] = text |
| 287 | if 4 in found: |
| 288 | return found[4] |
| 289 | if 1 in found: |
| 290 | return (found[1] + ' ' + found.get(2, '')).strip() |
| 291 | return '(unnamed)' |
| 292 | |
| 293 | |
| 294 | def demangle(sym): |
| 295 | """Legacy Rust mangling (`_ZN3foo3bar17h0123456789abcdefE`) to a path; anything else unchanged.""" |
| 296 | if not sym.startswith('_ZN'): |
| 297 | return sym |
| 298 | i = 3 |
| 299 | parts = [] |
| 300 | while i < len(sym) and sym[i] != 'E': |
| 301 | j = i |
| 302 | while j < len(sym) and sym[j].isdigit(): |
| 303 | j += 1 |
| 304 | if j == i: |
| 305 | break |
| 306 | n = int(sym[i:j]) |
| 307 | parts.append(sym[j:j + n]) |
| 308 | i = j + n |
| 309 | if parts and len(parts[-1]) == 17 and parts[-1].startswith('h'): |
| 310 | parts.pop() |
| 311 | return '::'.join(parts) |
| 312 | |
| 313 | |
| 314 | LEGACY_ESCAPES = ( |
| 315 | ('$LT$', '<'), ('$GT$', '>'), ('$u20$', ' '), ('$RF$', '&'), ('$BP$', '*'), ('$C$', ','), |
| 316 | ('$u7b$', '{'), ('$u7d$', '}'), ('$u5b$', '['), ('$u5d$', ']'), ('$u27$', "'"), ('$u22$', '"'), |
| 317 | ('$LP$', '('), ('$RP$', ')'), ('$SP$', '@'), ('..', '::'), |
| 318 | ) |
| 319 | |
| 320 | |
| 321 | def unescape(s): |
| 322 | """A legacy-mangled segment's escapes (`_$LT$a..b$GT$`) as the path they spell (`<a::b>`).""" |
| 323 | if s.startswith('_$'): |
| 324 | s = s[1:] |
| 325 | for a, b in LEGACY_ESCAPES: |
| 326 | s = s.replace(a, b) |
| 327 | return s |
| 328 | |
| 329 | |
| 330 | def module_of(name, depth): |
| 331 | """The module a function belongs to, to `depth` path segments. A trait impl |
| 332 | (`<a::B as c::D>::f`) is charged to its self type's module.""" |
| 333 | s = unescape(demangle(name)) |
| 334 | if s.startswith('<'): |
| 335 | inner = s[1:] |
| 336 | for stop in (' as ', '>'): |
| 337 | k = inner.find(stop) |
| 338 | if k >= 0: |
| 339 | inner = inner[:k] |
| 340 | break |
| 341 | s = inner.lstrip('&*').replace('mut ', '').replace('const ', '') |
| 342 | s = s.split('<')[0] |
| 343 | segs = [p for p in s.split('::') if p] |
| 344 | if len(segs) <= 1: |
| 345 | return '(unattributed)' if not segs else segs[0] |
| 346 | return '::'.join(segs[:depth]) |
| 347 | |
| 348 | |
| 349 | def measure(path, depth): |
| 350 | buf = open(path, 'rb').read() |
| 351 | secs = sections(buf) |
| 352 | out = { |
| 353 | 'path': path, |
| 354 | 'raw': len(buf), |
| 355 | 'gzip9': len(gzip.compress(buf, 9)), |
| 356 | 'brotli': len(brotli.compress(buf, quality=11)) if brotli else None, |
| 357 | 'sections': {}, |
| 358 | } |
| 359 | n_imports = 0 |
| 360 | bodies = [] |
| 361 | segments = [] |
| 362 | names = None |
| 363 | for (sid, name, start, size) in secs: |
| 364 | out['sections'][name] = out['sections'].get(name, 0) + size |
| 365 | if sid == 2: |
| 366 | n_imports = imported_functions(buf, start, size) |
| 367 | elif sid == 10: |
| 368 | bodies = code_bodies(buf, start, size) |
| 369 | elif sid == 11: |
| 370 | segments = data_segments(buf, start, size) |
| 371 | elif name == 'custom:name': |
| 372 | names = function_names(buf, start, size) |
| 373 | out['code'] = out['sections'].get('code', 0) |
| 374 | out['data'] = out['sections'].get('data', 0) |
| 375 | out['custom'] = sum(v for k, v in out['sections'].items() if k.startswith('custom:')) |
| 376 | img = memory_image(buf, segments) |
| 377 | fonts = fonts_in(img) |
| 378 | out['fonts'] = [{'offset': h, 'bytes': ln, 'name': font_name(img, h)} for (h, ln) in fonts] |
| 379 | out['font_bytes'] = sum(ln for (_, ln) in fonts) |
| 380 | out['raw_minus_fonts'] = out['raw'] - out['font_bytes'] |
| 381 | out['functions'] = len(bodies) |
| 382 | if names: |
| 383 | per = {} |
| 384 | crates = {} |
| 385 | for i, n in enumerate(bodies): |
| 386 | name = names.get(n_imports + i, '') |
| 387 | mod = module_of(name, depth) |
| 388 | per[mod] = per.get(mod, 0) + n |
| 389 | crate = module_of(name, 1) |
| 390 | crates[crate] = crates.get(crate, 0) + n |
| 391 | out['modules'] = sorted(per.items(), key=lambda kv: -kv[1]) |
| 392 | out['crates'] = sorted(crates.items(), key=lambda kv: -kv[1]) |
| 393 | else: |
| 394 | out['modules'] = None |
| 395 | out['crates'] = None |
| 396 | return out |
| 397 | |
| 398 | |
| 399 | def mb(n): |
| 400 | return '-' if n is None else '%.2f MB' % (n / 1048576.0) |
| 401 | |
| 402 | |
| 403 | def report(m, top): |
| 404 | print(m['path']) |
| 405 | print(' raw %12d %s' % (m['raw'], mb(m['raw']))) |
| 406 | print(' gzip -9 %12d %s' % (m['gzip9'], mb(m['gzip9']))) |
| 407 | if m['brotli'] is None: |
| 408 | print(' brotli %12s (python brotli module unavailable)' % '-') |
| 409 | else: |
| 410 | print(' brotli %12d %s' % (m['brotli'], mb(m['brotli']))) |
| 411 | print(' code section %12d %s (%d functions)' % (m['code'], mb(m['code']), m['functions'])) |
| 412 | print(' data section %12d %s' % (m['data'], mb(m['data']))) |
| 413 | print(' custom %12d %s' % (m['custom'], mb(m['custom']))) |
| 414 | print(' fonts in data %12d %s (%d faces)' % (m['font_bytes'], mb(m['font_bytes']), len(m['fonts']))) |
| 415 | print(' raw - fonts %12d %s' % (m['raw_minus_fonts'], mb(m['raw_minus_fonts']))) |
| 416 | for f in sorted(m['fonts'], key=lambda f: -f['bytes']): |
| 417 | print(' face %10d %s' % (f['bytes'], f['name'])) |
| 418 | if m['modules'] is None: |
| 419 | print(' (no name section: build with names retained for per-module attribution)') |
| 420 | return |
| 421 | for title, rows in (('crate', m['crates']), ('module', m['modules'])): |
| 422 | print(' code by %s (top %d):' % (title, top)) |
| 423 | for mod, n in rows[:top]: |
| 424 | print(' %10d %5.1f%% %s' % (n, 100.0 * n / max(1, m['code']), mod)) |
| 425 | |
| 426 | |
| 427 | def main(): |
| 428 | ap = argparse.ArgumentParser(description=__doc__.split('\n')[0]) |
| 429 | ap.add_argument('wasm', nargs='+') |
| 430 | ap.add_argument('--depth', type=int, default=2, help='module path segments for attribution') |
| 431 | ap.add_argument('--top', type=int, default=25) |
| 432 | ap.add_argument('--json', help='also write the measurements to this file') |
| 433 | ap.add_argument('--limit-mb', type=float, help='fail when the first binary is larger (raw)') |
| 434 | ap.add_argument('--compare', action='store_true', help='print a side-by-side table last') |
| 435 | a = ap.parse_args() |
| 436 | ms = [] |
| 437 | for p in a.wasm: |
| 438 | m = measure(p, a.depth) |
| 439 | ms.append(m) |
| 440 | report(m, a.top) |
| 441 | print() |
| 442 | if a.compare and len(ms) > 1: |
| 443 | print('%-44s %10s %10s %10s %10s %10s' % ('binary', 'raw', 'gzip', 'brotli', 'fonts', 'raw-fonts')) |
| 444 | for m in ms: |
| 445 | print('%-44s %10s %10s %10s %10s %10s' % ( |
| 446 | os.path.basename(m['path'])[:44], mb(m['raw']), mb(m['gzip9']), mb(m['brotli']), |
| 447 | mb(m['font_bytes']), mb(m['raw_minus_fonts']))) |
| 448 | base = ms[0] |
| 449 | for m in ms[1:]: |
| 450 | print(' %s / %s: raw %.1f%%, raw minus fonts %.1f%%' % ( |
| 451 | os.path.basename(base['path']), os.path.basename(m['path']), |
| 452 | 100.0 * base['raw'] / max(1, m['raw']), |
| 453 | 100.0 * base['raw_minus_fonts'] / max(1, m['raw_minus_fonts']))) |
| 454 | if a.json: |
| 455 | with open(a.json, 'w') as f: |
| 456 | json.dump(ms, f, indent=1) |
| 457 | if a.limit_mb is not None: |
| 458 | limit = a.limit_mb * 1048576.0 |
| 459 | ok = ms[0]['raw'] <= limit |
| 460 | print('pass line: %s raw %s against %.1f MB: %s' % ( |
| 461 | os.path.basename(ms[0]['path']), mb(ms[0]['raw']), a.limit_mb, 'PASS' if ok else 'FAIL')) |
| 462 | return 0 if ok else 1 |
| 463 | return 0 |
| 464 | |
| 465 | |
| 466 | if __name__ == '__main__': |
| 467 | sys.exit(main()) |