Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tools/wasm_measure.py

13.8 KiB, 15 runs, executable

created by r1870400018:65174, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1#!/usr/bin/env python3
2"""Size of a wasm binary, taken apart the way the Austenite size metric needs it.
3
4For each `.wasm` given: raw bytes, gzip -9 and brotli (quality 11); the byte split between the
5code, data and custom sections; the bytes of every font embedded in the data segments, found by
6walking each sfnt table directory rather than by guessing; and, when the binary keeps its `name`
7section, code size attributed to Rust modules.
8
9 tools/wasm_measure.py pkg/oxedyne_fe2o3_austenite_bg.wasm
10 tools/wasm_measure.py --compare ~/.../typst_ts_web_compiler_bg.wasm pkg/..._bg.wasm
11 tools/wasm_measure.py --limit-mb 10 --json out.json pkg/..._bg.wasm
12
13The comparison is like for like: typst.ts ships its fonts separately, so every row is also given
14as total minus embedded fonts. `--limit-mb` is the proposed pass line (owner decision 1): the
15first binary's raw size must not exceed it, and the exit status says whether it did.
16
17A release build strips the `name` section, so attribution needs a measurement build that keeps
18it, for instance `wasm-pack build --profiling --target web --no-default-features --features wasm` or a raw
19`cargo build --release --target wasm32-unknown-unknown --no-default-features --features wasm`. Without names the
20attribution table is omitted and the report says so.
21
22Standard library only: no dependency is installed for this. Brotli is reported when the
23`brotli` module is importable and marked unavailable otherwise, never estimated.
24"""
25
26import argparse
27import gzip
28import json
29import os
30import sys
31
32try:
33 import brotli # optional; the size is reported as unavailable without it
34except ImportError:
35 brotli = None
36
37SECTION_NAMES = {
38 0: 'custom', 1: 'type', 2: 'import', 3: 'function', 4: 'table', 5: 'memory', 6: 'global',
39 7: 'export', 8: 'start', 9: 'element', 10: 'code', 11: 'data', 12: 'datacount', 13: 'tag',
40}
41
42SFNT_MAGICS = (b'\x00\x01\x00\x00', b'OTTO', b'true', b'typ1')
43
44
45class Reader:
46 def __init__(self, buf, pos=0, end=None):
47 self.buf = buf
48 self.pos = pos
49 self.end = len(buf) if end is None else end
50
51 def byte(self):
52 if self.pos >= self.end:
53 raise ValueError('read past the end at byte %d' % self.pos)
54 b = self.buf[self.pos]
55 self.pos += 1
56 return b
57
58 def uleb(self):
59 result = 0
60 shift = 0
61 while True:
62 b = self.byte()
63 result |= (b & 0x7f) << shift
64 if b & 0x80 == 0:
65 return result
66 shift += 7
67
68 def sleb(self):
69 result = 0
70 shift = 0
71 while True:
72 b = self.byte()
73 result |= (b & 0x7f) << shift
74 shift += 7
75 if b & 0x80 == 0:
76 if b & 0x40:
77 result -= 1 << shift
78 return result
79
80 def bytes(self, n):
81 if self.pos + n > self.end:
82 raise ValueError('a %d-byte field overruns its section at byte %d' % (n, self.pos))
83 out = self.buf[self.pos:self.pos + n]
84 self.pos += n
85 return out
86
87 def name(self):
88 n = self.uleb()
89 return self.bytes(n).decode('utf-8', 'replace')
90
91 def const_expr(self):
92 # The offset expression of an active data segment: a constant or a global read, then `end`.
93 # Returns the constant, or None when the offset is not a plain constant.
94 value = None
95 while True:
96 op = self.byte()
97 if op == 0x0b:
98 return value
99 if op in (0x41, 0x42):
100 value = self.sleb()
101 elif op == 0x23:
102 self.uleb()
103 elif op == 0x44:
104 self.bytes(8)
105 elif op == 0x43:
106 self.bytes(4)
107 else:
108 raise ValueError('unexpected opcode 0x%02x in a constant expression' % op)
109
110
111def sections(buf):
112 if buf[:4] != b'\x00asm':
113 raise ValueError('not a wasm binary (no \\0asm magic)')
114 r = Reader(buf, 8)
115 out = []
116 while r.pos < r.end:
117 sid = r.byte()
118 size = r.uleb()
119 start = r.pos
120 name = SECTION_NAMES.get(sid, 'unknown%d' % sid)
121 if sid == 0:
122 sub = Reader(buf, start, start + size)
123 name = 'custom:' + sub.name()
124 out.append((sid, name, start, size))
125 r.pos = start + size
126 return out
127
128
129def imported_functions(buf, start, size):
130 r = Reader(buf, start, start + size)
131 count = 0
132 for _ in range(r.uleb()):
133 r.name()
134 r.name()
135 kind = r.byte()
136 if kind == 0x00:
137 r.uleb()
138 count += 1
139 elif kind == 0x01:
140 r.byte()
141 limits(r)
142 elif kind == 0x02:
143 limits(r)
144 elif kind == 0x03:
145 r.byte()
146 r.byte()
147 elif kind == 0x04:
148 r.byte()
149 r.uleb()
150 else:
151 raise ValueError('unknown import kind %d' % kind)
152 return count
153
154
155def limits(r):
156 flags = r.byte()
157 r.uleb()
158 if flags & 0x01:
159 r.uleb()
160
161
162def code_bodies(buf, start, size):
163 r = Reader(buf, start, start + size)
164 out = []
165 for _ in range(r.uleb()):
166 n = r.uleb()
167 out.append(n)
168 r.pos += n
169 return out
170
171
172def data_segments(buf, start, size):
173 r = Reader(buf, start, start + size)
174 out = []
175 for _ in range(r.uleb()):
176 flags = r.uleb()
177 addr = None
178 if flags == 0:
179 addr = r.const_expr()
180 elif flags == 2:
181 r.uleb()
182 addr = r.const_expr()
183 n = r.uleb()
184 out.append((r.pos, n, addr))
185 r.pos += n
186 return out
187
188
189def function_names(buf, start, size):
190 r = Reader(buf, start, start + size)
191 r.name()
192 names = {}
193 while r.pos < r.end:
194 sub = r.byte()
195 n = r.uleb()
196 end = r.pos + n
197 if sub == 1:
198 for _ in range(r.uleb()):
199 idx = r.uleb()
200 names[idx] = r.name()
201 r.pos = end
202 return names
203
204
205def sfnt_length(buf, at, end):
206 """The byte length of an sfnt font starting at `at`, or None when the bytes there are not one.
207
208 The table directory is checked, not trusted: a table count in a sane range, printable tags,
209 and a `head` table present; the length is the furthest table end, padded to four bytes."""
210 if at + 12 > end:
211 return None
212 num = int.from_bytes(buf[at + 4:at + 6], 'big')
213 if num < 4 or num > 64 or at + 12 + 16 * num > end:
214 return None
215 furthest = 0
216 tags = set()
217 for i in range(num):
218 rec = at + 12 + 16 * i
219 tag = buf[rec:rec + 4]
220 if not all(0x20 <= c < 0x7f for c in tag):
221 return None
222 tags.add(bytes(tag))
223 off = int.from_bytes(buf[rec + 8:rec + 12], 'big')
224 ln = int.from_bytes(buf[rec + 12:rec + 16], 'big')
225 if off < 12 + 16 * num or at + off + ln > end:
226 return None
227 furthest = max(furthest, off + ln)
228 if b'head' not in tags or not (b'cmap' in tags or b'CFF ' in tags or b'glyf' in tags):
229 return None
230 return (furthest + 3) & ~3
231
232
233def memory_image(buf, segments):
234 """The initial linear memory the active segments build. An optimiser splits the data at runs
235 of zeros, so a font spans many segments; it is only whole again in the image."""
236 placed = [(addr, pos, n) for (pos, n, addr) in segments if addr is not None]
237 if not placed:
238 return b''
239 lo = min(a for (a, _, _) in placed)
240 hi = max(a + n for (a, _, n) in placed)
241 img = bytearray(hi - lo)
242 for (a, pos, n) in placed:
243 img[a - lo:a - lo + n] = buf[pos:pos + n]
244 return bytes(img)
245
246
247def fonts_in(img):
248 found = []
249 end = len(img)
250 i = 0
251 while True:
252 hits = [h for h in (img.find(m, i, end) for m in SFNT_MAGICS) if h >= 0]
253 if not hits:
254 return found
255 h = min(hits)
256 ln = sfnt_length(img, h, end)
257 if ln:
258 found.append((h, ln))
259 i = h + ln
260 else:
261 i = h + 1
262
263
264def font_name(img, at):
265 """The face's own name, from its `name` table: the full name (ID 4), else family and style (IDs 1
266 and 2), else `(unnamed)`. A Windows or Unicode record is UTF-16BE, a Macintosh one Mac Roman, read
267 as Latin-1 for the ASCII names a font carries."""
268 num = int.from_bytes(img[at + 4:at + 6], 'big')
269 for i in range(num):
270 rec = at + 12 + 16 * i
271 if img[rec:rec + 4] != b'name':
272 continue
273 base = at + int.from_bytes(img[rec + 8:rec + 12], 'big')
274 count = int.from_bytes(img[base + 2:base + 4], 'big')
275 store = base + int.from_bytes(img[base + 4:base + 6], 'big')
276 found = {}
277 for k in range(count):
278 r = base + 6 + 12 * k
279 plat = int.from_bytes(img[r:r + 2], 'big')
280 nid = int.from_bytes(img[r + 6:r + 8], 'big')
281 n = int.from_bytes(img[r + 8:r + 10], 'big')
282 o = int.from_bytes(img[r + 10:r + 12], 'big')
283 raw = img[store + o:store + o + n]
284 text = raw.decode('utf-16-be', 'replace') if plat in (0, 3) else raw.decode('latin-1')
285 if plat in (0, 3) or nid not in found:
286 found[nid] = text
287 if 4 in found:
288 return found[4]
289 if 1 in found:
290 return (found[1] + ' ' + found.get(2, '')).strip()
291 return '(unnamed)'
292
293
294def demangle(sym):
295 """Legacy Rust mangling (`_ZN3foo3bar17h0123456789abcdefE`) to a path; anything else unchanged."""
296 if not sym.startswith('_ZN'):
297 return sym
298 i = 3
299 parts = []
300 while i < len(sym) and sym[i] != 'E':
301 j = i
302 while j < len(sym) and sym[j].isdigit():
303 j += 1
304 if j == i:
305 break
306 n = int(sym[i:j])
307 parts.append(sym[j:j + n])
308 i = j + n
309 if parts and len(parts[-1]) == 17 and parts[-1].startswith('h'):
310 parts.pop()
311 return '::'.join(parts)
312
313
314LEGACY_ESCAPES = (
315 ('$LT$', '<'), ('$GT$', '>'), ('$u20$', ' '), ('$RF$', '&'), ('$BP$', '*'), ('$C$', ','),
316 ('$u7b$', '{'), ('$u7d$', '}'), ('$u5b$', '['), ('$u5d$', ']'), ('$u27$', "'"), ('$u22$', '"'),
317 ('$LP$', '('), ('$RP$', ')'), ('$SP$', '@'), ('..', '::'),
318)
319
320
321def unescape(s):
322 """A legacy-mangled segment's escapes (`_$LT$a..b$GT$`) as the path they spell (`<a::b>`)."""
323 if s.startswith('_$'):
324 s = s[1:]
325 for a, b in LEGACY_ESCAPES:
326 s = s.replace(a, b)
327 return s
328
329
330def module_of(name, depth):
331 """The module a function belongs to, to `depth` path segments. A trait impl
332 (`<a::B as c::D>::f`) is charged to its self type's module."""
333 s = unescape(demangle(name))
334 if s.startswith('<'):
335 inner = s[1:]
336 for stop in (' as ', '>'):
337 k = inner.find(stop)
338 if k >= 0:
339 inner = inner[:k]
340 break
341 s = inner.lstrip('&*').replace('mut ', '').replace('const ', '')
342 s = s.split('<')[0]
343 segs = [p for p in s.split('::') if p]
344 if len(segs) <= 1:
345 return '(unattributed)' if not segs else segs[0]
346 return '::'.join(segs[:depth])
347
348
349def measure(path, depth):
350 buf = open(path, 'rb').read()
351 secs = sections(buf)
352 out = {
353 'path': path,
354 'raw': len(buf),
355 'gzip9': len(gzip.compress(buf, 9)),
356 'brotli': len(brotli.compress(buf, quality=11)) if brotli else None,
357 'sections': {},
358 }
359 n_imports = 0
360 bodies = []
361 segments = []
362 names = None
363 for (sid, name, start, size) in secs:
364 out['sections'][name] = out['sections'].get(name, 0) + size
365 if sid == 2:
366 n_imports = imported_functions(buf, start, size)
367 elif sid == 10:
368 bodies = code_bodies(buf, start, size)
369 elif sid == 11:
370 segments = data_segments(buf, start, size)
371 elif name == 'custom:name':
372 names = function_names(buf, start, size)
373 out['code'] = out['sections'].get('code', 0)
374 out['data'] = out['sections'].get('data', 0)
375 out['custom'] = sum(v for k, v in out['sections'].items() if k.startswith('custom:'))
376 img = memory_image(buf, segments)
377 fonts = fonts_in(img)
378 out['fonts'] = [{'offset': h, 'bytes': ln, 'name': font_name(img, h)} for (h, ln) in fonts]
379 out['font_bytes'] = sum(ln for (_, ln) in fonts)
380 out['raw_minus_fonts'] = out['raw'] - out['font_bytes']
381 out['functions'] = len(bodies)
382 if names:
383 per = {}
384 crates = {}
385 for i, n in enumerate(bodies):
386 name = names.get(n_imports + i, '')
387 mod = module_of(name, depth)
388 per[mod] = per.get(mod, 0) + n
389 crate = module_of(name, 1)
390 crates[crate] = crates.get(crate, 0) + n
391 out['modules'] = sorted(per.items(), key=lambda kv: -kv[1])
392 out['crates'] = sorted(crates.items(), key=lambda kv: -kv[1])
393 else:
394 out['modules'] = None
395 out['crates'] = None
396 return out
397
398
399def mb(n):
400 return '-' if n is None else '%.2f MB' % (n / 1048576.0)
401
402
403def report(m, top):
404 print(m['path'])
405 print(' raw %12d %s' % (m['raw'], mb(m['raw'])))
406 print(' gzip -9 %12d %s' % (m['gzip9'], mb(m['gzip9'])))
407 if m['brotli'] is None:
408 print(' brotli %12s (python brotli module unavailable)' % '-')
409 else:
410 print(' brotli %12d %s' % (m['brotli'], mb(m['brotli'])))
411 print(' code section %12d %s (%d functions)' % (m['code'], mb(m['code']), m['functions']))
412 print(' data section %12d %s' % (m['data'], mb(m['data'])))
413 print(' custom %12d %s' % (m['custom'], mb(m['custom'])))
414 print(' fonts in data %12d %s (%d faces)' % (m['font_bytes'], mb(m['font_bytes']), len(m['fonts'])))
415 print(' raw - fonts %12d %s' % (m['raw_minus_fonts'], mb(m['raw_minus_fonts'])))
416 for f in sorted(m['fonts'], key=lambda f: -f['bytes']):
417 print(' face %10d %s' % (f['bytes'], f['name']))
418 if m['modules'] is None:
419 print(' (no name section: build with names retained for per-module attribution)')
420 return
421 for title, rows in (('crate', m['crates']), ('module', m['modules'])):
422 print(' code by %s (top %d):' % (title, top))
423 for mod, n in rows[:top]:
424 print(' %10d %5.1f%% %s' % (n, 100.0 * n / max(1, m['code']), mod))
425
426
427def main():
428 ap = argparse.ArgumentParser(description=__doc__.split('\n')[0])
429 ap.add_argument('wasm', nargs='+')
430 ap.add_argument('--depth', type=int, default=2, help='module path segments for attribution')
431 ap.add_argument('--top', type=int, default=25)
432 ap.add_argument('--json', help='also write the measurements to this file')
433 ap.add_argument('--limit-mb', type=float, help='fail when the first binary is larger (raw)')
434 ap.add_argument('--compare', action='store_true', help='print a side-by-side table last')
435 a = ap.parse_args()
436 ms = []
437 for p in a.wasm:
438 m = measure(p, a.depth)
439 ms.append(m)
440 report(m, a.top)
441 print()
442 if a.compare and len(ms) > 1:
443 print('%-44s %10s %10s %10s %10s %10s' % ('binary', 'raw', 'gzip', 'brotli', 'fonts', 'raw-fonts'))
444 for m in ms:
445 print('%-44s %10s %10s %10s %10s %10s' % (
446 os.path.basename(m['path'])[:44], mb(m['raw']), mb(m['gzip9']), mb(m['brotli']),
447 mb(m['font_bytes']), mb(m['raw_minus_fonts'])))
448 base = ms[0]
449 for m in ms[1:]:
450 print(' %s / %s: raw %.1f%%, raw minus fonts %.1f%%' % (
451 os.path.basename(base['path']), os.path.basename(m['path']),
452 100.0 * base['raw'] / max(1, m['raw']),
453 100.0 * base['raw_minus_fonts'] / max(1, m['raw_minus_fonts'])))
454 if a.json:
455 with open(a.json, 'w') as f:
456 json.dump(ms, f, indent=1)
457 if a.limit_mb is not None:
458 limit = a.limit_mb * 1048576.0
459 ok = ms[0]['raw'] <= limit
460 print('pass line: %s raw %s against %.1f MB: %s' % (
461 os.path.basename(ms[0]['path']), mb(ms[0]['raw']), a.limit_mb, 'PASS' if ok else 'FAIL'))
462 return 0 if ok else 1
463 return 0
464
465
466if __name__ == '__main__':
467 sys.exit(main())