oxedyne/fe2o3/fe2o3_austenite/tools/mem_open_probe.mjs
8.0 KiB, 1 run, executable
created by r1870400018:66298, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | #!/usr/bin/env node |
| 2 | // mem_open_probe.mjs -- where the wasm heap goes on the live view's open, one experiment a process. |
| 3 | // |
| 4 | // The wasm linear memory only grows, so `heapMB()` after a call is that process's high-water mark so |
| 5 | // far. Each experiment runs in a fresh node process and prints one JSON line with the heap after each |
| 6 | // of its calls, the bytes of SVG the delta carried, and the page count. |
| 7 | // |
| 8 | // mem_open_probe.mjs --pkg DIR --doc FILE.typ --exp NAME [--order FILE] [--out FILE] |
| 9 | // |
| 10 | // Experiments: |
| 11 | // pdf one `compileProject` (PDF): the engine's own working set, the sink streaming; |
| 12 | // open `compileProjectDelta` with `known: []`, three times in one instance: does the second |
| 13 | // and third open reuse the heap the first grew, or grow it again; |
| 14 | // vector one `compileProjectVector` (every page's SVG in one array), the other live-view door; |
| 15 | // pdf_open a PDF, then an open: is the open's heap the engine's, or the open's own; |
| 16 | // open_pdf an open, then a PDF in the same instance: does the heap come back down (it cannot); |
| 17 | // open1 one open, the result held as a consumer holds it, with node's own figures beside the wasm's |
| 18 | // (RSS, V8 heap, external and array buffers) before, after and, run with --expose-gc, |
| 19 | // after the result is dropped (then --edits N edits, if asked), and the process RSS sampled every 250 ms from a worker thread |
| 20 | // while the open runs (the call is synchronous, so the wasm size cannot be read inside it; |
| 21 | // the partial experiments give it at 1, 10, 30 and 100 pages held instead); |
| 22 | // edits an open, then --edits one-letter edits carrying `known`: what an edit adds; |
| 23 | // ids an open, writing the page ids in order to --out for `partial`; |
| 24 | // partial:K an open whose `known` holds every id but the last K distinct ones (from --order), so |
| 25 | // K pages are rendered to SVG and held, the rest only laid out, hashed and dropped. |
| 26 | // |
| 27 | // The edits are heap_probe.mjs's: a letter typed onto a plain word of a body paragraph, the paragraphs |
| 28 | // spread evenly through the document. |
| 29 | import fs from 'node:fs'; |
| 30 | import path from 'node:path'; |
| 31 | import { pathToFileURL } from 'node:url'; |
| 32 | import { Worker } from 'node:worker_threads'; |
| 33 | |
| 34 | function arg(name, dflt) { |
| 35 | const a = process.argv.find((x) => x.startsWith(`--${name}=`)); |
| 36 | if (a) return a.slice(name.length + 3); |
| 37 | const i = process.argv.indexOf(`--${name}`); |
| 38 | return i >= 0 && i + 1 < process.argv.length ? process.argv[i + 1] : dflt; |
| 39 | } |
| 40 | |
| 41 | const pkg = arg('pkg'); |
| 42 | const doc = arg('doc'); |
| 43 | const exp = arg('exp'); |
| 44 | if (!pkg || !doc || !exp) { |
| 45 | process.stderr.write('mem_open_probe: --pkg, --doc and --exp are required\n'); |
| 46 | process.exit(2); |
| 47 | } |
| 48 | |
| 49 | const mod = await import(pathToFileURL(path.join(pkg, 'oxedyne_fe2o3_austenite.js')).href); |
| 50 | const wasm = fs.readFileSync(path.join(pkg, 'oxedyne_fe2o3_austenite_bg.wasm')); |
| 51 | const exports = await mod.default({ module_or_path: wasm }); |
| 52 | const aus = new mod.DaimondTypst(); |
| 53 | const src = fs.readFileSync(doc, 'utf8'); |
| 54 | const mib = () => exports.memory.buffer.byteLength / 1048576; |
| 55 | const hwm = () => { |
| 56 | const m = /VmHWM:\s+(\d+) kB/.exec(fs.readFileSync('/proc/self/status', 'utf8')); |
| 57 | return m ? +m[1] / 1024 : 0; |
| 58 | }; |
| 59 | const project = (text, extra) => ({ main: '/main.typ', sources: [['/main.typ', text]], assets: [], fonts: [], strict: true, ...extra }); |
| 60 | |
| 61 | const rec = { exp, doc: path.basename(doc), heap_start_mb: mib(), calls: [] }; |
| 62 | const note = (what, ret, extra) => { |
| 63 | const e = { what, heap_mb: mib(), ...extra }; |
| 64 | if (ret && ret.error) e.error = String(ret.error).slice(0, 200); |
| 65 | if (ret && ret.pages !== undefined) e.pages = ret.pages; |
| 66 | rec.calls.push(e); |
| 67 | return e; |
| 68 | }; |
| 69 | const svgBytes = (ret) => { |
| 70 | let n = 0; |
| 71 | for (const c of ret.changed || []) n += c.svg.length; |
| 72 | return n; |
| 73 | }; |
| 74 | const open = (known) => { |
| 75 | const t = performance.now(); |
| 76 | const ret = aus.compileProjectDelta(project(src, { known })); |
| 77 | const ms = Math.round(performance.now() - t); |
| 78 | if (!ret || ret.error) return { ret, e: note('open', ret, { ms }) }; |
| 79 | const e = note('open', ret, { ms, changed: ret.changed.length, svg_bytes: svgBytes(ret), order: ret.order.length, distinct: new Set(ret.order).size }); |
| 80 | return { ret, e }; |
| 81 | }; |
| 82 | const pdf = () => { |
| 83 | const t = performance.now(); |
| 84 | const ret = aus.compileProject(project(src)); |
| 85 | const e = note('pdf', ret, { ms: Math.round(performance.now() - t), bytes: ret.pdf ? ret.pdf.length : 0 }); |
| 86 | return { ret, e }; |
| 87 | }; |
| 88 | |
| 89 | // `n` one-letter edits after an open, each carrying the previous `order` as `known`. |
| 90 | const runEdits = (known, n) => { |
| 91 | let text = src; |
| 92 | const lines = text.split('\n'); |
| 93 | const body = []; |
| 94 | for (let i = 0; i < lines.length; i++) { |
| 95 | if (lines[i].length > 80 && !/^[=\-#$]/.test(lines[i])) body.push(i); |
| 96 | } |
| 97 | for (let k = 0; k < n; k++) { |
| 98 | const at = body[Math.min(Math.floor(((k + 0.5) / n) * body.length), body.length - 1)]; |
| 99 | const words = lines[at].split(' '); |
| 100 | const w = words.findIndex((x) => /^[a-z]{4,}$/.test(x)); |
| 101 | words[w] += 'x'; |
| 102 | lines[at] = words.join(' '); |
| 103 | text = lines.join('\n'); |
| 104 | const t = performance.now(); |
| 105 | const r = aus.compileProjectDelta(project(text, { known })); |
| 106 | const e = note('edit', r, { ms: Math.round(performance.now() - t) }); |
| 107 | if (r && !r.error) { |
| 108 | e.changed = r.changed.length; |
| 109 | e.svg_bytes = svgBytes(r); |
| 110 | known = r.order; |
| 111 | } |
| 112 | } |
| 113 | }; |
| 114 | |
| 115 | if (exp === 'pdf') { |
| 116 | pdf(); |
| 117 | } else if (exp === 'open') { |
| 118 | for (let i = 0; i < 3; i++) open([]); |
| 119 | } else if (exp === 'vector') { |
| 120 | const t = performance.now(); |
| 121 | const ret = aus.compileProjectVector(project(src)); |
| 122 | let n = 0; |
| 123 | for (const s of ret.svg || []) n += s.length; |
| 124 | note('vector', ret, { ms: Math.round(performance.now() - t), svg_bytes: n }); |
| 125 | } else if (exp === 'pdf_open') { |
| 126 | pdf(); |
| 127 | open([]); |
| 128 | } else if (exp === 'open_pdf') { |
| 129 | open([]); |
| 130 | pdf(); |
| 131 | } else if (exp === 'open1') { |
| 132 | const use = () => { |
| 133 | const u = process.memoryUsage(); |
| 134 | const m = (b) => Math.round((b / 1048576) * 10) / 10; |
| 135 | return { wasm_mb: mib(), rss_mb: m(u.rss), v8_heap_used_mb: m(u.heapUsed), external_mb: m(u.external), array_buffers_mb: m(u.arrayBuffers), vmhwm_mb: Math.round(hwm()) }; |
| 136 | }; |
| 137 | // A worker thread keeps sampling the process RSS while the main thread is inside the wasm call. |
| 138 | const sab = new SharedArrayBuffer(4 * (1 + 2 * 8192)); |
| 139 | const cells = new Int32Array(sab); |
| 140 | const worker = new Worker(` |
| 141 | const { workerData } = require('node:worker_threads'); |
| 142 | const fs = require('node:fs'); |
| 143 | const a = new Int32Array(workerData); |
| 144 | const t0 = Date.now(); |
| 145 | let n = 0; |
| 146 | setInterval(() => { |
| 147 | if (n >= 8190) return; |
| 148 | const m = /VmRSS:\\s+(\\d+) kB/.exec(fs.readFileSync('/proc/self/status', 'utf8')); |
| 149 | a[1 + 2 * n] = Date.now() - t0; |
| 150 | a[2 + 2 * n] = m ? +m[1] : 0; |
| 151 | n++; |
| 152 | Atomics.store(a, 0, n); |
| 153 | }, 250); |
| 154 | `, { eval: true, workerData: sab }); |
| 155 | await new Promise((r) => setTimeout(r, 600)); |
| 156 | rec.before = use(); |
| 157 | let held = open([]).ret; |
| 158 | rec.after = use(); |
| 159 | const n = Atomics.load(cells, 0); |
| 160 | const samples = []; |
| 161 | for (let i = 0; i < n; i++) samples.push([cells[1 + 2 * i] / 1000, Math.round(cells[2 + 2 * i] / 1024)]); |
| 162 | rec.rss_samples_s_mb = samples.filter((_, i) => i % 8 === 0 || i === n - 1); |
| 163 | rec.rss_sample_max_mb = Math.max(...samples.map((x) => x[1])); |
| 164 | await worker.terminate(); |
| 165 | const order = held.order; |
| 166 | held = null; |
| 167 | if (global.gc) { |
| 168 | global.gc(); |
| 169 | rec.after_gc = use(); |
| 170 | } |
| 171 | if (+arg('edits', '0') > 0) { |
| 172 | runEdits(order, +arg('edits', '0')); |
| 173 | rec.after_edits = use(); |
| 174 | } |
| 175 | } else if (exp === 'edits') { |
| 176 | const { ret } = open([]); |
| 177 | runEdits(ret.order, +arg('edits', '5')); |
| 178 | } else if (exp === 'ids') { |
| 179 | const { ret } = open([]); |
| 180 | fs.writeFileSync(arg('out'), JSON.stringify(ret.order)); |
| 181 | } else if (exp.startsWith('partial:')) { |
| 182 | const k = +exp.slice(8); |
| 183 | const order = JSON.parse(fs.readFileSync(arg('order'), 'utf8')); |
| 184 | const distinct = [...new Set(order)]; |
| 185 | const known = distinct.slice(0, Math.max(distinct.length - k, 0)); |
| 186 | open(known); |
| 187 | } else { |
| 188 | process.stderr.write(`mem_open_probe: unknown experiment ${exp}\n`); |
| 189 | process.exit(2); |
| 190 | } |
| 191 | rec.heap_end_mb = mib(); |
| 192 | rec.vmhwm_mb = hwm(); |
| 193 | process.stdout.write(JSON.stringify(rec) + '\n'); |