Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_austenite/tools/mem_open_probe.mjs

8.0 KiB, 1 run, executable

created by r1870400018:66298, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1#!/usr/bin/env node
2// mem_open_probe.mjs -- where the wasm heap goes on the live view's open, one experiment a process.
3//
4// The wasm linear memory only grows, so `heapMB()` after a call is that process's high-water mark so
5// far. Each experiment runs in a fresh node process and prints one JSON line with the heap after each
6// of its calls, the bytes of SVG the delta carried, and the page count.
7//
8// mem_open_probe.mjs --pkg DIR --doc FILE.typ --exp NAME [--order FILE] [--out FILE]
9//
10// Experiments:
11// pdf one `compileProject` (PDF): the engine's own working set, the sink streaming;
12// open `compileProjectDelta` with `known: []`, three times in one instance: does the second
13// and third open reuse the heap the first grew, or grow it again;
14// vector one `compileProjectVector` (every page's SVG in one array), the other live-view door;
15// pdf_open a PDF, then an open: is the open's heap the engine's, or the open's own;
16// open_pdf an open, then a PDF in the same instance: does the heap come back down (it cannot);
17// open1 one open, the result held as a consumer holds it, with node's own figures beside the wasm's
18// (RSS, V8 heap, external and array buffers) before, after and, run with --expose-gc,
19// after the result is dropped (then --edits N edits, if asked), and the process RSS sampled every 250 ms from a worker thread
20// while the open runs (the call is synchronous, so the wasm size cannot be read inside it;
21// the partial experiments give it at 1, 10, 30 and 100 pages held instead);
22// edits an open, then --edits one-letter edits carrying `known`: what an edit adds;
23// ids an open, writing the page ids in order to --out for `partial`;
24// partial:K an open whose `known` holds every id but the last K distinct ones (from --order), so
25// K pages are rendered to SVG and held, the rest only laid out, hashed and dropped.
26//
27// The edits are heap_probe.mjs's: a letter typed onto a plain word of a body paragraph, the paragraphs
28// spread evenly through the document.
29import fs from 'node:fs';
30import path from 'node:path';
31import { pathToFileURL } from 'node:url';
32import { Worker } from 'node:worker_threads';
33
34function arg(name, dflt) {
35 const a = process.argv.find((x) => x.startsWith(`--${name}=`));
36 if (a) return a.slice(name.length + 3);
37 const i = process.argv.indexOf(`--${name}`);
38 return i >= 0 && i + 1 < process.argv.length ? process.argv[i + 1] : dflt;
39}
40
41const pkg = arg('pkg');
42const doc = arg('doc');
43const exp = arg('exp');
44if (!pkg || !doc || !exp) {
45 process.stderr.write('mem_open_probe: --pkg, --doc and --exp are required\n');
46 process.exit(2);
47}
48
49const mod = await import(pathToFileURL(path.join(pkg, 'oxedyne_fe2o3_austenite.js')).href);
50const wasm = fs.readFileSync(path.join(pkg, 'oxedyne_fe2o3_austenite_bg.wasm'));
51const exports = await mod.default({ module_or_path: wasm });
52const aus = new mod.DaimondTypst();
53const src = fs.readFileSync(doc, 'utf8');
54const mib = () => exports.memory.buffer.byteLength / 1048576;
55const hwm = () => {
56 const m = /VmHWM:\s+(\d+) kB/.exec(fs.readFileSync('/proc/self/status', 'utf8'));
57 return m ? +m[1] / 1024 : 0;
58};
59const project = (text, extra) => ({ main: '/main.typ', sources: [['/main.typ', text]], assets: [], fonts: [], strict: true, ...extra });
60
61const rec = { exp, doc: path.basename(doc), heap_start_mb: mib(), calls: [] };
62const note = (what, ret, extra) => {
63 const e = { what, heap_mb: mib(), ...extra };
64 if (ret && ret.error) e.error = String(ret.error).slice(0, 200);
65 if (ret && ret.pages !== undefined) e.pages = ret.pages;
66 rec.calls.push(e);
67 return e;
68};
69const svgBytes = (ret) => {
70 let n = 0;
71 for (const c of ret.changed || []) n += c.svg.length;
72 return n;
73};
74const open = (known) => {
75 const t = performance.now();
76 const ret = aus.compileProjectDelta(project(src, { known }));
77 const ms = Math.round(performance.now() - t);
78 if (!ret || ret.error) return { ret, e: note('open', ret, { ms }) };
79 const e = note('open', ret, { ms, changed: ret.changed.length, svg_bytes: svgBytes(ret), order: ret.order.length, distinct: new Set(ret.order).size });
80 return { ret, e };
81};
82const pdf = () => {
83 const t = performance.now();
84 const ret = aus.compileProject(project(src));
85 const e = note('pdf', ret, { ms: Math.round(performance.now() - t), bytes: ret.pdf ? ret.pdf.length : 0 });
86 return { ret, e };
87};
88
89// `n` one-letter edits after an open, each carrying the previous `order` as `known`.
90const runEdits = (known, n) => {
91 let text = src;
92 const lines = text.split('\n');
93 const body = [];
94 for (let i = 0; i < lines.length; i++) {
95 if (lines[i].length > 80 && !/^[=\-#$]/.test(lines[i])) body.push(i);
96 }
97 for (let k = 0; k < n; k++) {
98 const at = body[Math.min(Math.floor(((k + 0.5) / n) * body.length), body.length - 1)];
99 const words = lines[at].split(' ');
100 const w = words.findIndex((x) => /^[a-z]{4,}$/.test(x));
101 words[w] += 'x';
102 lines[at] = words.join(' ');
103 text = lines.join('\n');
104 const t = performance.now();
105 const r = aus.compileProjectDelta(project(text, { known }));
106 const e = note('edit', r, { ms: Math.round(performance.now() - t) });
107 if (r && !r.error) {
108 e.changed = r.changed.length;
109 e.svg_bytes = svgBytes(r);
110 known = r.order;
111 }
112 }
113};
114
115if (exp === 'pdf') {
116 pdf();
117} else if (exp === 'open') {
118 for (let i = 0; i < 3; i++) open([]);
119} else if (exp === 'vector') {
120 const t = performance.now();
121 const ret = aus.compileProjectVector(project(src));
122 let n = 0;
123 for (const s of ret.svg || []) n += s.length;
124 note('vector', ret, { ms: Math.round(performance.now() - t), svg_bytes: n });
125} else if (exp === 'pdf_open') {
126 pdf();
127 open([]);
128} else if (exp === 'open_pdf') {
129 open([]);
130 pdf();
131} else if (exp === 'open1') {
132 const use = () => {
133 const u = process.memoryUsage();
134 const m = (b) => Math.round((b / 1048576) * 10) / 10;
135 return { wasm_mb: mib(), rss_mb: m(u.rss), v8_heap_used_mb: m(u.heapUsed), external_mb: m(u.external), array_buffers_mb: m(u.arrayBuffers), vmhwm_mb: Math.round(hwm()) };
136 };
137 // A worker thread keeps sampling the process RSS while the main thread is inside the wasm call.
138 const sab = new SharedArrayBuffer(4 * (1 + 2 * 8192));
139 const cells = new Int32Array(sab);
140 const worker = new Worker(`
141 const { workerData } = require('node:worker_threads');
142 const fs = require('node:fs');
143 const a = new Int32Array(workerData);
144 const t0 = Date.now();
145 let n = 0;
146 setInterval(() => {
147 if (n >= 8190) return;
148 const m = /VmRSS:\\s+(\\d+) kB/.exec(fs.readFileSync('/proc/self/status', 'utf8'));
149 a[1 + 2 * n] = Date.now() - t0;
150 a[2 + 2 * n] = m ? +m[1] : 0;
151 n++;
152 Atomics.store(a, 0, n);
153 }, 250);
154 `, { eval: true, workerData: sab });
155 await new Promise((r) => setTimeout(r, 600));
156 rec.before = use();
157 let held = open([]).ret;
158 rec.after = use();
159 const n = Atomics.load(cells, 0);
160 const samples = [];
161 for (let i = 0; i < n; i++) samples.push([cells[1 + 2 * i] / 1000, Math.round(cells[2 + 2 * i] / 1024)]);
162 rec.rss_samples_s_mb = samples.filter((_, i) => i % 8 === 0 || i === n - 1);
163 rec.rss_sample_max_mb = Math.max(...samples.map((x) => x[1]));
164 await worker.terminate();
165 const order = held.order;
166 held = null;
167 if (global.gc) {
168 global.gc();
169 rec.after_gc = use();
170 }
171 if (+arg('edits', '0') > 0) {
172 runEdits(order, +arg('edits', '0'));
173 rec.after_edits = use();
174 }
175} else if (exp === 'edits') {
176 const { ret } = open([]);
177 runEdits(ret.order, +arg('edits', '5'));
178} else if (exp === 'ids') {
179 const { ret } = open([]);
180 fs.writeFileSync(arg('out'), JSON.stringify(ret.order));
181} else if (exp.startsWith('partial:')) {
182 const k = +exp.slice(8);
183 const order = JSON.parse(fs.readFileSync(arg('order'), 'utf8'));
184 const distinct = [...new Set(order)];
185 const known = distinct.slice(0, Math.max(distinct.length - k, 0));
186 open(known);
187} else {
188 process.stderr.write(`mem_open_probe: unknown experiment ${exp}\n`);
189 process.exit(2);
190}
191rec.heap_end_mb = mib();
192rec.vmhwm_mb = hwm();
193process.stdout.write(JSON.stringify(rec) + '\n');