Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_spendowner.mjs

19.9 KiB, 1 run

created by r2519314175:689, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_spendowner.mjs — spend is billed to the thing that SPENT it, not to
2// whatever the rail happens to be showing when the money is counted.
3//
4// `recordSpend` attributed every metered turn to `currentDiamond && currentDiamond.id`.
5// All three of its callers reach it after an await, so the figure on a Diamond tile
6// -- the one whose own stylesheet comment says it exists to answer "which Diamond is
7// eating the money" -- recorded where the USER was, not where the WORK was:
8//
9// * a Diamond fans four workers out and the user goes to look at another
10// Diamond. Each worker lands and is billed to whatever is on screen. That is
11// not an edge case; leaving a fan-out running is what a fan-out is FOR;
12// * the user opens an ordinary chat instead. `selectChat` nulls the global with
13// the comment "a chat is not a Diamond", so the same spend is billed to nobody;
14// * and the chat's own turn, landing while a Diamond is open behind it, was
15// billed to that Diamond -- money spent on a conversation it had never seen.
16//
17// Measured before the fix, in this world, with the real page: a worker dispatched by
18// `0da1000000f2` and landing while `0da1000000e1` was selected went into the index as
19// {"0da1000000e1":{"turns":1,"usd":0.0000372}}
20//
21// THE CHECK THAT MATTERS IS ATTRIBUTION UNDER NAVIGATION, and it must not be
22// passable by accident. Every attribution check here is paired with a CONTROL that
23// reads back what was selected at the instant the money was counted -- through
24// `DaimondDiamond.current()`, which is the very global the defect read -- so a run
25// where the work landed before the user moved fails as loudly as a wrong id.
26//
27// It also holds the invariant behind `maxOutCeiling`'s note (defect M): `modelFamily`
28// still resolves by containment where `DaimondPricing.contextWindow` no longer does,
29// and that is only safe while every `MAX_OUT` row sits at or below its model's
30// published window, which is what makes a borrowed ceiling tighten rather than widen.
31// That is data, so it is checked rather than argued about.
32//
33// node dev/verify_spendowner.mjs --break onscreen # the defect, exactly as it was
34// node dev/verify_spendowner.mjs --break workerarg # the worker forgets its Diamond
35// node dev/verify_spendowner.mjs --break meterarg # the daimon turn forgets its own
36// node dev/verify_spendowner.mjs --break maxoutwide # a ceiling above its own window
37// node dev/verify_spendowner.mjs --break ceilwins # the ceiling widens instead of tightening
38// node dev/verify_spendowner.mjs # and then, clean
39import fs from 'node:fs';
40import path from 'node:path';
41import vm from 'node:vm';
42import { fileURLToPath } from 'node:url';
43import { open, newChat, errors } from './harness.mjs';
44
45const HERE = path.dirname(fileURLToPath(import.meta.url));
46const WWW = path.join(HERE, '..', 'www');
47
48const ok = [], bad = [];
49const check = (name, pass, detail) => {
50 (pass ? ok : bad).push(name);
51 console.log((pass ? ' ok ' : ' FAIL ') + name + (detail ? ' — ' + detail : ''));
52};
53const die = (why) => { console.error('ABORT: ' + why); process.exit(2); };
54
55// ── The breaks ─────────────────────────────────────────────────────
56//
57// Each restores ONE half of the defect in the source text of `daimond.js` as it is
58// SERVED to the page, so what runs is the shipped file with one line changed rather
59// than a copy of it. Three, not one, because the fix has three independent halves
60// and a single break reddening all of them would not say which one was missing.
61const BREAK = (() => {
62 const i = process.argv.indexOf('--break');
63 return i > 0 ? String(process.argv[i + 1] || '') : '';
64})();
65const BREAKS = {
66 onscreen: {
67 what: 'recordSpend reads the selected Diamond again — the defect as it shipped',
68 edit: (src) => src.replace(
69 '\t\t\t\t\tdiamondId: diamondId || \'\',',
70 '\t\t\t\t\tdiamondId: (currentDiamond && currentDiamond.id) || \'\','),
71 },
72 workerarg: {
73 what: 'a worker stops naming the Diamond that dispatched it',
74 edit: (src) => src.replace(
75 'recordSpend(run.model, _pt, _ct, _ca, _cost, run.provider, run.diamondId || \'\');',
76 'recordSpend(run.model, _pt, _ct, _ca, _cost, run.provider);'),
77 },
78 meterarg: {
79 what: 'a daimon turn stops naming its own Diamond',
80 edit: (src) => src.split('meterDiamondTurn(fa, diamondId);').join('meterDiamondTurn(fa);'),
81 },
82 maxoutwide: {
83 what: 'a MAX_OUT ceiling above its own model\'s window, so a borrowed one could widen',
84 pure: true,
85 edit: (src) => src.replace(
86 '\t\t\'claude-opus-4-1\': 32000,',
87 '\t\t\'claude-opus-4-1\': 320000,'),
88 },
89 ceilwins: {
90 what: 'maxOutCeiling takes the LARGER of the table row and the window, so a borrowed'
91 + ' ceiling widens instead of tightening',
92 pure: true,
93 edit: (src) => src.replace(
94 '\t\tif (pub && ctx) return Math.min(pub, ctx);',
95 '\t\tif (pub && ctx) return Math.max(pub, ctx);'),
96 },
97};
98if (BREAK && !BREAKS[BREAK]) die(`no break called "${BREAK}"`);
99if (BREAK) console.log(`\n*** BREAK ${BREAK}: ${BREAKS[BREAK].what} — failures below are the point ***\n`);
100
101const SRC = fs.readFileSync(path.join(WWW, 'js/daimond.js'), 'utf8');
102const BROKEN = BREAKS[BREAK] ? BREAKS[BREAK].edit(SRC) : SRC;
103if (BREAK && BROKEN === SRC) die(`the "${BREAK}" break no longer matches www/js/daimond.js`);
104
105// ══════════════════════════════════════════════════════════════════
106// 1. Two resolvers, one table (defect M) — pure, no browser
107// ══════════════════════════════════════════════════════════════════
108//
109// `modelFamily` and its ceiling table are read OUT OF THE SOURCE rather than
110// re-typed here. A copy would go stale the day a model is added, which is the same
111// failure mode -- a second list of the same facts -- that the note in `maxOutCeiling`
112// is about.
113console.log('\n1. modelFamily, MAX_OUT and the tightening property\n');
114
115const cut = (label, re) => {
116 const m = BROKEN.match(re);
117 if (!m) die(`could not find ${label} in www/js/daimond.js; the anchor has moved`);
118 return m[0];
119};
120const MAX_OUT_SRC = cut('the MAX_OUT table', /\tvar MAX_OUT = \{[\s\S]*?\n\t\};/);
121const MODELFAMILY_SRC = cut('modelFamily', /\tfunction modelFamily\(model\) \{[\s\S]*?\n\t\}/);
122const CEILING_SRC = cut('maxOutCeiling', /\tfunction maxOutCeiling\(model, provider\) \{[\s\S]*?\n\t\}/);
123
124const sandbox = { window: {}, console };
125vm.createContext(sandbox);
126vm.runInContext(fs.readFileSync(path.join(WWW, 'js/pricing.js'), 'utf8'), sandbox);
127// `window` is the global object in a browser, so the extracted source reaches
128// `DaimondPricing` bare. In a sandbox it is a property, hence the alias.
129vm.runInContext('var DaimondPricing = window.DaimondPricing;', sandbox);
130vm.runInContext(`${MAX_OUT_SRC}\n${MODELFAMILY_SRC}\n${CEILING_SRC}\n`
131 + 'this.MAX_OUT = MAX_OUT; this.modelFamily = modelFamily; this.maxOutCeiling = maxOutCeiling;',
132 sandbox);
133const { MAX_OUT, modelFamily, maxOutCeiling } = sandbox;
134const C = sandbox.window.DaimondPricing._core;
135
136// Every canonical id and every alias must land on ITSELF. Containment is only ever
137// a fallback, and a table that collides with itself would have it deciding cases it
138// was never meant to see.
139{
140 const wrong = [];
141 for (const id of Object.keys(C.TABLE)) {
142 if (modelFamily(id) !== id) wrong.push(`${id} -> ${modelFamily(id) || '(nothing)'}`);
143 for (const a of (C.TABLE[id].alias || [])) {
144 if (modelFamily(a) !== id) wrong.push(`${a} -> ${modelFamily(a) || '(nothing)'}`);
145 }
146 }
147 check('every table id and alias resolves to itself', wrong.length === 0, wrong.slice(0, 3).join('; '));
148}
149
150// A MAX_OUT key that is not a table id is a ceiling nothing can ever reach:
151// `modelFamily` answers canonical ids, so a typo here is silently dead.
152{
153 const orphans = Object.keys(MAX_OUT).filter(k => !C.TABLE[k]);
154 check('every MAX_OUT key is a real table id', orphans.length === 0, orphans.join(', '));
155}
156
157// THE INVARIANT THE NOTE RESTS ON. `maxOutCeiling` takes `min(pub, ctx)`, so a
158// borrowed ceiling can only ever be smaller than the answer with no row at all --
159// which is what makes a wrong containment hit degrade into a shorter reply, said
160// out loud by `capCut`, rather than into a request the provider refuses.
161{
162 const over = Object.keys(MAX_OUT).filter((k) => {
163 const ctx = C.TABLE[k] && C.TABLE[k].ctx;
164 return ctx == null || MAX_OUT[k] > ctx;
165 }).map(k => `${k}: out ${MAX_OUT[k]} > ctx ${(C.TABLE[k] || {}).ctx}`);
166 check('every MAX_OUT ceiling sits at or below its model\'s window',
167 over.length === 0, over.join('; '));
168}
169
170// And the property itself, measured on the real function rather than reasoned about:
171// for every drifted id that containment places on a MAX_OUT row, the ceiling it
172// yields must be no larger than the one the same id would get with no row at all.
173//
174// A PUBLISHED WINDOW HAS TO BE IN PLAY OR THIS CHECK IS VACUOUS. `contextWindow`
175// answers null for a containment-only id -- that was defect 6's whole fix -- so with
176// no provider the comparison degenerates to "a number is smaller than no ceiling",
177// which is true of anything and proves nothing. So a provider IS supplied, with a
178// window under every MAX_OUT row, which is the only arrangement in which the two
179// answers can differ at all: `min(pub, ctx)` against `ctx`.
180{
181 const LIVE_CTX = 50000; // below every MAX_OUT row, so `min` has work to do
182 sandbox.window.DaimondModels = {
183 rateFor: (provider, model) => (provider ? { inPerM: 1, outPerM: 1, ctx: LIVE_CTX } : null),
184 };
185 const ids = [...Object.keys(C.TABLE), ...Object.values(C.TABLE).flatMap(e => e.alias || [])];
186 const drift = ['-20251001', '@20260101', '-latest', '-thinking', ':free', '-1', '-3', '.3', '.7'];
187 const pre = ['anthropic/', 'us.anthropic.', 'openrouter/'];
188 const probes = [];
189 for (const b of ids) { for (const d of drift) probes.push(b + d); for (const p of pre) probes.push(p + b); }
190 const hits = probes.filter(m => !C.INDEX[C.norm(m)] && MAX_OUT[modelFamily(m)]);
191 const widened = hits.filter((m) => {
192 // What the id would get with no borrowed row: the window the provider published.
193 const without = sandbox.window.DaimondPricing.contextWindow(m, 'live') || 0;
194 return maxOutCeiling(m, 'live') > without;
195 });
196 check('a borrowed output ceiling can only tighten, never widen',
197 widened.length === 0 && hits.length > 0,
198 `${hits.length} containment hits examined against a ${LIVE_CTX} window, `
199 + `${widened.length} widened` + (widened.length ? ': ' + widened.slice(0, 3).join(', ') : ''));
200 delete sandbox.window.DaimondModels;
201}
202
203if (BREAKS[BREAK] && BREAKS[BREAK].pure) {
204 // A pure break says nothing about the browser half, so the browser half is not
205 // run: a page reporting green under a break aimed elsewhere reads as a pass.
206 console.log(`\n${ok.length} ok, ${bad.length} failed`);
207 if (bad.length) console.log(' ' + bad.join('\n '));
208 console.log(bad.length ? '\nTHE BREAK WAS CAUGHT.' : '\nTHE BREAK WAS NOT CAUGHT: this check proves nothing');
209 process.exit(bad.length ? 0 : 1);
210}
211
212// ══════════════════════════════════════════════════════════════════
213// 2. Attribution under navigation — the real page
214// ══════════════════════════════════════════════════════════════════
215
216const s = await open({
217 name: 'spendowner' + (BREAK ? '-' + BREAK : ''),
218 route: BREAK ? (async (page) => {
219 await page.route('**/js/daimond.js', (r) => r.fulfill({
220 status: 200, contentType: 'application/javascript', body: BROKEN,
221 }));
222 }) : null,
223});
224const p = s.page;
225
226await p.waitForFunction(() => !!window.DaimondSignals && !!window.DaimondWorkers
227 && !!window.DaimondDiamond && !!window.DaimondModels, null, { timeout: 20000 });
228
229// The two Diamonds the app seeds on a first boot. Named A and B here and nowhere
230// else: which is which does not matter, only that they are two.
231const [A, B] = await p.evaluate(() =>
232 [...document.querySelectorAll('#diamond-list .diamond-box')].map(b => b.dataset.id));
233if (!A || !B || A === B) die('this world does not hold two Diamonds to navigate between');
234
235// THE INSTRUMENT. Every call into the index is recorded with what was SELECTED at
236// that instant, read through `DaimondDiamond.current()` -- the same global the
237// defect read. Without this the checks below could pass on a run where the work
238// landed before the user moved, and prove nothing whatever.
239const spy = async () => p.evaluate(() => window.__spendSpy.slice());
240const armSpy = () => p.evaluate(() => {
241 if (!window.__spendSpyOn) {
242 window.__spendSpyOn = true;
243 const real = window.DaimondSignals.noteTurn;
244 window.DaimondSignals.noteTurn = function (ev) {
245 const cur = window.DaimondDiamond.current();
246 window.__spendSpy.push({
247 billed: (ev && ev.diamondId) || '',
248 onScreen: cur ? cur.id : '',
249 usd: (ev && ev.usd) || 0,
250 });
251 return real.apply(this, arguments);
252 };
253 }
254 window.__spendSpy = [];
255 window.DaimondSignals.reset();
256});
257const goDiamond = (id) => p.evaluate((i) => {
258 // `el.click()`, not Playwright's: headless Chrome without a display gives the
259 // rail no frame to consider stable, and the actionability wait never resolves.
260 document.querySelector(`#diamond-list .diamond-box[data-id="${i}"]`).click();
261}, id);
262const waitSpy = async (n, ms = 30000) => {
263 const t0 = Date.now();
264 while (Date.now() - t0 < ms) {
265 if ((await p.evaluate(() => window.__spendSpy.length)) >= n) return true;
266 await p.waitForTimeout(200);
267 }
268 return false;
269};
270const pick = await p.evaluate(() => {
271 const r = window.DaimondModels.resolve('', '');
272 return r ? { provider: r.provider, model: r.model } : null;
273});
274if (!pick) die('no model is connected in this world, so nothing can spend anything');
275
276// ── 2a. A worker of A's, landing while B is on screen ──────────────
277console.log('\n2. a worker lands while another Diamond is on screen\n');
278{
279 await armSpy();
280 await goDiamond(A);
281 await p.waitForTimeout(200);
282 await p.evaluate(({ A, pick }) => window.DaimondWorkers.dispatch(
283 A, 'A', [{ name: 'away', task: '@slow 5000' }], false, pick, 0), { A, pick });
284 await p.waitForTimeout(700);
285 await goDiamond(B);
286 await p.waitForTimeout(300);
287 const during = await p.evaluate(() => ({
288 sel: (window.DaimondDiamond.current() || {}).id || '',
289 running: window.DaimondWorkers.runs.filter(r => r.status === 'running' || r.status === 'queued').length,
290 }));
291 check('the control: the worker was still running when B came up',
292 during.running > 0 && during.sel === B, JSON.stringify(during));
293
294 const landed = await waitSpy(1);
295 const rows = await spy();
296 check('the worker\'s spend was recorded at all', landed && rows.length === 1,
297 JSON.stringify(rows));
298 // THE CONTROL. Without this the next check passes on a run where the worker
299 // finished before the click, which is a different test entirely.
300 check('the control: B was the Diamond on screen when the money was counted',
301 rows.length === 1 && rows[0].onScreen === B, JSON.stringify(rows[0] || null));
302 check('the worker was billed to A, the Diamond that dispatched it',
303 rows.length === 1 && rows[0].billed === A, JSON.stringify(rows[0] || null));
304
305 const ix = await p.evaluate(() => window.DaimondSignals.snapshot().diamonds);
306 check('and the index shows A spending and B untouched',
307 !!(ix[A] && ix[A].usd > 0) && !ix[B], JSON.stringify(ix));
308}
309
310// ── 2b. A worker of A's, landing while an ordinary CHAT is on screen ──
311console.log('\n3. a worker lands while an ordinary chat is on screen\n');
312{
313 await armSpy();
314 await goDiamond(A);
315 await p.waitForTimeout(200);
316 await p.evaluate(({ A, pick }) => window.DaimondWorkers.dispatch(
317 A, 'A', [{ name: 'away2', task: '@slow 5000' }], false, pick, 0), { A, pick });
318 await p.waitForTimeout(700);
319 await newChat(s);
320 await p.waitForTimeout(300);
321 const during = await p.evaluate(() => ({
322 sel: (window.DaimondDiamond.current() || {}).id || '',
323 running: window.DaimondWorkers.runs.filter(r => r.status === 'running' || r.status === 'queued').length,
324 }));
325 check('the control: the worker was still running when the chat came up',
326 during.running > 0 && during.sel === '', JSON.stringify(during));
327
328 await waitSpy(1);
329 const rows = await spy();
330 // `selectChat` nulls the global, so under the defect this landed on NOBODY.
331 check('the control: no Diamond was selected when the money was counted',
332 rows.length === 1 && rows[0].onScreen === '', JSON.stringify(rows[0] || null));
333 check('the worker was still billed to A, not to nobody',
334 rows.length === 1 && rows[0].billed === A, JSON.stringify(rows[0] || null));
335}
336
337// ── 2c. A daimon's own turn, metered while another Diamond is on screen ──
338console.log('\n4. a daimon turn is metered while another Diamond is on screen\n');
339{
340 await armSpy();
341 await goDiamond(A);
342 await p.waitForTimeout(400);
343 await p.evaluate(() => {
344 const el = document.getElementById('chat-input');
345 el.value = '@slow 5000';
346 el.dispatchEvent(new Event('input', { bubbles: true }));
347 document.getElementById('chat-send').click();
348 });
349 await p.waitForTimeout(900);
350 await goDiamond(B);
351 await p.waitForTimeout(300);
352 check('the control: B was selected before the steer landed',
353 (await p.evaluate(() => (window.DaimondDiamond.current() || {}).id || '')) === B);
354
355 const landed = await waitSpy(1);
356 const rows = await spy();
357 check('the daimon turn\'s spend was recorded at all', landed && rows.length >= 1,
358 JSON.stringify(rows));
359 check('the control: B was on screen when the daimon turn was counted',
360 rows.length >= 1 && rows[0].onScreen === B, JSON.stringify(rows[0] || null));
361 check('the daimon turn was billed to A, whose daimon ran it',
362 rows.length >= 1 && rows[0].billed === A, JSON.stringify(rows[0] || null));
363}
364
365// ── 2d. An ordinary chat's own turn, landing while a Diamond is on screen ──
366console.log('\n5. an ordinary chat\'s turn lands while a Diamond is on screen\n');
367{
368 await armSpy();
369 await newChat(s);
370 await p.waitForTimeout(300);
371 await p.evaluate(() => {
372 const el = document.getElementById('chat-input');
373 el.value = '@slow 5000';
374 el.dispatchEvent(new Event('input', { bubbles: true }));
375 document.getElementById('chat-send').click();
376 });
377 await p.waitForTimeout(900);
378 await goDiamond(B);
379 await p.waitForTimeout(300);
380 check('the control: B was selected before the chat turn landed',
381 (await p.evaluate(() => (window.DaimondDiamond.current() || {}).id || '')) === B);
382
383 const landed = await waitSpy(1);
384 const rows = await spy();
385 check('the chat turn\'s spend was recorded at all', landed && rows.length >= 1,
386 JSON.stringify(rows));
387 check('the control: B was on screen when the chat turn was counted',
388 rows.length >= 1 && rows[0].onScreen === B, JSON.stringify(rows[0] || null));
389 // An ordinary chat is not a Diamond and has no row in this index, so the honest
390 // answer is nobody. What it must NOT be is the Diamond the user wandered to.
391 check('the chat turn was billed to nobody, not to the Diamond on screen',
392 rows.length >= 1 && rows[0].billed === '', JSON.stringify(rows[0] || null));
393 const ix = await p.evaluate(() => window.DaimondSignals.snapshot().diamonds);
394 check('and B\'s row did not move for a conversation it never saw',
395 !ix[B], JSON.stringify(ix));
396}
397
398const errs = errors(s).filter(e => !/favicon|401|402|502|Unauthorized|Payment|Bad Gateway/i.test(e));
399check('no console errors', errs.length === 0, errs.slice(0, 3).join(' | '));
400
401await s.close();
402
403console.log(`\n${ok.length} ok, ${bad.length} failed`);
404if (bad.length) console.log(' ' + bad.join('\n '));
405if (BREAK) {
406 console.log(bad.length
407 ? '\nTHE BREAK WAS CAUGHT.'
408 : '\nTHE BREAK WAS NOT CAUGHT: this check proves nothing');
409 process.exit(bad.length ? 0 : 1);
410}
411process.exit(bad.length ? 1 : 0);