Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_costtruth.mjs

10.5 KiB, 1 run

created by r2519314175:319, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_costtruth.mjs — what a turn cost, said by the only party that knows.
2//
3// Daimond priced every turn from a table it carries: tokens times a surveyed rate. That is the
4// right answer when nobody better has spoken, and the WRONG answer the moment a provider states
5// its own figure — which routers do, on every reply, in `usage.cost`, alongthe count of prompt
6// tokens they served from cache rather than charged for. A 10k-token prompt that was 90% cache
7// hits costs a fraction of what a table says it costs, and the app was billing the table.
8//
9// Three claims, and the first is the one the whole feature rests on:
10//
11// 1. A REPORTED cost is recorded verbatim. Not approximated, not re-derived, not blended with
12// the table -- the ledger entry carries the provider's own number and is marked as reported,
13// so a total containing it is not dressed up with an "≈".
14// 2. The account balance in the rail is fresh. Almost every credit-spending gateway reply
15// states the resulting balance; the app used to throw those away, so the header sat at
16// whatever the last explicit /api/balance call had said. Spending money must move the
17// number, with no reload and no panel opened.
18// 3. Bytes written by the PAGE (a compiled PDF, a saved message, an upload) go through the
19// wasm write, which is what applies the per-account namespace. The page used to walk the
20// origin OPFS root itself, so a secondary account's files landed in the primary account's
21// workspace -- one account's PDFs readable by another person at the same browser.
22//
23// The gateway is fetch-stubbed (as verify_credits does) rather than run: check 2 needs a reply
24// that STATES a lower balance, which is a two-line stub and a provisioned account plus a real
25// spend otherwise.
26import { open, signInAs, connectMock, chat, shot } from './harness.mjs';
27
28const ok = [], bad = [];
29const check = (name, pass, detail) => {
30 (pass ? ok : bad).push(name + (detail ? ' — ' + detail : ''));
31 console.log((pass ? ' ok ' : ' FAIL ') + name + (detail ? ' — ' + detail : ''));
32};
33
34// The figures the mock will report. Chosen so no table could arrive at them by accident:
35// COST is not tokens-times-any-plausible-rate, and CACHED is 90% of the prompt, which is the
36// case a table cannot express at all.
37const IN = 10240;
38const OUT = 128;
39const COST = 0.0021;
40const CACHED = 9216;
41
42// The stubbed gateway's balance, in minor units, before and after the spend.
43const BAL_BEFORE = 840; // $8.40
44const BAL_AFTER = 500; // $5.00
45
46const CORS = { 'access-control-allow-origin': '*', 'access-control-allow-headers': '*' };
47const json = (body, status = 200) => ({
48 status, contentType: 'application/json', headers: CORS, body: JSON.stringify(body),
49});
50
51const ACCOUNT_NS = 'd~costtruth'; // a stand-in for a secondary account's namespace
52
53let bal = BAL_BEFORE;
54
55/// The gateway's four boot calls, the balance, and the one endpoint that spends.
56///
57/// Installed BEFORE the identity is unlocked, so the app's own `afterUnlock()` runs bootstrap
58/// against a gateway that answers -- which is what paints the account row in the rail in the
59/// first place. Stubbing after sign-in would leave the row reading "unreachable" from boot, and
60/// a check against it would be measuring the harness rather than the app.
61async function stubGateway(page) {
62 await page.route('**/api/account', r => r.fulfill(json({ ok: true })));
63 await page.route('**/api/auth/challenge', r => r.fulfill(json({ ok: true, challenge: 'chal-ct', challenge_id: 'cid-ct' })));
64 await page.route('**/api/auth/verify', r => r.fulfill(json({ ok: true })));
65 await page.route('**/api/balance', r => r.fulfill(json({ ok: true, credits_minor: bal, currency: 'usd', entries: [] })));
66 // Not Pro: nothing here tests a Pro surface, and a Pro account starts sync pushing at
67 // endpoints this stub does not answer.
68 await page.route('**/api/licence', r => r.fulfill(json({ ok: true, pro: false })));
69 // The credit-spending action. Reading a page costs credits, and the reply states what is
70 // left -- which is the shape nearly every spending endpoint has.
71 await page.route('**/api/web/fetch', r => {
72 bal = BAL_AFTER;
73 return r.fulfill(json({ ok: true, credits_minor: bal, currency: 'usd',
74 url: 'https://example.test/', title: 'Example', text: 'A page the agent read.' }));
75 });
76}
77
78const s = await open({ name: 'costtruth', signIn: false });
79const p = s.page;
80await stubGateway(p);
81await signInAs(s, 'costtruth');
82await connectMock(s);
83await p.waitForTimeout(800);
84
85// ── 1. A reported cost is what is recorded ──────────────────────────────
86
87await p.evaluate(() => { try { localStorage.removeItem('daimond-ledger'); } catch (e) {} });
88await chat(s, `@usage ${IN} ${OUT} ${COST} ${CACHED}`);
89await p.waitForTimeout(600);
90
91const led = await p.evaluate(() => {
92 var raw = [];
93 try { raw = JSON.parse(localStorage.getItem('daimond-ledger') || '[]'); } catch (e) {}
94 return { n: raw.length, last: raw[raw.length - 1] || null };
95});
96const e = led.last || {};
97check('the turn reaches the ledger at all', led.n >= 1, led.n + ' entries');
98check("the ledger holds the PROVIDER's figure, exactly",
99 e.u === COST, `u = ${e.u} (wanted ${COST})`);
100check('and marks it as reported, so no total dresses it as an estimate',
101 e.r === 1 && !e.e, `r = ${e.r}, e = ${e.e}`);
102check('the cached prompt tokens are carried, not dropped',
103 e.ca === CACHED, `ca = ${e.ca} (wanted ${CACHED})`);
104check('and the provider is named, so a per-key breakdown can find it',
105 typeof e.pv === 'string' && e.pv.length > 0, `pv = ${JSON.stringify(e.pv)}`);
106// The token counts must still be the turn's DELTA, not the session cumulative: the reported
107// cost rides the same accounting, and a bug that double-counted one would double-count both.
108check('the token counts are the turn, not the session total',
109 e.p === IN && e.c === OUT, `p = ${e.p}, c = ${e.c}`);
110
111// A second turn on the same chat: the deltas must not compound. This is where a cumulative
112// read masquerading as a delta shows itself -- the second entry would carry 2×IN.
113await chat(s, `@usage ${IN} ${OUT} ${COST} ${CACHED}`);
114await p.waitForTimeout(600);
115const led2 = await p.evaluate(() => {
116 var raw = [];
117 try { raw = JSON.parse(localStorage.getItem('daimond-ledger') || '[]'); } catch (e) {}
118 return raw[raw.length - 1] || null;
119});
120check("a second turn's cost is its own, not the session's running total",
121 led2 && led2.u === COST && led2.p === IN && led2.ca === CACHED,
122 led2 ? `u = ${led2.u}, p = ${led2.p}, ca = ${led2.ca}` : '(no entry)');
123
124// ── 2. Spending money moves the number in the rail ──────────────────────
125//
126// No reload, no panel opened: the row has to repaint because the gateway reply said what was
127// left and something was listening for it.
128//
129// The spend is `DaimondWeb.fetch`, which is the exact function the agent's `web_fetch` tool
130// reaches through the wasm -- and deliberately NOT a whole agent turn, because a turn ends by
131// repainting most of the app anyway. Driving it through a turn would go green on the end-of-turn
132// repaint and prove nothing about whether a spend is heard.
133
134const railBefore = await p.evaluate(() =>
135 (document.getElementById('astat-account') || {}).textContent || '');
136check('the rail shows the balance before the spend',
137 /8\.40/.test(railBefore), railBefore.trim());
138
139const spent = await p.evaluate(async () => {
140 try { await window.DaimondWeb.fetch('https://example.test/'); return ''; }
141 catch (e) { return String(e && e.message ? e.message : e); }
142});
143await p.waitForTimeout(700);
144const railAfter = await p.evaluate(() =>
145 (document.getElementById('astat-account') || {}).textContent || '');
146check('the spend went through', spent === '', spent);
147check('spending credits refreshes the balance in the rail, with no reload and no panel opened',
148 railAfter !== railBefore && /5\.00/.test(railAfter),
149 `before "${railBefore.trim()}" → after "${railAfter.trim()}"`);
150
151// ── 3. The page's own writes go through the wasm, and the namespace ─────
152//
153// A secondary account resolves the workspace inside its own OPFS subdirectory. The wasm write
154// applies that; a hand-rolled `navigator.storage.getDirectory()` walk in the page cannot. So
155// the namespace is set and a file is written the way a USER writes one -- the Workspace panel's
156// upload, which shares `writeWorkspaceBytes` with the Typst compile and with saved mail -- and
157// the question is which root the bytes landed in.
158//
159// A real second account is not created: what is under test is which root the write resolves
160// against, and setting the namespace is the whole of what a second account does to it.
161
162await p.evaluate(() => {
163 // The panel must be open for its upload control to exist.
164 try { if (window.DaimondPanels) DaimondPanels.show('work'); } catch (e) {}
165});
166await p.waitForTimeout(600);
167await p.evaluate(async (ns) => {
168 const mod = await import('../pkg/oxedyne_daimond.js');
169 mod.set_account_ns(ns);
170}, ACCOUNT_NS);
171
172const NAME = 'ns-probe.txt';
173const chooser = p.waitForEvent('filechooser', { timeout: 8000 });
174await p.click('[data-act="upload"]', { force: true });
175await (await chooser).setFiles({
176 name: NAME, mimeType: 'text/plain', buffer: Buffer.from('written under a namespace'),
177});
178await p.waitForTimeout(1200);
179
180const landed = await p.evaluate(async ({ ns, name }) => {
181 const root = await navigator.storage.getDirectory();
182 const at = async (d, n) => { try { await d.getFileHandle(n); return true; } catch (e) { return false; } };
183 let sub = null;
184 try { sub = await root.getDirectoryHandle(ns); } catch (e) { sub = null; }
185 return {
186 inNamespace: sub ? await at(sub, name) : false,
187 atRoot: await at(root, name),
188 nsExists: !!sub,
189 };
190}, { ns: ACCOUNT_NS, name: NAME });
191
192check("a page write lands in the account's own namespace",
193 landed.inNamespace === true, `in ${ACCOUNT_NS}/: ${landed.inNamespace}, namespace present: ${landed.nsExists}`);
194check("and NOT in the primary account's workspace",
195 landed.atRoot === false, `at the OPFS root: ${landed.atRoot}`);
196
197// Put the namespace back, so nothing after this point is reading a stranger's root.
198await p.evaluate(async () => {
199 const mod = await import('../pkg/oxedyne_daimond.js');
200 mod.set_account_ns('');
201});
202
203await shot(s, 'costtruth');
204const errs = s.errs.filter(x => !/favicon|404|401|net::ERR/.test(x));
205console.log('\nconsole errors:', errs.slice(0, 4));
206await s.close();
207
208console.log(`\n${ok.length} passed, ${bad.length} failed`);
209if (bad.length) console.log('FAILED:\n ' + bad.join('\n '));
210process.exit(bad.length ? 1 : 0);