Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_money.mjs

12.5 KiB, 1 run

created by r2519314175:541, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_money.mjs -- whose money the rail is talking about.
2//
3// The rail carried one row, labelled "Credits", showing the balance held with
4// Daimond. To somebody running on their own provider key it said
5// **"Credits $0.00"** while that key was paying for every turn. The one number
6// on screen about money told them they were broke, about money that was not
7// theirs.
8//
9// The four rules, and each is a check here:
10//
11// 1. Never a bare "Credits". Every label names an owner.
12// 2. The strongest true statement, and otherwise NOTHING. Exact, then an
13// estimate, then what has been spent -- and no row at all if none of those
14// is true. A dash sits where the answer goes and reads as zero.
15// 3. Warn on runway, not on a threshold. $2 is fine at a penny an hour and
16// gone in ten minutes during a fan-out; only time can tell them apart.
17// 4. At risk, show the CONSEQUENCE rather than the figure, so the reader is
18// not left doing the division.
19//
20// Pure: these are wording rules and they are tested without a browser.
21//
22// node dev/verify_money.mjs
23
24import fs from 'node:fs';
25import path from 'node:path';
26import { fileURLToPath } from 'node:url';
27
28const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
29let failures = 0;
30const check = (cond, msg, detail) => {
31 if (!cond) failures++;
32 console.log((cond ? ' ok ' : ' FAIL ') + msg + (detail ? ' -- ' + detail : ''));
33};
34
35const mod = { exports: {} };
36new Function('module', fs.readFileSync(path.join(ROOT, 'www/js/money.js'), 'utf8'))(mod);
37const M = mod.exports;
38
39const prov = (o) => Object.assign({ id: 'p', name: 'OpenAI', paid: false, hasKey: true,
40 credit: null, spentUsd: 0 }, o);
41
42// ══ THE BUG ITSELF ═════════════════════════════════════════════════
43// An account with no Daimond balance, running on its own funded key. The old
44// rail said "Credits $0.00" and nothing else.
45{
46 const rows = M.rows({
47 authed: true, creditsUsd: 0, creditsLabel: 'Daimond credits',
48 providers: [prov({ credit: { mode: 'auto', usd: 42.5 } })],
49 rateUsdPerMin: 0.01,
50 });
51 const labels = rows.map(r => r.label);
52 check(rows.some(r => /OpenAI/.test(r.label)),
53 'the user’s own funded key gets a row of its own', labels.join(' | '));
54 check(rows.every(r => r.label !== 'Credits'),
55 'and no row is labelled just "Credits" -- every label says whose money it is',
56 labels.join(' | '));
57 const own = rows.find(r => /OpenAI/.test(r.label));
58 check(own && own.usd === 42.5 && own.kind === 'exact',
59 'the funded key reports its real balance', own && `${own.kind} ${own.usd}`);
60 // The zero is still shown, because zero Daimond credits IS true and the row
61 // now says which zero it is.
62 const cr = rows.find(r => r.label === 'Daimond credits');
63 check(!!cr && cr.usd === 0,
64 'the Daimond balance is still reported, but now named', cr && String(cr.usd));
65}
66
67// ══ An EMPTY pot is not a pot running low ══════════════════════════
68// Found by rendering it: zero divided by any rate is zero minutes, so the rail
69// said "Daimond credits, ~1 min left at this rate" -- a prediction about money
70// that had already gone -- beside a key with $42.50 on it.
71{
72 const r = M.rowFor({ label: 'Daimond credits', exactUsd: 0 }, 0.01);
73 check(r.minutes === null && r.atRisk === false,
74 'an empty pot makes no runway claim -- there is no future to predict',
75 `${r.minutes} / ${r.atRisk}`);
76
77 // Empty AND the only money there is: that is worth a warning.
78 const alone = M.rows({ authed: true, creditsUsd: 0, creditsLabel: 'Daimond credits',
79 providers: [], rateUsdPerMin: 0.01 });
80 check(alone.length === 1 && alone[0].tone === 'warn' && alone[0].empty === true,
81 'but empty with nothing else funding the work is', JSON.stringify(alone[0]));
82
83 // Empty beside a funded key of the user's own: not a warning. Colouring it
84 // would put a caution on the rail of somebody whose work is fully paid for.
85 const beside = M.rows({ authed: true, creditsUsd: 0, creditsLabel: 'Daimond credits',
86 providers: [prov({ credit: { mode: 'auto', usd: 42.5 } })], rateUsdPerMin: 0.01 });
87 const cr2 = beside.find(x => x.label === 'Daimond credits');
88 check(cr2 && cr2.tone === 'ok' && !cr2.atRisk,
89 'and empty beside a funded key of your own is not',
90 cr2 && `${cr2.tone} / ${cr2.atRisk}`);
91}
92
93// ══ Rule 2: the strongest true statement ═══════════════════════════
94{
95 const exact = M.rowFor({ label: 'Your key', exactUsd: 10, estimateUsd: 99, spentUsd: 99 }, null);
96 check(exact.kind === 'exact' && exact.usd === 10,
97 'an exact balance wins over an estimate', exact.kind);
98
99 const est = M.rowFor({ label: 'Your key', exactUsd: null, estimateUsd: 7, spentUsd: 99 }, null);
100 check(est.kind === 'estimate' && est.usd === 7,
101 'an estimate wins over spend-to-date', est.kind);
102
103 const spent = M.rowFor({ label: 'Your key', exactUsd: null, estimateUsd: null, spentUsd: 3 }, null);
104 check(spent.kind === 'spent' && spent.usd === 3,
105 'and spend-to-date is what is left when no balance can be known', spent.kind);
106
107 const none = M.rowFor({ label: 'Your key', exactUsd: null, estimateUsd: null, spentUsd: 0 }, null);
108 check(none === null,
109 'with nothing true to say there is NO ROW -- a dash reads as zero to anyone scanning',
110 String(none));
111}
112
113// A provider whose balance nobody can read, and which has been used. Most
114// providers are this case, so it must not be the one that produces a blank.
115{
116 const rows = M.rows({
117 authed: false,
118 providers: [prov({ name: 'Anthropic', credit: null, spentUsd: 1.25 })],
119 rateUsdPerMin: null,
120 });
121 check(rows.length === 1 && rows[0].kind === 'spent',
122 'a key with no readable balance still says what it has cost',
123 JSON.stringify(rows.map(r => r.kind)));
124 check(!rows.some(r => r.label === 'Daimond credits'),
125 'and an app with no account grows no Daimond row -- that would be an advert where a fact goes');
126}
127
128// ══ Rule 3: runway, not a threshold ════════════════════════════════
129{
130 // The same $2, at two rates. An absolute threshold cannot tell these apart,
131 // which is the whole argument for using time.
132 const calm = M.rowFor({ label: 'Your key', exactUsd: 2 }, 0.001); // ~33 hours
133 const burst = M.rowFor({ label: 'Your key', exactUsd: 2 }, 0.5); // 4 minutes
134 check(calm.atRisk === false, 'two dollars at a slow burn is not a warning', String(calm.minutes));
135 check(burst.atRisk === true, 'the same two dollars during a fan-out is', String(burst.minutes));
136 check(calm.tone === 'ok' && burst.tone === 'warn',
137 'and the tone follows the runway rather than the figure',
138 calm.tone + ' / ' + burst.tone);
139
140 // A large balance burning fast is also at risk. A threshold on the figure
141 // would miss this one entirely.
142 const big = M.rowFor({ label: 'Your key', exactUsd: 100 }, 20);
143 check(big.atRisk === true,
144 'a hundred dollars at twenty a minute is at risk too -- the figure is no guide',
145 String(Math.round(big.minutes)));
146}
147
148// No rate, no runway: an idle app must not claim to know how long money lasts.
149{
150 const r = M.rowFor({ label: 'Your key', exactUsd: 2 }, null);
151 check(r.minutes === null && r.atRisk === false,
152 'with no burn rate there is no runway claim', String(r.minutes));
153 const zero = M.rowFor({ label: 'Your key', exactUsd: 2 }, 0);
154 check(zero.minutes === null,
155 'and a zero rate does not divide into a runway of years', String(zero.minutes));
156}
157
158// ══ Rule 4: the consequence, not the figure ════════════════════════
159// The module marks the row; the app swaps the text. What is asserted here is
160// that the mark is present exactly when the reader needs to be told.
161{
162 const r = M.rowFor({ label: 'Your key', exactUsd: 0.05 }, 0.05); // 1 minute
163 check(r.atRisk === true && Math.round(r.minutes) === 1,
164 'a minute of runway is flagged, with the minutes to say it',
165 `${r.minutes} min, atRisk ${r.atRisk}`);
166}
167
168// ══ Several keys ═══════════════════════════════════════════════════
169{
170 // Two keys, both readable: they may be summed.
171 const both = M.rows({ authed: false, rateUsdPerMin: null, providers: [
172 prov({ id: 'a', name: 'A', credit: { mode: 'auto', usd: 5 } }),
173 prov({ id: 'b', name: 'B', credit: { mode: 'auto', usd: 7 } }),
174 ] });
175 check(both.length === 1 && both[0].usd === 12,
176 'two readable keys are summed into one row', JSON.stringify(both[0]));
177 check(!/A|B/.test(both[0].label),
178 'named collectively, because four provider names is a list and not a status',
179 both[0].label);
180
181 // One readable, one not. Summing would produce a WRONG number.
182 const mixed = M.rows({ authed: false, rateUsdPerMin: null, providers: [
183 prov({ id: 'a', name: 'A', credit: { mode: 'auto', usd: 5 }, spentUsd: 1 }),
184 prov({ id: 'b', name: 'B', credit: null, spentUsd: 2 }),
185 ] });
186 check(mixed.length === 1 && mixed[0].kind === 'spent',
187 'a total that could only count half the keys is not shown as a balance',
188 JSON.stringify(mixed[0]));
189 check(mixed[0].usd === 3,
190 'it falls back to what all of them together have cost, which IS true',
191 String(mixed[0].usd));
192
193 // An estimate anywhere in the sum makes the sum an estimate.
194 const est = M.rows({ authed: false, rateUsdPerMin: null, providers: [
195 prov({ id: 'a', name: 'A', credit: { mode: 'auto', usd: 5 } }),
196 prov({ id: 'b', name: 'B', credit: { mode: 'manual', usd: 7 } }),
197 ] });
198 check(est[0].kind === 'estimate',
199 'and one estimated part makes the whole an estimate', est[0].kind);
200}
201
202// ══ A key with no key ══════════════════════════════════════════════
203{
204 const rows = M.rows({ authed: false, rateUsdPerMin: null,
205 providers: [prov({ hasKey: false, credit: { mode: 'auto', usd: 5 } })] });
206 check(rows.length === 0,
207 'a provider the user holds no key for is not their money and gets no row',
208 JSON.stringify(rows));
209}
210
211// ══ THE REAL RAIL ══════════════════════════════════════════════════
212// The pure half above was proved with a mock box injected into the page, which
213// tested the wording rules and NOTHING about whether the rail draws them. It
214// did not, and the case it missed is the commonest one there is: a user on
215// their own key with no Daimond account. `moneyRows` was called only for an
216// authed account, so that user saw no money row at all -- which is the whole
217// complaint this file exists to answer. Never again from a mock.
218if (!process.argv.includes('--pure')) {
219 const { open, connectMock, signInAs, scratch } = await import('./harness.mjs');
220 const s = await open({ name: 'moneyrail', signIn: false, connect: false,
221 profile: scratch('pw', 'moneyrail-' + process.pid) });
222 const { page: p } = s;
223 try {
224 await signInAs(s, 'moneyrail');
225 await connectMock(s);
226 // Spend on the user's OWN key, and no gateway account at all.
227 await p.evaluate(() => {
228 const id = (DaimondModels.providers()[0] || {}).id || 'custom';
229 for (let i = 0; i < 5; i++) {
230 DaimondLedger.record({ ts: Date.now() - i * 60000, model: 'mock/fast',
231 promptTokens: 1000, completionTokens: 400, cachedTokens: 0,
232 costUsd: 0.11, provider: id });
233 }
234 DaimondAdmin.status();
235 });
236 await p.waitForTimeout(700);
237
238 const seen = await p.evaluate(() => {
239 const rows = [...document.querySelectorAll('.astat-row')]
240 .filter(r => getComputedStyle(r).display !== 'none')
241 .map(r => r.textContent.trim());
242 const gw = window.DaimondGateway && DaimondGateway.state();
243 return { rows, authed: gw ? gw.authed : null };
244 });
245 check(seen.authed === false,
246 'this is the no-account case, which is the one that was broken',
247 String(seen.authed));
248 const money = seen.rows.filter(x => /spent|\$/.test(x));
249 check(money.length > 0,
250 'a user on their own key with NO Daimond account still sees a money row',
251 seen.rows.join(' | '));
252 check(money.some(x => /key/i.test(x)),
253 'and it names the key as theirs rather than saying "Credits"',
254 money.join(' | '));
255 check(!seen.rows.some(x => /^Credits/.test(x)),
256 'nothing on the rail is labelled just "Credits"', seen.rows.join(' | '));
257 } catch (e) {
258 failures++;
259 console.log(' FAIL rail half threw -- ' + (e && e.message ? e.message.split('\n')[0] : e));
260 } finally {
261 await s.close();
262 }
263}
264
265console.log('');
266console.log(failures ? `verify_money: ${failures} FAILED` : 'verify_money: all checks pass.');
267process.exit(failures ? 1 : 0);