Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/www/js/draftmodel.test.mjs

9.4 KiB, 1 run

created by r2519314175:1361, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1/* ============================================================
2 Test — the dedicated drafting model (Track D).
3 ------------------------------------------------------------
4 The note→proposal drafting in triage.js runs on the CHAT model
5 until the user picks a model of its own in AI settings. This
6 drives the REAL www/js/models.js and www/js/triage.js in a
7 minimal browser sandbox and asserts the three things the design
8 turns on:
9
10 (a) UNSET → CHAT. With no drafting model chosen, `resolveDraft`
11 answers exactly what `resolve('','')` does — same provider,
12 same model, same key — so behaviour is the pre-Track-D one
13 and there is zero regression.
14
15 (b) SET → CHOSEN. Once a drafting model is set, `resolveDraft`
16 answers THAT model, and clearing it (empty model) falls
17 back to the chat model again.
18
19 (c) THE COST LINE FOLLOWS. `DaimondTriage.estimate()` prices the
20 model `resolveDraft` returns, so the ceiling a person reads
21 before the drafting press is the drafting model's, not the
22 chat model's, whenever one is set.
23
24 Each check is proven able to fail: the two --break modes damage
25 the shipped `resolveDraft` the two ways it could plausibly be got
26 wrong, and the checks that guard each defect go red under it.
27
28 node www/js/draftmodel.test.mjs --break nofallback # unset stops being chat
29 node www/js/draftmodel.test.mjs --break ignoredraft # the choice is ignored
30 node www/js/draftmodel.test.mjs # and then, clean
31 ============================================================ */
32import { readFileSync } from 'node:fs';
33import { fileURLToPath } from 'node:url';
34import { dirname, join } from 'node:path';
35
36const HERE = dirname(fileURLToPath(import.meta.url));
37
38let failures = 0, checks = 0;
39function check(name, cond, detail) {
40 checks++;
41 if (cond) console.log(' ok ' + name + (detail ? ' — ' + detail : ''));
42 else { console.log(' FAIL ' + name + (detail ? ' — ' + detail : '')); failures++; }
43}
44
45const BREAK = (() => {
46 const i = process.argv.indexOf('--break');
47 return i > 0 ? String(process.argv[i + 1] || '') : '';
48})();
49
50// ── The patches that prove the checks can fail ─────────────────
51// Each damages the ONE shipped function this feature lives in, the
52// two ways it could plausibly be got wrong.
53//
54// nofallback the unset case stops answering the chat model — the
55// regression the whole "defaults to chat" rule exists
56// to prevent. (a) goes red.
57// ignoredraft the user's chosen model is never consulted — drafting
58// silently stays on chat. (b) and (c) go red.
59function patchModels(src) {
60 if (BREAK === 'nofallback') {
61 return src.replace('if (dr.model) return resolve(dr.provider, dr.model);\n\t\treturn resolve(\'\', \'\');',
62 'if (dr.model) return resolve(dr.provider, dr.model);\n\t\treturn null;');
63 }
64 if (BREAK === 'ignoredraft') {
65 return src.replace('if (dr.model) return resolve(dr.provider, dr.model);',
66 'if (false) return resolve(dr.provider, dr.model);');
67 }
68 return src;
69}
70
71// ── A minimal browser sandbox ──────────────────────────────────
72// models.js and triage.js are classic-script IIFEs reading the bare
73// globals `window`, `document`, `localStorage`, and (guarded) the
74// timers. They attach `DaimondModels` / `DaimondTriage` onto window.
75const store = new Map();
76const localStorage = {
77 getItem: (k) => (store.has(k) ? store.get(k) : null),
78 setItem: (k, v) => store.set(k, String(v)),
79 removeItem: (k) => store.delete(k),
80};
81// `window` IS globalThis here, deliberately: in a browser `window.DaimondModels =
82// x` also creates the bare global `DaimondModels`, and triage.js's `pick()` reads
83// that bare global (`return DaimondModels.resolveDraft()`) — so a separate `win`
84// object would leave that reference undefined and every `pick()` would swallow a
85// ReferenceError and answer null. Making window the real global is what makes the
86// cross-module references resolve the way they do on the page.
87const win = globalThis;
88win.dispatchEvent = () => true;
89// triage.js registers a top-level click listener; models.js a guarded
90// visibilitychange one. Neither is fired here — the stub only has to
91// exist so the modules load.
92const documentShim = {
93 addEventListener: () => {},
94 getElementById: () => null,
95};
96function loadScript(rel, transform) {
97 let body = readFileSync(join(HERE, rel), 'utf8');
98 if (transform) body = transform(body);
99 const fn = new Function('window', 'document', 'localStorage', 'console', 'globalThis', body);
100 fn(win, documentShim, localStorage, console, globalThis);
101}
102
103loadScript('models.js', patchModels);
104loadScript('triage.js');
105
106const M = win.DaimondModels;
107const T = win.DaimondTriage;
108if (!M || !M.resolveDraft) { console.error('ABORT: DaimondModels.resolveDraft missing'); process.exit(2); }
109if (!T || !T.estimate) { console.error('ABORT: DaimondTriage.estimate missing'); process.exit(2); }
110
111// A pricing stub keyed by model, so a run's ceiling differs by which model priced
112// it — the only way to show the cost line followed the drafting model rather than
113// merely naming it. Non-null `rate` makes `estimate` go on to `priceFor`.
114const PRICE = { 'chat-model': 0.11, 'draft-model': 0.42 };
115win.DaimondPricing = {
116 rate: () => ({}),
117 priceFor: (model) => ({ usd: PRICE[model] != null ? PRICE[model] : 0 }),
118};
119
120const NOTES = [{ id: 'n1', text: 'the box loses my text', at: 1 }];
121
122/// The drafting model resolveDraft answers, or '' when it answers nothing — so a
123/// broken build that returns null reads as a clean FAIL rather than crashing the run.
124const draftModel = () => { const r = M.resolveDraft(); return r ? r.model : ''; };
125
126async function main() {
127 M.init({});
128 M.addProvider('fireworks', { url: 'https://api.fireworks.ai/x', name: 'Fireworks' });
129 await M.setKey('fireworks', 'chat-key'); // plaintext-at-rest: no identity here
130 M.setDefault('fireworks', 'chat-model');
131
132 // ── (a) UNSET → CHAT ───────────────────────────────────────
133 console.log('(a) unset drafting model resolves to the chat model');
134 const chat = M.resolve('', '');
135 const dUn = M.resolveDraft();
136 check('resolveDraft answers a config when chat is set', !!dUn);
137 check('unset drafting model === chat model',
138 !!dUn && dUn.model === 'chat-model' && dUn.provider === 'fireworks',
139 dUn ? dUn.model : String(dUn));
140 check('unset drafting config equals resolve("","")',
141 !!dUn && !!chat && dUn.model === chat.model && dUn.provider === chat.provider
142 && dUn.apiKey === chat.apiKey && dUn.baseUrl === chat.baseUrl);
143 check('getDraft reports nothing set', M.getDraft().model === '' && M.getDraft().provider === '');
144 const eUn = T.estimate(NOTES, []);
145 check('estimate prices the chat model when unset',
146 eUn.model === 'chat-model' && eUn.known === true && eUn.usd === PRICE['chat-model'],
147 'usd=' + eUn.usd + ' model=' + eUn.model);
148
149 // ── (b) SET → CHOSEN ───────────────────────────────────────
150 console.log('(b) a chosen drafting model wins, and diverges from chat');
151 M.setDraft('fireworks', 'draft-model');
152 const dSet = M.resolveDraft();
153 check('set drafting model resolves to the chosen model',
154 !!dSet && dSet.model === 'draft-model', dSet ? dSet.model : String(dSet));
155 check('the chat model is UNCHANGED by the drafting choice',
156 M.resolve('', '').model === 'chat-model');
157 check('getDraft reports the chosen model',
158 M.getDraft().model === 'draft-model' && M.getDraft().provider === 'fireworks');
159
160 // ── (c) THE COST LINE FOLLOWS ──────────────────────────────
161 console.log('(c) the cost estimate prices the drafting model, not the chat model');
162 const eSet = T.estimate(NOTES, []);
163 check('estimate names the drafting model', eSet.model === 'draft-model', eSet.model);
164 check('estimate prices the DRAFTING model, not the chat model',
165 eSet.usd === PRICE['draft-model'] && eSet.usd !== PRICE['chat-model'],
166 'usd=' + eSet.usd);
167
168 // ── clearing falls back to chat ────────────────────────────
169 console.log('(b′) clearing the drafting model falls back to chat');
170 M.setDraft('', '');
171 check('cleared drafting model resolves to chat again', draftModel() === 'chat-model');
172 check('estimate prices the chat model again after clearing',
173 T.estimate(NOTES, []).usd === PRICE['chat-model']);
174
175 // ── a drafting model whose provider is removed falls back ──
176 console.log('(d) removing a drafting model’s provider falls back to chat, never to nothing');
177 M.addProvider('together', { url: 'https://api.together.xyz/x', name: 'Together' });
178 await M.setKey('together', 'together-key');
179 M.setDraft('together', 'together-model');
180 check('drafting model set on a second provider', draftModel() === 'together-model');
181 M.removeProvider('together');
182 check('after its provider is removed, drafting falls back to chat',
183 draftModel() === 'chat-model');
184 check('and getDraft reports nothing set again', M.getDraft().model === '');
185
186 console.log('\n' + (failures ? 'FAIL' : 'PASS') + ' — ' + (checks - failures) + '/' + checks
187 + ' checks' + (BREAK ? ' (--break ' + BREAK + ')' : ''));
188 process.exit(failures ? 1 : 0);
189}
190
191main().catch((e) => { console.error('ABORT:', e && e.stack ? e.stack : e); process.exit(2); });