Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_workermodel.mjs

21.1 KiB, 1 run

created by r2519314175:821, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_workermodel.mjs — a Diamond's workers run on the model the USER chose.
2//
3// The defect this is built around: a worker was constructed on `cfg.model` — a
4// view of the STARRED DEFAULT — however the Diamond that dispatched it was
5// pinned. So a Diamond deliberately set to a strong model fanned its workers out
6// onto whatever happened to be starred, and the whole fan-out was billed to that
7// provider's key. Nothing on screen said so.
8//
9// Everything here is asserted on the OUTBOUND REQUEST, never on a field the test
10// has just set: the question is which model and whose key actually went on the
11// wire. Three models are in play at once, so the three answers are told apart —
12//
13// mock/fast the starred default (the defect's answer)
14// mock/thinker the Diamond's own model (a half-fix's answer)
15// mock2/worker the chosen worker model, on a SECOND provider with its own
16// key and its own endpoint (the right answer)
17//
18// The second provider is a mock served from this file, so "the provider travels
19// with the model" is proved by the request arriving at the other endpoint with
20// the other key, rather than by reading a variable back.
21//
22// node dev/verify_workermodel.mjs
23//
24// Needs dev/serve.mjs (DAIMOND_PORT, default 8777) and dev/mockllm.mjs
25// (DAIMOND_MOCK_PORT, default 9099). Starts its own second provider on :9300 and stops
26// it at the end.
27
28import fs from 'node:fs';
29import http from 'node:http';
30import path from 'node:path';
31import { fileURLToPath } from 'node:url';
32import { open, clearMockLog, mockLog, shot, errors, MOCK } from './harness.mjs';
33
34// Only so the preflight below can NAME the file it reads. Derived exactly as
35// `harness.mjs` derives it, since it does not export the path itself.
36const MOCK_LOG_PATH = process.env.DAIMOND_MOCK_LOG
37 || path.join(path.dirname(fileURLToPath(import.meta.url)), 'mockllm.log');
38
39// THE PARCEL CEILING, READ FROM daimond.js RATHER THAN RESTATED.
40//
41// This was `10 * 1024 * 1024` written out here, under a check that read "still far
42// inside its 10 MiB ceiling" -- a sentence that stayed green and stopped being true
43// the day `SYNC_PARCEL_MAX` moved. A restated constant is right only until somebody
44// changes the real one; `dev/verify_crystalcap.mjs` reads its two the same way.
45const PARCEL_MAX = (() => {
46 const src = fs.readFileSync(path.join(path.dirname(fileURLToPath(import.meta.url)),
47 '..', 'www', 'js', 'daimond.js'), 'utf8');
48 const m = /var\s+SYNC_PARCEL_MAX\s*=\s*(\d+)\s*\*\s*(\d+)\s*\*\s*(\d+)\s*;/.exec(src);
49 if (!m) throw new Error('verify_workermodel: SYNC_PARCEL_MAX not found in www/js/daimond.js');
50 return Number(m[1]) * Number(m[2]) * Number(m[3]);
51})();
52
53// Offset by the world, so two worlds do not fight over one second provider.
54//
55// IN ITS OWN BAND, not adjacent to the world's. The base was 9097, two below the
56// mock base of 9099, so world N's second provider was world N-2's MOCK: world 4
57// bound 9101 and took world 2's provider out from under whoever was driving it,
58// and the gate's world 9 sits on world 7's. `dev/world.sh` hands out 8777+N and
59// 9099+N; 9300+N meets neither for any world worth numbering.
60const PORT2 = Number(process.env.DAIMOND_MOCK2_PORT
61 || 9300 + (Number(process.env.DAIMOND_PORT || 8777) - 8777));
62const URL2 = `http://127.0.0.1:${PORT2}/v1/chat/completions`;
63const KEY2 = 'key-two-only-mock2-holds-this';
64const MODEL2 = 'mock2/worker';
65
66const ok = [], bad = [];
67const check = (name, pass, detail) => {
68 (pass ? ok : bad).push(name);
69 console.log((pass ? ' ok ' : ' FAIL ') + name + (detail != null ? ' — ' + detail : ''));
70};
71
72// ── The second provider ─────────────────────────────────────────────
73//
74// Every request it receives is kept whole: the model asked for, the key it was
75// sent with, and the transcript, so a worker can be recognised by its task.
76const seen = [];
77const cors = (res) => {
78 res.setHeader('Access-Control-Allow-Origin', '*');
79 res.setHeader('Access-Control-Allow-Headers', '*');
80 res.setHeader('Access-Control-Allow-Methods', 'GET,POST,OPTIONS');
81};
82const mock2 = http.createServer((req, res) => {
83 if (req.method === 'OPTIONS') { cors(res); res.writeHead(204); return res.end(); }
84 if (req.method === 'GET' && req.url.startsWith('/v1/models')) {
85 const body = JSON.stringify({ object: 'list', data: [{ id: MODEL2, object: 'model' }] });
86 cors(res);
87 res.writeHead(200, { 'content-type': 'application/json' });
88 return res.end(body);
89 }
90 let raw = '';
91 req.on('data', (c) => { raw += c; });
92 req.on('end', async () => {
93 let payload = {};
94 try { payload = JSON.parse(raw); } catch { payload = {}; }
95 seen.push({
96 url: req.url,
97 model: payload.model,
98 auth: req.headers.authorization || '',
99 messages: payload.messages || [],
100 });
101 cors(res);
102 res.writeHead(200, {
103 'content-type': 'text/event-stream',
104 'cache-control': 'no-cache',
105 'connection': 'keep-alive',
106 });
107 const frame = (delta, finish = null) => ({
108 id: 'chatcmpl-mock2', object: 'chat.completion.chunk', created: 1700000000,
109 model: payload.model || MODEL2,
110 choices: [{ index: 0, delta, finish_reason: finish }],
111 });
112 const send = (o) => res.write(`data: ${JSON.stringify(o)}\n\n`);
113 send(frame({ role: 'assistant', content: '' }));
114 for (const w of ['Worker', 'reporting', 'from', 'the', 'second', 'provider.']) {
115 send(frame({ content: w + ' ' }));
116 await new Promise((r) => setTimeout(r, 5));
117 }
118 send(frame({}, 'stop'));
119 send({ id: 'chatcmpl-mock2', object: 'chat.completion.chunk', model: payload.model || MODEL2,
120 choices: [], usage: { prompt_tokens: 11, completion_tokens: 7, total_tokens: 18 } });
121 res.write('data: [DONE]\n\n');
122 res.end();
123 });
124});
125await new Promise((r) => mock2.listen(PORT2, '127.0.0.1', r));
126console.log(`second provider on ${URL2}`);
127
128// ── The shared mock must write the log this file reads ──────────────
129//
130// Half of what follows is asserted on the SHARED mock's log, and an empty log
131// reads exactly like a model that was never called. So the two are proved to be
132// one pair before anything is measured: a probe request goes to the mock this
133// run drives, and the log this run reads has to grow by it.
134//
135// The 99c838e gate failed here and it was read as a product defect for a day.
136// `dev/world.sh N --up` reuses a world that is already listening, so world 9 was
137// still the mock an EARLIER gate had started in ITS worktree -- appending to
138// `.wt/gate-b536d60/dev/mockllm-9.log` while every verifier in the newer
139// worktree read its own, permanently empty, copy. `mockLog()` returned nothing,
140// and the two checks that ask what the daimon ran on said the daimon ran on
141// nothing. This refuses instead, and names the two paths, because a suite that
142// measures a log nobody writes cannot say anything about the app at all.
143const PROBE = 'workermodel-preflight-probe';
144let live = '';
145try {
146 const before = mockLog().length;
147 const r = await fetch(MOCK, {
148 method: 'POST',
149 headers: { 'content-type': 'application/json' },
150 body: JSON.stringify({ model: 'mock/fast', stream: false,
151 messages: [{ role: 'user', content: PROBE }] }),
152 });
153 if (!r.ok) live = `the mock answered ${r.status}`;
154 else {
155 await r.text();
156 if (mockLog().length <= before) live = 'the probe was not written to the log';
157 }
158} catch (e) {
159 live = 'the mock could not be reached: ' + e.message;
160}
161if (live) {
162 console.log(' REFUSED ' + live);
163 console.log(` driving: ${MOCK}`);
164 console.log(` reading: ${MOCK_LOG_PATH}`);
165 console.log(' These are not one world. Everything below would measure an empty');
166 console.log(' log and blame the app. `bash dev/world.sh N --down` then --up.');
167 mock2.close();
168 process.exit(2);
169}
170clearMockLog();
171
172// ── Helpers ─────────────────────────────────────────────────────────
173
174/// Wait for a condition, polling. Returns whether it came true.
175const until = async (fn, ms = 20000) => {
176 const t0 = Date.now();
177 while (Date.now() - t0 < ms) {
178 if (await fn()) return true;
179 await new Promise((r) => setTimeout(r, 200));
180 }
181 return false;
182};
183
184/// Whether a logged request is a WORKER's turn on `task`, rather than the
185/// daimon's own turn about it.
186///
187/// Mentioning the task is not enough to tell them apart: the daimon's transcript
188/// carries its `spawn_agent` call, arguments and all, so every round of the
189/// dispatching turn quotes the task too. A worker's turn is the one where the
190/// task IS the user message — that distinction is the whole point, since a check
191/// that cannot make it passes against the defect it is meant to catch.
192const isWorkerTurn = (e, task) => (e.messages || []).some(
193 (m) => m.role === 'user' && typeof m.content === 'string' && m.content.trim() === task);
194
195/// The WORKER turns the shared mock received for `task` — the ones that should
196/// never exist when the worker model belongs to another provider.
197const sharedFor = (task) => mockLog().filter((e) => isWorkerTurn(e, task));
198/// Same, for the second provider.
199const mock2For = (task) => seen.filter((e) => isWorkerTurn(e, task));
200
201/// Choose one option of a `<select>` by model AND provider, then fire `change`
202/// the way a user's click does. The same model id can sit under two providers,
203/// so `sel.value = m` is not enough — it takes whichever comes first.
204const chooseIn = (page, selector, nth, model, provider) => page.evaluate(
205 ({ selector, nth, model, provider }) => {
206 const sel = [...document.querySelectorAll(selector)][nth];
207 if (!sel) return 'no such select';
208 const opt = [...sel.querySelectorAll('option')]
209 .find((o) => o.value === model && o.dataset.provider === provider);
210 if (!opt) return 'no such option';
211 opt.selected = true;
212 sel.dispatchEvent(new Event('change', { bubbles: true }));
213 return '';
214 }, { selector, nth, model, provider });
215
216const s = await open({ name: 'wmodel' + Date.now() });
217const p = s.page;
218
219// ── The second provider, as the app sees it ─────────────────────────
220const added = await p.evaluate(async ({ url, key }) => {
221 DaimondModels.addProvider('mock2', { name: 'Mock Two', url: url });
222 await DaimondModels.setKey('mock2', key);
223 await DaimondModels.fetchModels('mock2');
224 const r = DaimondModels.resolve('mock2', 'mock2/worker');
225 const d = DaimondModels.getDefault();
226 return { resolved: !!r, url: r && r.baseUrl, defProv: d.provider, defModel: d.model };
227}, { url: URL2, key: KEY2 });
228// The first provider's id is minted from its URL, so it is read rather than guessed.
229const DEF = added.defProv;
230check('a second provider is configured, with its own key and endpoint',
231 added.resolved && added.url === URL2, added.url);
232check('the starred default is still the FIRST provider\'s model',
233 added.defModel === 'mock/fast' && DEF !== 'mock2', DEF + ' ' + added.defModel);
234
235// ── The New Diamond dialog ──────────────────────────────────────────
236await p.click('#new-diamond-btn');
237await p.waitForSelector('.dlg-select', { timeout: 8000 });
238const dlg = await p.evaluate(() => {
239 const sels = [...document.querySelectorAll('.dlg-select')];
240 const labs = [...document.querySelectorAll('.cfg-fieldlabel')].map((l) => l.textContent);
241 return {
242 count: sels.length,
243 // The worker pulldown is BELOW the model pulldown: same parent, later in it.
244 below: sels.length === 2
245 && sels[0].compareDocumentPosition(sels[1]) === Node.DOCUMENT_POSITION_FOLLOWING,
246 labelled: sels.length === 2 && !!sels[1].getAttribute('aria-label'),
247 labs,
248 };
249});
250check('the New Diamond dialog has a worker-model pulldown below the model one',
251 dlg.count === 2 && dlg.below, JSON.stringify(dlg.labs));
252check('the worker pulldown says what happens if it is left alone', dlg.labelled);
253
254await p.fill('.dlg-input', 'Worker Model Test');
255// Move the Diamond's OWN model, and the worker pulldown should follow it: that
256// is what "defaults to the main model" means while the dialog is still open.
257const chose = await chooseIn(p, '.dlg-select', 0, 'mock/thinker', DEF);
258check('the Diamond model can be chosen', chose === '', chose);
259const followed = await p.evaluate(() =>
260 [...document.querySelectorAll('.dlg-select')][1].value);
261check('the worker pulldown follows the Diamond\'s model until it is moved',
262 followed === 'mock/thinker', followed);
263// Now move it off, onto the second provider's model.
264const chose2 = await chooseIn(p, '.dlg-select', 1, MODEL2, 'mock2');
265check('the worker model can be chosen from another provider', chose2 === '', chose2);
266await shot(s, 'workermodel-1-dialog');
267await p.click('.dlg-ok');
268await p.waitForTimeout(1500);
269
270const stored = await p.evaluate(() => {
271 const all = JSON.parse(localStorage.getItem('daimond-diamond-models') || '{}');
272 const ids = Object.keys(all);
273 return { id: ids[ids.length - 1], rec: all[ids[ids.length - 1]] };
274});
275check('the Diamond records both pairs',
276 !!stored.rec && stored.rec.model === 'mock/thinker'
277 && stored.rec.workerModel === MODEL2 && stored.rec.workerProvider === 'mock2',
278 JSON.stringify(stored.rec));
279
280// ── The behavioural check: what actually goes on the wire ───────────
281const TASK1 = 'INSPECT-THE-LEDGER-ALPHA';
282clearMockLog();
283seen.length = 0;
284await p.fill('#chat-input',
285 `@tools spawn_agent {"name":"alpha","task":"${TASK1}"}`);
286await p.keyboard.press('Enter');
287const ran1 = await until(async () => mock2For(TASK1).length > 0, 30000);
288await p.waitForTimeout(1200);
289await shot(s, 'workermodel-2-dispatched');
290
291const w1 = mock2For(TASK1)[0];
292check('the worker turn reached the chosen model\'s provider', ran1 && !!w1,
293 ran1 ? '' : 'nothing arrived at the second provider');
294check('the worker ran on the model the user chose for workers',
295 !!w1 && w1.model === MODEL2, w1 && w1.model);
296check('the worker carried THAT provider\'s key, not the default provider\'s',
297 !!w1 && w1.auth === 'Bearer ' + KEY2, w1 && (w1.auth || '').slice(0, 16) + '…');
298// The defect, stated as the thing that must not happen: the worker's task never
299// went to the starred default's endpoint at all — on any model.
300check('the worker never went to the starred default\'s provider',
301 sharedFor(TASK1).length === 0,
302 sharedFor(TASK1).map((e) => e.model).join(','));
303// And the daimon itself still ran on the DIAMOND's model, so a fix that simply
304// moved everything onto the worker model would be caught too.
305const steer1 = mockLog().filter((e) => JSON.stringify(e.messages || []).includes('spawn_agent'));
306check('the daimon still ran on the Diamond\'s own model',
307 steer1.length > 0 && steer1.every((e) => e.model === 'mock/thinker'),
308 steer1.map((e) => e.model).join(','));
309
310// ── A Diamond that predates the setting ─────────────────────────────
311//
312// Its record carries a model and NOTHING about workers. Absent has to mean the
313// Diamond's own model; reading it as the starred default is the defect, kept.
314await p.click('#new-diamond-btn');
315await p.waitForSelector('.dlg-select', { timeout: 8000 });
316await p.fill('.dlg-input', 'Legacy Diamond');
317await chooseIn(p, '.dlg-select', 0, 'mock/thinker', DEF);
318await p.click('.dlg-ok');
319await p.waitForTimeout(1500);
320const legacy = await p.evaluate(() => {
321 const all = JSON.parse(localStorage.getItem('daimond-diamond-models') || '{}');
322 const ids = Object.keys(all);
323 const id = ids[ids.length - 1];
324 // Exactly what an older build wrote: the two fields it knew about.
325 all[id] = { provider: all[id].provider, model: all[id].model };
326 localStorage.setItem('daimond-diamond-models', JSON.stringify(all));
327 return { id, rec: all[id] };
328});
329check('a Diamond written by an older build carries no worker model',
330 !('workerModel' in legacy.rec), JSON.stringify(legacy.rec));
331
332const TASK2 = 'INSPECT-THE-LEDGER-BETA';
333clearMockLog();
334seen.length = 0;
335await p.fill('#chat-input', `@tools spawn_agent {"name":"beta","task":"${TASK2}"}`);
336await p.keyboard.press('Enter');
337const ran2 = await until(async () => sharedFor(TASK2).length > 0, 30000);
338await p.waitForTimeout(1200);
339const w2 = sharedFor(TASK2)[0];
340check('an existing Diamond\'s workers run on the Diamond\'s own model',
341 ran2 && !!w2 && w2.model === 'mock/thinker', w2 && w2.model);
342check('an existing Diamond\'s workers do NOT fall back to the starred default',
343 sharedFor(TASK2).every((e) => e.model !== 'mock/fast'),
344 sharedFor(TASK2).map((e) => e.model).join(','));
345
346// ── The chat tile ───────────────────────────────────────────────────
347await p.click('#new-session-btn', { force: true });
348await p.waitForTimeout(600);
349const tile = await p.evaluate(() => {
350 const box = document.querySelector('.session-box.pending');
351 const sels = box ? [...box.querySelectorAll('select.tile-model')] : [];
352 return {
353 count: sels.length,
354 worker: sels.length === 2 && sels[1].classList.contains('tile-worker-model'),
355 below: sels.length === 2
356 && sels[0].compareDocumentPosition(sels[1]) === Node.DOCUMENT_POSITION_FOLLOWING,
357 same: sels.length === 2 && sels[0].value === sels[1].value,
358 };
359});
360check('a pending chat carries a worker-model pulldown below its model',
361 tile.count === 2 && tile.worker && tile.below, JSON.stringify(tile));
362check('the chat\'s worker pulldown starts on the chat\'s own model', tile.same);
363
364await chooseIn(p, '.session-box.pending select.tile-model', 0, 'mock/thinker', DEF);
365const moved = await p.evaluate(() => {
366 const sels = [...document.querySelectorAll('.session-box.pending select.tile-model')];
367 return { main: sels[0].value, worker: sels[1].value };
368});
369check('the chat\'s worker pulldown follows its model until it is moved',
370 moved.worker === 'mock/thinker', JSON.stringify(moved));
371
372await chooseIn(p, '.session-box.pending select.tile-model', 1, MODEL2, 'mock2');
373await chooseIn(p, '.session-box.pending select.tile-model', 0, 'mock/fast', DEF);
374const pinned = await p.evaluate(() => {
375 const sels = [...document.querySelectorAll('.session-box.pending select.tile-model')];
376 return { main: sels[0].value, worker: sels[1].value };
377});
378check('a worker model chosen by hand stops following the chat\'s model',
379 pinned.main === 'mock/fast' && pinned.worker === MODEL2, JSON.stringify(pinned));
380
381await p.click('.session-box.pending .tile-start', { force: true });
382await p.waitForTimeout(1200);
383await shot(s, 'workermodel-3-tile');
384
385// ── Persisted, and carried in the parcel ────────────────────────────
386const persisted = await p.evaluate(async () => {
387 const st = await DaimondCore.chatStore().stored();
388 const c = (Array.isArray(st) ? st : []).find((x) => x && x.workerModel);
389 return c ? { model: c.model, worker: c.workerModel, prov: c.workerProvider } : null;
390});
391check('a started chat persists the worker model it was given',
392 !!persisted && persisted.worker === MODEL2 && persisted.prov === 'mock2',
393 JSON.stringify(persisted));
394
395const parcel = await p.evaluate(async () => {
396 const j = await DaimondCore.collectSync();
397 const chats = (j.chats || []).filter((c) => c.workerModel);
398 const dias = (j.diamonds || []).filter((d) => d.model && d.model.workerModel);
399 return {
400 bytes: JSON.stringify(j).length,
401 chatWorker: chats.length ? chats[0].workerModel : '',
402 diaWorker: dias.length ? dias[0].model.workerModel : '',
403 diaProv: dias.length ? dias[0].model.workerProvider : '',
404 };
405});
406check('the parcel\'s CHAT section carries the chat\'s worker model',
407 parcel.chatWorker === MODEL2, parcel.chatWorker);
408check('the parcel\'s DIAMOND section carries the Diamond\'s worker model',
409 parcel.diaWorker === MODEL2 && parcel.diaProv === 'mock2',
410 parcel.diaWorker + ' / ' + parcel.diaProv);
411check('the parcel is still far inside the ceiling daimond.js sets itself',
412 parcel.bytes < PARCEL_MAX, parcel.bytes + ' of ' + PARCEL_MAX + ' bytes');
413
414// ── A Diamond cut from a chat inherits both ─────────────────────────
415//
416// The fold picker refuses an empty chat, so the chat says something first.
417await p.fill('#chat-input', '@text something worth keeping');
418await p.click('#chat-send', { force: true });
419await p.waitForTimeout(3000);
420await p.evaluate(() => {
421 const b = document.querySelector('.session-box.active .tile-fold');
422 if (b) b.click();
423});
424await p.waitForTimeout(400);
425const opened = await p.evaluate(() => {
426 const item = document.querySelector('.fold-menu-item.new');
427 if (!item) return false;
428 item.click();
429 return true;
430});
431if (opened) {
432 await p.waitForSelector('.dlg-input', { timeout: 8000 });
433 await p.fill('.dlg-input', 'Cut From Chat');
434 await p.click('.dlg-ok');
435 await p.waitForTimeout(2500);
436}
437const cut = await p.evaluate(() => {
438 const all = JSON.parse(localStorage.getItem('daimond-diamond-models') || '{}');
439 const ids = Object.keys(all);
440 return all[ids[ids.length - 1]] || null;
441});
442check('a Diamond cut from a chat inherits that chat\'s worker model',
443 opened && !!cut && cut.workerModel === MODEL2 && cut.workerProvider === 'mock2',
444 JSON.stringify(cut));
445
446console.log('\nconsole errors:', errors(s).slice(0, 4).join(' | ') || '(none)');
447console.log(`\nworker model: ${ok.length} ok, ${bad.length} failed.`);
448if (bad.length) console.log('failed: ' + bad.join(' | '));
449await s.close();
450mock2.close();
451process.exit(bad.length ? 1 : 0);