Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_pickers.mjs

10.1 KiB, 1 run

created by r2519314175:591, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_pickers.mjs — a chat and a Diamond each run on the model they were started with.
2//
3// With one provider, "which model" and "whose key" were the same question. With a key per
4// provider they are two, and a picker that answers only the first is worse than none: it lets a
5// user choose a model on provider B and then sends it, with provider A's key, to provider A.
6//
7// So the picker is grouped by provider and carries the provider on the option, and the proof is
8// not that the right words are on screen. It is that the request lands at the right BASE URL with
9// the right KEY — which is checked here by pointing two "providers" at two different mock servers
10// and seeing which one the traffic goes to.
11import { open, shot, errors, MOCK, PASS, storedChats } from './harness.mjs';
12import http from 'node:http';
13
14const ok = [], bad = [];
15const check = (name, pass, detail) => {
16 (pass ? ok : bad).push(name + (detail ? ' — ' + detail : ''));
17 console.log((pass ? ' ok ' : ' FAIL ') + name + (detail ? ' — ' + detail : ''));
18};
19
20// ── Two stand-in providers, each recording what it is asked ────────────
21//
22// They speak just enough OpenAI to answer /models and /chat/completions.
23function provider(port, models, tag) {
24 const seen = [];
25 const srv = http.createServer((req, res) => {
26 let body = '';
27 req.on('data', c => (body += c));
28 req.on('end', () => {
29 const auth = req.headers.authorization || '';
30 if (req.url.endsWith('/models')) {
31 res.writeHead(200, { 'content-type': 'application/json', 'access-control-allow-origin': '*' });
32 return res.end(JSON.stringify({ data: models.map(id => ({ id })) }));
33 }
34 seen.push({ url: req.url, auth, model: (() => { try { return JSON.parse(body).model; } catch { return ''; } })() });
35 res.writeHead(200, {
36 'content-type': 'application/json',
37 'access-control-allow-origin': '*',
38 'access-control-allow-headers': '*',
39 });
40 res.end(JSON.stringify({
41 choices: [{ message: { role: 'assistant', content: `answered by ${tag}` }, finish_reason: 'stop' }],
42 usage: { prompt_tokens: 5, completion_tokens: 5 },
43 }));
44 });
45 });
46 // CORS preflight, or the browser never sends the real request.
47 srv.on('request', (req, res) => {});
48 const orig = srv.listeners('request')[0];
49 srv.removeAllListeners('request');
50 srv.on('request', (req, res) => {
51 if (req.method === 'OPTIONS') {
52 res.writeHead(204, {
53 'access-control-allow-origin': '*',
54 'access-control-allow-headers': '*',
55 'access-control-allow-methods': 'POST, GET, OPTIONS',
56 });
57 return res.end();
58 }
59 orig(req, res);
60 });
61 srv.listen(port, '127.0.0.1');
62 return { seen, close: () => srv.close() };
63}
64
65// THE PORTS FOLLOW THE WORLD, and did not until 2026-08-14.
66//
67// They were 9101 and 9102, written as literals. A world's mock provider listens on
68// `9099 + N` (dev/world.sh), so 9101 IS world 2's mock and 9102 IS world 3's — and
69// world 3 is a lane another agent is told to use. On 2026-08-13 this file died with
70// `EADDRINUSE 127.0.0.1:9102` inside the release gate while a six-hour-old world 3
71// held the port, and the bare Node stack read as a product failure. Two ports are
72// wanted, so the world claims a PAIR, in a band no world's own numbering reaches.
73const WORLD = Number(process.env.DAIMOND_PORT || 8777) - 8777;
74const PORT_A = Number(process.env.DAIMOND_PICKER_PORT || 9160 + WORLD * 2);
75const PORT_B = PORT_A + 1;
76
77const A = provider(PORT_A, ['alpha-large', 'shared-model'], 'ALPHA');
78const B = provider(PORT_B, ['beta-small', 'shared-model'], 'BETA');
79
80const URL_A = `http://127.0.0.1:${PORT_A}/v1/chat/completions`;
81const URL_B = `http://127.0.0.1:${PORT_B}/v1/chat/completions`;
82
83const s = await open({ name: 'pickers', connect: false });
84const p = s.page;
85await p.waitForTimeout(1200);
86
87// Seed two providers with keys, and star ALPHA's model as the default.
88await p.evaluate(async ({ ua, ub }) => {
89 const M = window.DaimondModels;
90 M.addProvider('provA', { name: 'Provider A', url: ua });
91 M.addProvider('provB', { name: 'Provider B', url: ub });
92 await M.setKey('provA', 'key-for-A');
93 await M.setKey('provB', 'key-for-B');
94 await M.fetchModels('provA');
95 await M.fetchModels('provB');
96 M.setDefault('provA', 'alpha-large');
97}, { ua: URL_A, ub: URL_B });
98await p.waitForTimeout(600);
99
100// ── The picker lists every provider, grouped ───────────────────────────
101
102const picker = await p.evaluate(() => {
103 document.getElementById('new-session-btn').click();
104 return new Promise(r => setTimeout(() => {
105 const sel = document.querySelector('.tile-model');
106 const groups = [...sel.querySelectorAll('optgroup')].map(g => g.label);
107 const opts = [...sel.querySelectorAll('option')].map(o => ({
108 model: o.value, provider: o.dataset.provider,
109 }));
110 r({ groups, opts, selected: sel.value, selProv: sel.selectedOptions[0].dataset.provider });
111 }, 700));
112});
113check('the picker groups the models under the provider that runs them',
114 picker.groups.length === 2 && picker.groups.includes('Provider A') && picker.groups.includes('Provider B'),
115 picker.groups.join(' | '));
116check('every model carries the provider whose key runs it',
117 picker.opts.length === 4 && picker.opts.every(o => o.provider),
118 picker.opts.map(o => o.provider + ':' + o.model).join(', '));
119check('a new chat opens on the starred default, provider and all',
120 picker.selected === 'alpha-large' && picker.selProv === 'provA',
121 picker.selProv + ':' + picker.selected);
122
123// The same model name sits on both providers. This is the case a bare model id cannot answer.
124const dupes = picker.opts.filter(o => o.model === 'shared-model');
125check('a model served by two providers appears once under each, not once in total',
126 dupes.length === 2 && dupes[0].provider !== dupes[1].provider,
127 dupes.map(o => o.provider).join(' + '));
128
129// ── A chat started on the NON-default provider goes to that provider ───
130
131await p.evaluate(() => {
132 const sel = document.querySelector('.tile-model');
133 // Choose Provider B's own model, which is not the default.
134 const want = [...sel.querySelectorAll('option')]
135 .find(o => o.dataset.provider === 'provB' && o.value === 'beta-small');
136 want.selected = true;
137 sel.dispatchEvent(new Event('change', { bubbles: true }));
138 document.querySelector('.tile-start').click();
139});
140await p.waitForTimeout(900);
141await p.fill('#chat-input', 'hello');
142await p.click('#chat-send', { force: true });
143await p.waitForTimeout(3500);
144
145check('a chat started on Provider B reaches Provider B, not the default',
146 B.seen.length === 1 && A.seen.length === 0,
147 `A saw ${A.seen.length}, B saw ${B.seen.length}`);
148check('and it is sent with THAT provider\'s key',
149 B.seen.length > 0 && B.seen[0].auth.includes('key-for-B'),
150 B.seen[0] ? B.seen[0].auth.replace(/Bearer /, '') : '(nothing sent)');
151check('and asks for the model that was picked',
152 B.seen.length > 0 && B.seen[0].model === 'beta-small',
153 B.seen[0] ? B.seen[0].model : '(nothing sent)');
154
155// The chat's provider must survive a reload, or the next turn falls back to the default key.
156const persisted = await storedChats(s).then((raw) => {
157 const c = raw[0] || {};
158 return { model: c.model, provider: c.provider };
159});
160check('the chat records its provider, so a reload does not move it to the default key',
161 persisted.provider === 'provB' && persisted.model === 'beta-small',
162 persisted.provider + ':' + persisted.model);
163
164await shot(s, 'picker-chat');
165
166// ── A Diamond is created on a model the user chose ───────────────────────
167
168const before = { a: A.seen.length, b: B.seen.length };
169
170await p.click('#new-diamond-btn');
171await p.waitForSelector('.dlg-select', { timeout: 8000 });
172
173const dlg = await p.evaluate(() => {
174 const sel = document.querySelector('.dlg-select');
175 return {
176 groups: [...sel.querySelectorAll('optgroup')].map(g => g.label),
177 selected: sel.value,
178 selProv: sel.selectedOptions[0].dataset.provider,
179 };
180});
181check('New Diamond asks which model it should think with',
182 dlg.groups.length === 2, dlg.groups.join(' | '));
183check('and offers the starred default first', dlg.selected === 'alpha-large' && dlg.selProv === 'provA',
184 dlg.selProv + ':' + dlg.selected);
185
186// Name it, and put it on Provider B — deliberately NOT the default.
187await p.fill('.dlg-input:not(.dlg-select)', 'Diamond on B');
188await p.evaluate(() => {
189 const sel = document.querySelector('.dlg-select');
190 [...sel.querySelectorAll('option')]
191 .find(o => o.dataset.provider === 'provB' && o.value === 'beta-small').selected = true;
192 sel.dispatchEvent(new Event('change', { bubbles: true }));
193});
194await p.click('.dlg-ok');
195await p.waitForTimeout(2000);
196
197// Steering is a paid turn. It must go to the Diamond's OWN provider.
198const steered = await p.evaluate(() => {
199 const box = document.getElementById('chat-input');
200 const go = document.getElementById('chat-send');
201 if (!box || !go) return false;
202 box.value = 'tighten it';
203 box.dispatchEvent(new Event('input', { bubbles: true }));
204 go.click();
205 return true;
206});
207check('the steer control is there to drive', steered === true);
208await p.waitForTimeout(6000);
209
210const dA = A.seen.length - before.a, dB = B.seen.length - before.b;
211check('a Diamond created on Provider B thinks on Provider B',
212 dB > 0 && dA === 0, `A +${dA}, B +${dB}`);
213// Only the requests made SINCE the Diamond was created count — the chat's earlier request also
214// went to B, and would otherwise let this pass without the Diamond having done anything at all.
215const sinceDiamond = B.seen.slice(before.b);
216check('with that provider\'s key and model',
217 sinceDiamond.length > 0
218 && sinceDiamond.every(r => r.auth.includes('key-for-B') && r.model === 'beta-small'),
219 sinceDiamond.length ? sinceDiamond[0].model + ' / ' + sinceDiamond[0].auth.replace('Bearer ', '') : '(no steer request)');
220
221await shot(s, 'picker-focus');
222
223const errs = errors(s).filter(e => !/favicon|404|401|502|Bad Gateway|net::ERR/.test(e));
224console.log('\nconsole errors:', errs.slice(0, 5));
225check('nothing throws while all this happens', errs.length === 0, errs[0] || '');
226
227await s.close();
228A.close(); B.close();
229console.log(`\n${ok.length} passed, ${bad.length} failed`);
230if (bad.length) console.log('FAILED:\n ' + bad.join('\n '));
231process.exit(bad.length ? 1 : 0);