Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_interject.mjs

18.8 KiB, 1 run

created by r2519314175:491, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_interject.mjs — you can speak into a turn that is already running.
2//
3// A turn is not one request; it is a round of them, each carrying the last
4// round's tool results. Nothing can be added to a request already in flight, but
5// the message list is rebuilt BETWEEN rounds, and the agent drains an
6// interjection queue at exactly that seam -- after the tool replies go back (the
7// API requires every tool_call to be answered by a tool message) and before the
8// next request goes out. So a model twenty tool calls into the wrong approach can
9// be corrected without stopping it and without waiting for it to finish.
10//
11// Four promises are checked here, and they are not the same promise:
12//
13// 1. It ARRIVES IN THAT TURN. The proof is the mock's own log, not the screen:
14// one request carries the original question, a tool reply, and the
15// interjection, in that order. Before the fix the interjection could only
16// ever appear in a request of its own, after the turn had ended.
17// 2. It is DRAWN WHERE IT LANDED. A correction shown at the bottom, under the
18// work it was meant to redirect, reads as one that was ignored.
19// 3. The QUEUE still behaves as it did. A turn with no tool call in it has no
20// seam, so what was typed is still waiting when the turn ends -- and then it
21// falls back to the queue, which sends it as its own turn, or hands it back
22// to the composer if the turn failed or was stopped. A chat left in the
23// background keeps its badge and drains on return, exactly as before.
24// 4. STOP still means stop. A turn the user killed hands back what they typed
25// rather than spending money on it.
26//
27// node dev/verify_interject.mjs
28//
29// Needs dev/serve.mjs (DAIMOND_PORT, default 8777) and dev/mockllm.mjs
30// (DAIMOND_MOCK_PORT, default 9099). A verifier that reports "Daimond could not answer"
31// is missing the mock, not finding a bug.
32
33import { open, newChat, shot, errors, signInAs } from './harness.mjs';
34
35const ok = [], bad = [];
36const check = (name, pass, detail) => {
37 (pass ? ok : bad).push(name + (detail ? ' — ' + detail : ''));
38 console.log((pass ? ' ok ' : ' FAIL ') + name + (detail ? ' — ' + detail : ''));
39 return pass;
40};
41const sleep = ms => new Promise(r => setTimeout(r, ms));
42
43const s = await open({ name: 'interject' });
44const p = s.page;
45
46// Toasts remove themselves after ~4s, so they are recorded as they appear.
47await p.evaluate(() => {
48 window.__toasts = [];
49 new MutationObserver(muts => {
50 for (const m of muts) for (const n of m.addedNodes) {
51 if (n.nodeType === 1 && n.classList && n.classList.contains('daimond-toast')) {
52 window.__toasts.push({ text: n.textContent, err: n.classList.contains('err') });
53 }
54 }
55 }).observe(document.body, { childList: true });
56});
57const toasts = () => p.evaluate(() => window.__toasts.slice());
58
59// What actually went out on the wire, captured by wrapping the page's own
60// `fetch`. The claim being checked is about the request the model received, and
61// nothing the app draws is evidence of that -- the DOM is the app marking its own
62// homework. The mock's own log holds the same thing from the server's side, but a
63// suite that does not set DAIMOND_MOCK_LOG shares one file with every other on the
64// machine, and a concurrent run truncates it mid-pass; this reads the bytes this
65// page sent.
66await p.evaluate(() => {
67 window.__sent = [];
68 const orig = window.fetch;
69 window.fetch = function (req) {
70 try {
71 if (req && req.url && /chat\/completions/.test(req.url) && typeof req.clone === 'function') {
72 req.clone().text().then((txt) => {
73 try { window.__sent.push(JSON.parse(txt)); } catch (e) { /* not JSON */ }
74 });
75 }
76 } catch (e) { /* capture must never break the request */ }
77 return orig.apply(this, arguments);
78 };
79});
80const sent = () => p.evaluate(() => (window.__sent || []).slice());
81
82/// The shape of the thread: the real turns, what landed mid-turn, and what is
83/// still waiting to.
84const shape = () => p.evaluate(() => {
85 const out = document.getElementById('chat-output');
86 const send = document.getElementById('chat-send');
87 const kind = (n) =>
88 n.classList.contains('chat-msg-user') ? 'user'
89 : n.classList.contains('chat-msg-interjected') ? 'cut'
90 : n.classList.contains('tool-block') ? 'tool'
91 : n.classList.contains('chat-msg-assistant') ? 'asst'
92 : n.id === 'chat-queued' ? 'waiting'
93 : '';
94 const users = [...out.querySelectorAll('.chat-msg-user')];
95 const cuts = [...out.querySelectorAll('.chat-msg-interjected')];
96 return {
97 order: [...out.children].map(kind).filter(Boolean).join(','),
98 users: users.length,
99 turns: users.map(u => u.dataset.turn).join(','),
100 userText: users.map(u => u.querySelector('.chat-msg-content').textContent).join(' | '),
101 cuts: cuts.map(c => c.querySelector('.chat-msg-content').textContent),
102 cutTurns: cuts.map(c => c.dataset.turn).join(','),
103 cutsAreUsers: cuts.filter(c => c.classList.contains('chat-msg-user')).length,
104 waiting: [...out.querySelectorAll('.chat-msg-queued')]
105 .map(e => e.querySelector('.chat-msg-content').textContent),
106 heads: [...out.querySelectorAll('.chat-queued-head')].map(e => e.textContent),
107 composer: document.getElementById('chat-input').value,
108 sendTitle: send.getAttribute('title') || '',
109 stopMode: send.classList.contains('stop'),
110 text: out.innerText,
111 busy: !!(window.DaimondCore && window.DaimondCore.busy()),
112 };
113});
114
115/// Wait until the button offers Send again with an empty box — the app's own idea
116/// of "this turn is over".
117async function idle(timeout = 90000) {
118 const t0 = Date.now();
119 while (Date.now() - t0 < timeout) {
120 if (!(await shape()).stopMode) return true;
121 await sleep(250);
122 }
123 return false;
124}
125
126/// Type into the composer and press Enter, which is how a message is sent — and,
127/// mid-turn, how it is said into the turn.
128async function type(text) {
129 await p.fill('#chat-input', text);
130 await p.press('#chat-input', 'Enter');
131 await sleep(400);
132}
133
134/// Arrange for `text` to be typed the instant the first tool step appears.
135///
136/// This is the only way to be sure the message is typed while a tool is actually
137/// running: an OPFS tool returns in milliseconds, so a sleep long enough to be
138/// reliable is far longer than the window it is aiming at. The observer fires
139/// synchronously on the DOM write that `renderToolCall` makes, which is before
140/// the tool has returned and long before the round's seam.
141async function typeWhenToolRuns(text) {
142 await p.evaluate((msg) => {
143 window.__cutAt = null;
144 const out = document.getElementById('chat-output');
145 const obs = new MutationObserver(() => {
146 if (window.__cutAt) return;
147 if (!out.querySelector('.tool-block')) return;
148 window.__cutAt = Date.now();
149 obs.disconnect();
150 const box = document.getElementById('chat-input');
151 box.value = msg;
152 box.dispatchEvent(new Event('input', { bubbles: true }));
153 box.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }));
154 });
155 obs.observe(out, { childList: true, subtree: true });
156 }, text);
157}
158
159/// Click something if it is there. A verifier has to be able to RUN against the
160/// code before the fix, where these controls do not exist — a hard click would
161/// abort the pass with a Playwright timeout instead of failing the check that is
162/// meant to catch it.
163async function clickIf(sel) {
164 const el = await p.$(sel);
165 if (!el) return false;
166 await el.click({ force: true });
167 await sleep(400);
168 return true;
169}
170
171await newChat(s);
172
173// ── 1. It reaches the model IN the turn it was typed into ────────────
174//
175// `@chain` is the shape a real agentic turn takes: a tool call, then a second
176// one, then a text reply. The interjection is typed on the first tool step, so
177// it has a seam to land in without waiting for the turn to end.
178const CUT = 'NUDGE-MIDTURN';
179await typeWhenToolRuns(CUT);
180await p.fill('#chat-input', '@chain file_list {"path":"."}');
181await p.click('#chat-send', { force: true });
182await sleep(1200);
183const cutAt = await p.evaluate(() => window.__cutAt);
184check('the message was typed while a tool step was running, not after the turn',
185 !!cutAt, cutAt ? 'yes' : 'the tool step never appeared');
186await idle();
187await sleep(800);
188
189const log = await sent();
190/// A request that carries the interjection AT THE SEAM.
191///
192/// The message immediately before it must be a tool reply. That is the whole
193/// claim, and it is what separates landing in the turn from merely appearing in
194/// the transcript afterwards: a message queued and sent as its own turn also ends
195/// up in a request with the question and the tool replies above it -- but with the
196/// turn's final answer sitting between them. Only a delivery at the seam puts the
197/// user's words directly after the last tool result, before the model has spoken
198/// again.
199const atSeam = log.filter(r => {
200 const ms = r.messages || [];
201 const iCut = ms.findIndex(m => m.role === 'user' && String(m.content || '').trim() === CUT);
202 const iAsked = ms.findIndex(m => m.role === 'user' && String(m.content || '').startsWith('@chain'));
203 if (iCut <= 0 || iAsked === -1 || iAsked >= iCut) return false;
204 return ms[iCut - 1].role === 'tool';
205});
206check('the model was sent the message IN that turn, straight after a tool reply',
207 atSeam.length > 0,
208 `${atSeam.length} of ${log.length} requests carried it at the seam`);
209check('and it went as a user turn, which is the only role a correction can wear',
210 atSeam.length > 0 && atSeam[0].messages
211 .filter(m => String(m.content || '').trim() === CUT)
212 .every(m => m.role === 'user'));
213
214const one = await shape();
215check('no second turn was started for it: it joined the one already running',
216 one.users === 1, `${one.users} user turns — ${one.userText.slice(0, 90)}`);
217check('the model answered the interjection, so it really had it',
218 one.text.includes(CUT) && /Mock reply to: NUDGE-MIDTURN/.test(one.text),
219 one.text.slice(-120).replace(/\n/g, ' '));
220
221// ── 2. It is drawn WHERE it landed ───────────────────────────────────
222check('it is drawn in the thread as a message that arrived',
223 one.cuts.length === 1 && one.cuts[0] === CUT, JSON.stringify(one.cuts));
224check('after the tool step it cut into, and before the answer that followed it',
225 /tool,(tool,)*cut,asst/.test(one.order), one.order);
226check('nothing is left drawn as waiting once it has landed',
227 one.waiting.length === 0, JSON.stringify(one.waiting));
228check('it belongs to the turn it cut into, not a turn of its own',
229 one.cutTurns === '1' && one.turns === '1',
230 `cut@${one.cutTurns} users@${one.turns}`);
231check('and it is not a .chat-msg-user, which is what the numbering counts',
232 one.cutsAreUsers === 0);
233await shot(s, 'interject-landed');
234
235// ── 3. Waiting is drawn as waiting, and says what will happen ────────
236//
237// `@long` writes prose straight through with no tool call, so there is no seam
238// in it at all. That is the honest limit, and the app has to say so rather than
239// promise a delivery it cannot make.
240await p.fill('#chat-input', '@long 60');
241await p.click('#chat-send', { force: true });
242await sleep(1200);
243const empty = await shape();
244check('with nothing typed, the button is still Stop — the signal everything reads',
245 empty.stopMode === true, empty.sendTitle);
246await p.fill('#chat-input', 'WAITING-ONE');
247await p.dispatchEvent('#chat-input', 'input');
248await sleep(200);
249const typed = await shape();
250check('but with a correction typed into it, the button is Send, not Stop',
251 typed.stopMode === false && /send/i.test(typed.sendTitle), typed.sendTitle);
252await p.click('#chat-send', { force: true });
253await sleep(600);
254const waiting = await shape();
255check('pressing it sends rather than killing the turn it was steering',
256 waiting.busy === true && waiting.waiting.length === 1
257 && waiting.waiting[0] === 'WAITING-ONE', JSON.stringify(waiting.waiting));
258check('a message not yet delivered is drawn as waiting, not as part of the thread',
259 waiting.cuts.length === 1, `${waiting.cuts.length} landed so far`);
260check('and the line above it says both when it goes in and that it may have to wait',
261 waiting.heads.some(h => /step/i.test(h) && /finish/i.test(h)),
262 JSON.stringify(waiting.heads));
263check('the composer is cleared, as it is on an ordinary send',
264 waiting.composer === '', JSON.stringify(waiting.composer));
265await shot(s, 'interject-waiting');
266
267// ── 4. A turn with no seam falls back to the queue, and is not lost ──
268await idle();
269await sleep(2500);
270await idle();
271await sleep(1200);
272const fell = await shape();
273check('a message that found no seam is sent as its own turn rather than dropped',
274 /WAITING-ONE/.test(fell.userText), fell.userText.slice(-120));
275check('nothing is left waiting once it has gone',
276 fell.waiting.length === 0 && fell.busy === false, JSON.stringify(fell.waiting));
277check('and the turn numbering still has no gap, so a fold maps through it',
278 fell.turns === Array.from({ length: fell.users }, (_, i) => i + 1).join(','), fell.turns);
279
280// ── 5. The × takes a waiting message back out ────────────────────────
281await p.fill('#chat-input', '@long 60');
282await p.click('#chat-send', { force: true });
283await sleep(1200);
284await type('CANCEL-ME');
285await clickIf('#chat-queued .queue-x');
286const cancelled = await shape();
287check('the × takes a waiting message back out',
288 cancelled.waiting.length === 0, JSON.stringify(cancelled.waiting));
289await idle();
290await sleep(1500);
291const afterCancel = await shape();
292check('and a cancelled message is never sent, to the turn or after it',
293 !/CANCEL-ME/.test(afterCancel.text), afterCancel.text.slice(-80).replace(/\n/g, ' '));
294
295// ── 6. Stop still hands back what was typed ──────────────────────────
296const beforeStop = await shape();
297await p.fill('#chat-input', '@long 120');
298await p.click('#chat-send', { force: true });
299await sleep(1500);
300await type('AFTER-STOP');
301await p.click('#chat-send', { force: true }); // box empty: this is Stop
302await sleep(2500);
303const stopped = await shape();
304check('Stop returns what was said into the turn to the composer, unsent',
305 stopped.composer === 'AFTER-STOP' && stopped.waiting.length === 0,
306 `composer=${JSON.stringify(stopped.composer)} waiting=${JSON.stringify(stopped.waiting)}`);
307check('and says so, rather than leaving the text to be wondered about',
308 (await toasts()).some(x => x.err && /unsent|back in the box/i.test(x.text)),
309 JSON.stringify((await toasts()).slice(-2)));
310await sleep(3000);
311const afterStop = await shape();
312check('nothing said into a stopped turn is sent after it',
313 afterStop.users === beforeStop.users + 1,
314 `${beforeStop.users} → ${afterStop.users} (the stopped turn only)`);
315await p.fill('#chat-input', '');
316await shot(s, 'interject-stopped');
317
318// ── 7. A chat left in the background is unchanged ────────────────────
319//
320// The queue is still the right answer for a conversation the user has walked
321// away from: runTurn draws into the live thread, so a turn started for a chat in
322// the background would write itself into the one on screen. It waits there,
323// badged, and drains on return -- which is what it did before any of this.
324const tiles = () => p.evaluate(() => [...document.querySelectorAll('#session-list .session-box')]
325 .map(b => ({
326 id: b.dataset.id,
327 badge: (b.querySelector('.queue-badge') || {}).textContent || '',
328 })));
329async function addChat() {
330 await p.click('#new-session-btn', { force: true });
331 await sleep(400);
332 const start = p.locator('.tile-start').first();
333 if (await start.count()) await start.click({ force: true });
334 await sleep(500);
335}
336async function openTile(id) {
337 await p.evaluate((cid) => {
338 const box = [...document.querySelectorAll('#session-list .session-box')]
339 .find(b => b.dataset.id === cid);
340 if (box) box.click();
341 }, id);
342 await sleep(600);
343}
344
345const beforeB = (await tiles()).map(x => x.id);
346await addChat();
347const chatB = (await tiles()).map(x => x.id).find(id => beforeB.indexOf(id) === -1) || '';
348const chatA = beforeB[0] || '';
349check('a second chat can be opened beside the first', !!chatA && !!chatB && chatA !== chatB,
350 `A=${chatA} B=${chatB}`);
351
352await openTile(chatA);
353await p.fill('#chat-input', '@long 30');
354await p.click('#chat-send', { force: true });
355await sleep(1200);
356await type('LEFT-BEHIND');
357await openTile(chatB);
358const away = await tiles();
359check('the tile of the chat left behind still says something is waiting on it',
360 !!(away.find(x => x.id === chatA) || {}).badge,
361 JSON.stringify((away.find(x => x.id === chatA) || {}).badge));
362check('and the chat being looked at is not badged for someone else’s',
363 !(away.find(x => x.id === chatB) || {}).badge);
364await sleep(7000);
365const stillAway = await tiles();
366check('it is still there once that turn has finished — nothing is sent behind your back',
367 !!(stillAway.find(x => x.id === chatA) || {}).badge,
368 JSON.stringify((stillAway.find(x => x.id === chatA) || {}).badge));
369
370await openTile(chatA);
371await idle();
372await sleep(1500);
373const returned = await shape();
374const cleared = await tiles();
375check('going back to the chat sends what was left on it, as a real turn',
376 /LEFT-BEHIND/.test(returned.userText), returned.userText.slice(-120));
377check('as a turn, not a bubble, with the numbering still unbroken',
378 returned.waiting.length === 0
379 && returned.turns === Array.from({ length: returned.users }, (_, i) => i + 1).join(','),
380 `waiting=${JSON.stringify(returned.waiting)} turns=${returned.turns}`);
381check('and the badge clears once it has gone',
382 !(cleared.find(x => x.id === chatA) || {}).badge,
383 JSON.stringify((cleared.find(x => x.id === chatA) || {}).badge));
384await shot(s, 'interject-background');
385
386// ── 8. It survives a reload, drawn where it landed ───────────────────
387await p.reload({ waitUntil: 'domcontentloaded' });
388await sleep(1500);
389await signInAs(s, 'interject'); // a reload lands on the passphrase gate
390await sleep(1500);
391await openTile(chatA);
392await sleep(800);
393const reloaded = await p.evaluate(() => {
394 const out = document.getElementById('chat-output');
395 if (!out) return { cuts: [], users: 0 };
396 return {
397 cuts: [...out.querySelectorAll('.chat-msg-interjected .chat-msg-content')].map(e => e.textContent),
398 users: out.querySelectorAll('.chat-msg-user').length,
399 };
400});
401check('a landed message is still in the transcript after a reload',
402 reloaded.cuts.includes(CUT), JSON.stringify(reloaded.cuts));
403check('and is still not counted as a question of its own',
404 reloaded.users > 0 && !reloaded.cuts.some(c => c === ''), `${reloaded.users} user turns`);
405
406const errs = errors(s).filter(e => !/favicon|404|401|402|502|Bad Gateway|net::ERR/.test(e));
407console.log('\nconsole errors:', errs.slice(0, 5));
408check('nothing throws while all this happens', errs.length === 0, errs[0] || '');
409
410await s.close();
411console.log(`\n${ok.length} passed, ${bad.length} failed`);
412if (bad.length) console.log('FAILED:\n ' + bad.join('\n '));
413process.exit(bad.length ? 1 : 0);