oxedyne/daimond — dev/verify_multichat.mjs

oxedyne/daimond/dev/verify_multichat.mjs

6.9 KiB, 1 run

created by r2519314175:543, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1	// You must be able to work in a second chat while the first is still running.
2	//
3	// THE HEADER USED TO SAY "and Stop must hit the chat you are looking at -- not
4	// whichever started last". Nothing below has ever pressed Stop, so the sentence
5	// has gone: a claim in a comment is not a check, and this one had been read as
6	// one for as long as the file has existed.
7	//
8	// WHAT THE SECOND CHAT'S REPLY IS LOOKED FOR IN. `bSent` used to be
9	// `/Reply in B/.test(#chat-output innerText)`, and the phrase it looked for is
10	// the one this file TYPES: the transcript renders the user's own message, so the
11	// check passed on a turn that never got an answer at all -- the same defect
12	// verify_backup had. The reply is now a nonce the mock is asked to say back, and
13	// it is looked for in `.chat-msg-assistant` alone.
14	//
15	// PROVED AGAINST TWO BREAKS FIRST:
16	// --break noreply B's message is sent with `@err 500`, so the user's line is
17	// rendered and no assistant message ever arrives. The old
18	// check went green on exactly that; this one must go red.
19	// --break shorta A's turn is short enough to be over before B's finishes,
20	// which is the state every run of this file could have been
21	// in without anybody knowing.
22	//
23	// node dev/verify_multichat.mjs --break noreply # expected to FAIL
24	// node dev/verify_multichat.mjs --break shorta # expected to FAIL
25	// node dev/verify_multichat.mjs # and then, clean
26	import { open, newChat, shot, errors } from './harness.mjs';
27
28	const ok = [], bad = [];
29	const check = (name, pass, detail) => {
30	(pass ? ok : bad).push(name + (detail ? ' — ' + detail : ''));
31	console.log((pass ? ' ok ' : ' FAIL ') + name + (detail ? ' — ' + detail : ''));
32	};
33
34	const BREAK = (() => {
35	const i = process.argv.indexOf('--break');
36	return i > 0 ? String(process.argv[i + 1] \|\| '') : '';
37	})();
38	if (BREAK && !['noreply', 'shorta'].includes(BREAK)) {
39	console.error(`unknown break '${BREAK}'; known: noreply, shorta`);
40	process.exit(2);
41	}
42
43	const s = await open({ name: 'multichat' });
44	if (BREAK) console.log(`\n* RUNNING UNDER --break ${BREAK}: failures below are the point *\n`);
45
46	// Chat A: start a LONG turn (streams slowly), do not wait for it.
47	await newChat(s);
48	// `shorta` gives A a turn that is over before B's is: still running when the
49	// first check looks (2.4s of chunks), finished long before the last one does.
50	await s.page.fill('#chat-input', BREAK === 'shorta' ? '@long 20' : '@long 200');
51	await s.page.click('#chat-send');
52	await s.page.waitForTimeout(800);
53
54	const aRunning = await s.page.evaluate(() => {
55	const b = document.getElementById('chat-send');
56	return { stopMode: /stop/i.test((b.getAttribute('title') \|\| '') + (b.className \|\| '')), disabled: document.getElementById('chat-input').disabled };
57	});
58	// The composer is NOT disabled mid-turn any more: a turn cannot be interrupted, but the next
59	// thing to say can be typed while the answer arrives and is sent when the turn ends. Stop-mode on
60	// the send button is the signal that a turn is running; an unusable box never was.
61	check('A LONG TURN IN CHAT A IS RUNNING — stop-mode on the send button, composer still live',
62	aRunning.stopMode && !aRunning.disabled, JSON.stringify(aRunning));
63
64	// Start a SECOND chat while A still streams.
65	await s.page.click('#new-session-btn');
66	await s.page.waitForTimeout(400);
67	// Start the pending tile. `.tile-start` by class, NOT has-text("Start") —
68	// the substring match also hits the BYOK form's hidden "Save & start".
69	const startBtn = s.page.locator('.tile-start').first();
70	const started = await startBtn.count();
71	// This used to be a bare `if`: with no Start button the file went on to measure
72	// chat A a second time and printed three greens about it.
73	check('a second chat can be started while the first one runs', !!started, `${started} .tile-start on the rail`);
74	if (started) await startBtn.click();
75	await s.page.waitForTimeout(600);
76
77	const bComposer = await s.page.evaluate(() => {
78	const inp = document.getElementById('chat-input');
79	const b = document.getElementById('chat-send');
80	return { inputDisabled: inp.disabled, stopMode: /stop/i.test((b.getAttribute('title') \|\| '') + (b.className \|\| '')) };
81	});
82	check('CHAT B IS USABLE WHILE A RUNS — input enabled, and its send button is in send-mode not stop-mode',
83	!bComposer.inputDisabled && !bComposer.stopMode, JSON.stringify(bComposer));
84
85	// Type and send in B while A still runs. The nonce is what the mock is asked to
86	// say back, so finding it in an ASSISTANT bubble is the answer arriving.
87	const NONCE = 'replyB-' + Math.random().toString(36).slice(2, 8);
88	let b = { user: false, asst: false, text: '' };
89	if (!bComposer.inputDisabled) {
90	await s.page.fill('#chat-input', BREAK === 'noreply' ? `@err 500 ${NONCE}` : `@text ${NONCE}`);
91	await s.page.click('#chat-send');
92	await s.page.waitForTimeout(4000);
93	b = await s.page.evaluate((nonce) => {
94	const txt = (sel) => [...document.querySelectorAll(sel)].map(e => e.textContent).join('\n');
95	const asst = txt('#chat-output .chat-msg-assistant .chat-msg-content');
96	return {
97	user: txt('#chat-output .chat-msg-user .chat-msg-content').includes(nonce),
98	asst: asst.includes(nonce),
99	text: asst.slice(-120),
100	};
101	}, NONCE);
102	}
103	await shot(s, 'multichat-B-active');
104
105	check('B\'s message was accepted into B\'s transcript', b.user, `looking for ${NONCE} in .chat-msg-user`);
106	check('B\'S TURN ANSWERED WHILE A RAN — the answer is in an assistant bubble, not merely the line we typed',
107	b.asst, `looking for ${NONCE} in .chat-msg-assistant, which holds ${JSON.stringify(b.text)}`);
108
109	// "WHILE A RAN" WAS NEVER CHECKED. A's turn was seen to start and was never
110	// looked at again, so a run in which it had finished before B was even opened
111	// read exactly the same. It streams 200 chunks at 120ms each in the mock (24s of
112	// server-side sleep, whatever this machine's speed), and everything above takes
113	// about eight -- so it must still be going, and the composer says so when the
114	// tile is selected again.
115	const idle = s.page.locator('#session-list .session-box:not(.active)');
116	const others = await idle.count();
117	check('chat A is still on the rail, unselected', others === 1, `${others} unselected chat tiles`);
118	let back = { stopMode: null };
119	if (others === 1) {
120	await idle.first().click();
121	await s.page.waitForTimeout(700);
122	back = await s.page.evaluate(() => {
123	const b = document.getElementById('chat-send');
124	return { stopMode: /stop/i.test((b.getAttribute('title') \|\| '') + (b.className \|\| '')) };
125	});
126	}
127	check('AND A WAS STILL RUNNING THROUGHOUT — its own composer is still in stop-mode',
128	back.stopMode === true, `stop-mode=${back.stopMode}`);
129
130	const errs = errors(s).filter(e => !/502\|Bad Gateway/.test(e));
131	check('nothing threw', errs.length === 0, errs.slice(0, 2).join(' \| '));
132
133	await s.close();
134	console.log(`\n${ok.length} passed, ${bad.length} failed`);
135	if (bad.length) { bad.forEach(x => console.log(' FAILED: ' + x)); process.exit(1); }