oxedyne/daimond — dev/verify_ending.mjs

oxedyne/daimond/dev/verify_ending.mjs

15.8 KiB, 1 run

created by r2519314175:397, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1	// verify_ending.mjs — every turn says how it ended, and it reads as furniture.
2	//
3	// ── WHY ──────────────────────────────────────────────────────────────────────
4	//
5	// A model announced "let me rewrite from line 43 through 49 applying the
6	// lessons" and then ended its turn having written nothing. The spinner stopped,
7	// which was correct — the turn HAD ended — and the owner was left saying **"I
8	// have no visibility on what occurred here."** `AgentEvent::Ended` now closes
9	// every turn and `dev/CONTRACT_CLAIMS.md` §3 says how it is drawn.
10	//
11	// ── WHAT IT ASSERTS, WHICH IS THE FURNITURE RULE AND NOT THE FEATURE ─────────
12	//
13	// Drawing a line is the easy half and it is not the half that matters. §3 is a
14	// rule about NOT crying wolf, and two checks here are the whole of it:
15	//
16	// - a toolless turn draws nothing at all. A pure chat has no tool log, so
17	// a line under every message in it is noise with nothing behind it.
18	// - an ordinary successful turn is not styled as a notice. An app that
19	// appends a warning to every turn teaches its reader to skip the one that
20	// mattered, which is the failure this whole mechanism exists to prevent.
21	// Only `refused`, `failed` or a missing path may promote it.
22	//
23	// Without those two this becomes a warning nobody reads, and every other check
24	// here would still be green.
25	//
26	// node dev/verify_ending.mjs
27	// node dev/verify_ending.mjs --break nofurniture # a toolless turn draws a line
28	// node dev/verify_ending.mjs --break alwaysnotice # every ending is a notice
29	// node dev/verify_ending.mjs --break nopersist # a reload loses the ending
30	//
31	// A `--break` run EXPECTS to fail: exit 0 when something reddened, 1 when
32	// nothing did, because a break that changes nothing is itself a failing run.
33	//
34	// Needs dev/serve.mjs (:8777) and dev/mockllm.mjs (:9099).
35
36	import fs from 'node:fs';
37	import path from 'node:path';
38	import { fileURLToPath } from 'node:url';
39	import { open, newChat, signInAs, shot, errors } from './harness.mjs';
40
41	const HERE = path.dirname(fileURLToPath(import.meta.url));
42	const WWW = path.join(HERE, '..', 'www');
43
44	const ok = [], bad = [];
45	const check = (name, pass, detail) => {
46	(pass ? ok : bad).push(name + (detail ? ' — ' + detail : ''));
47	console.log((pass ? ' ok ' : ' FAIL ') + name + (detail ? ' — ' + detail : ''));
48	};
49	const sleep = ms => new Promise(r => setTimeout(r, ms));
50
51	// ── The breaks ───────────────────────────────────────────────────────────────
52	//
53	// Each is scoped to survive every check but the one it proves. `nofurniture`
54	// drops the `offered === 0` guard, which is the rule that keeps a pure chat
55	// clean; `alwaysnotice` promotes every ending, which is the app teaching its
56	// reader to skip the line; `nopersist` leaves the live drawing alone and stops
57	// the reload redrawing it, so only the durability check moves.
58	const BREAK = (() => { const i = process.argv.indexOf('--break'); return i > 0 ? process.argv[i + 1] : ''; })();
59	const BREAKS = {
60	// BOTH GUARDS, and the fact that there are two is the finding. Patching
61	// `endingParts` alone reddened NOTHING on the first run: `endLogOf` refuses to
62	// store a toolless ending as well, so the line was never reached even with the
63	// drawing rule removed. That is belt and braces rather than a defect -- but a
64	// break that restores only half of a two-place rule launders a plain run as
65	// proof of the other half, which is exactly what this note exists to stop the
66	// next reader repeating.
67	nofurniture: [{
68	file: 'js/daimond.js',
69	find: " // Rule 2, first, so no caller can forget it.\n if (!e \|\| !((e.offered \| 0) > 0)) return null;",
70	with: " if (!e) return null; // --break nofurniture",
71	}, {
72	file: 'js/daimond.js',
73	find: " if (!ev \|\| !((ev.offered \| 0) > 0)) return null;",
74	with: " if (!ev) return null; // --break nofurniture",
75	}],
76	alwaysnotice: [{
77	file: 'js/daimond.js',
78	find: " notice: !!(refused \|\| failed \|\| missing.length),",
79	with: " notice: true, /* --break alwaysnotice */",
80	}],
81	nopersist: [{
82	file: 'js/daimond.js',
83	find: " else if (m.role === 'end_log') { appendEnding(m); }",
84	with: " else if (m.role === 'end_log') { /* --break nopersist */ }",
85	}],
86	};
87
88	function damagedFiles() {
89	const byFile = new Map();
90	for (const spec of (BREAKS[BREAK] \|\| [])) {
91	// From what is ALREADY DAMAGED where a break edits one file twice, or the
92	// second edit would drop the first and a two-place rule would be half
93	// restored -- silently, since both anchors are found either way.
94	const src = byFile.get(spec.file) \|\| fs.readFileSync(path.join(WWW, spec.file), 'utf8');
95	if (!src.includes(spec.find)) {
96	// A break whose anchor is not there patches nothing and launders a plain
97	// run as proof. Loud, and fatal.
98	console.error(`--break ${BREAK}: anchor not found in ${spec.file}. The break is stale.`);
99	process.exit(1);
100	}
101	byFile.set(spec.file, src.replace(spec.find, spec.with));
102	}
103	return byFile;
104	}
105
106	async function serveBreaks(page) {
107	if (!BREAK) return;
108	for (const [file, body] of damagedFiles()) {
109	await page.route('**/' + file, r => r.fulfill({
110	status: 200, contentType: 'application/javascript', body,
111	}));
112	}
113	}
114
115	const s = await open({ name: 'ending', route: serveBreaks });
116	const p = s.page;
117
118	/// Every ending line in the chat on screen, with whether it is a notice.
119	const endings = () => p.evaluate(() =>
120	[...document.querySelectorAll('#chat-output .chat-msg-ended')].map(e => ({
121	text: (e.textContent \|\| '').trim(),
122	notice: e.classList.contains('ended-notice'),
123	title: e.getAttribute('title') \|\| '',
124	})));
125
126	/// Come back from a reload, past the gate.
127	///
128	/// A RELOAD IS A LOCK — `boot()` finds the stored identity and returns before
129	/// `renderAll`, so waiting only for `__DAIMOND_READY` reads the lock screen (see
130	/// dev/verify_reopen.mjs). AND THE LOCK LANDS AFTER THAT FLAG: a single read of
131	/// `body.locked` straight after it can be taken in the gap before the gate is up,
132	/// which skips the sign-in and then waits thirty seconds on a modal nobody
133	/// answered. So the gate is waited FOR, briefly, and only then answered.
134	///
135	/// Nothing here throws: a deadline that expires leaves the checks below to report
136	/// what they see, because a race that takes the process with it is a run with no
137	/// check attributed to it.
138	async function reboot() {
139	await p.reload({ waitUntil: 'domcontentloaded' });
140	await p.waitForFunction(() => window.__DAIMOND_READY === true, null, { timeout: 30000 })
141	.catch(() => {});
142	const gate = Date.now() + 10000;
143	let locked = false;
144	while (Date.now() < gate) {
145	locked = await p.evaluate(() => document.body.classList.contains('locked'));
146	if (locked) break;
147	await sleep(200);
148	}
149	if (locked) await signInAs(s, 'ending').catch(() => {});
150	await p.waitForFunction(() => !document.body.classList.contains('locked'), null,
151	{ timeout: 30000 }).catch(() => {});
152	await sleep(600);
153	}
154
155	/// Send one message and wait for the turn to stop.
156	///
157	/// NOT `harness.chat`, which opens a chat of its own when none is in focus. Each
158	/// case here owns its chat, so the send is the send and nothing else.
159	async function say(text, timeout = 40000) {
160	await p.fill('#chat-input', text);
161	await p.click('#chat-send', { force: true });
162	await p.waitForTimeout(300);
163	const until = Date.now() + timeout;
164	while (Date.now() < until) {
165	const busy = await p.evaluate(() => {
166	const b = document.getElementById('chat-send');
167	if (!b) return false;
168	const t = (b.getAttribute('title') \|\| '') + (b.className \|\| '');
169	return /stop/i.test(t) \|\| b.disabled;
170	});
171	if (!busy) break;
172	await sleep(200);
173	}
174	await sleep(500);
175	}
176
177	// ── 1. THE OWNER'S CASE: tools on the table, and no call made ────────────────
178	//
179	// `offered > 0, calls === 0`. This is the turn he watched: nothing technically
180	// wrong, so nothing was said. The line is what replaces that silence — and it is
181	// FURNITURE, because a turn that simply answered is not a fault and drawing it
182	// as one is how the reader learns to skip the line.
183	await newChat(s);
184	await say('@text I will rewrite lines 43 through 49 applying the lessons.');
185	const idle = await endings();
186	check('1a a turn that used no tool still says how it ended',
187	idle.length === 1, JSON.stringify(idle));
188	check('1b and it says NO TOOLS WERE USED, which is the fact he could not see',
189	idle.length === 1 && /no tools used/i.test(idle[0].text), (idle[0] \|\| {}).text \|\| '(none)');
190	check('1c AND IT IS NOT A NOTICE — a turn that answered is not a fault',
191	idle.length === 1 && idle[0].notice === false,
192	idle.length ? ('notice=' + idle[0].notice) : '(no line)');
193	check('1d the figures nobody put on the line are in its title',
194	idle.length === 1 && /\d/.test(idle[0].title), (idle[0] \|\| {}).title \|\| '(no title)');
195	await shot(s, 'ending-1-no-tools');
196
197	// ── 2. An ordinary successful turn, WITH a tool call ─────────────────────────
198	//
199	// The second half of the not-crying-wolf rule: the commonest turn in the app
200	// must read as quietly as the one above.
201	await newChat(s);
202	const dir = await p.evaluate(() => {
203	const f = window.DaimondAttach.focus();
204	return window.DaimondAttach.chatScratch(f.id);
205	});
206	await say(`@tool file_write {"path":"${dir}/ok.txt","content":"hi"}`);
207	const good = await endings();
208	check('2a a turn that called a tool says so',
209	good.length === 1 && /1/.test(good[0].text), JSON.stringify(good));
210	check('2b AND AN ORDINARY SUCCESSFUL TURN IS NOT STYLED AS A NOTICE',
211	good.length === 1 && good[0].notice === false,
212	good.length ? ('notice=' + good[0].notice + ' — ' + good[0].text) : '(no line)');
213	await shot(s, 'ending-2-ordinary');
214
215	// ── 3. A refusal, which is what a notice is FOR ──────────────────────────────
216	//
217	// A write outside the chat's fence comes back `CallOutcome::Refused` — the door
218	// turned it away, nothing was written, and the turn finished normally. That is
219	// exactly the ending a reader has to be able to pick out of a run of quiet ones.
220	await newChat(s);
221	await say('@tool file_write {"path":"/etc/passwd","content":"no"}');
222	const refused = await endings();
223	check('3a a refused call is counted on the line',
224	refused.length === 1 && /1/.test(refused[0].text), JSON.stringify(refused));
225	check('3b AND IT IS PROMOTED TO A NOTICE',
226	refused.length === 1 && refused[0].notice === true,
227	refused.length ? ('notice=' + refused[0].notice + ' — ' + refused[0].text) : '(no line)');
228	await shot(s, 'ending-3-refused');
229
230	// ── 4. A reload still shows how the turn ended ───────────────────────────────
231	//
232	// Persisted the way `think_log` and `fold_log` are. A RELOAD IS A LOCK: `boot()`
233	// finds the stored identity and returns before `renderAll`, so waiting only for
234	// `__DAIMOND_READY` reads the lock screen — see dev/verify_reopen.mjs.
235	await reboot();
236	// The app restores the chat that was open, which is the refused one above.
237	let after = [];
238	const until = Date.now() + 20000;
239	while (Date.now() < until) {
240	after = await endings();
241	if (after.length) break;
242	await sleep(250);
243	}
244	check('4 A RELOAD STILL SHOWS HOW THE TURN ENDED',
245	after.length === 1 && after[0].notice === true, JSON.stringify(after));
246	await shot(s, 'ending-4-reloaded');
247
248	// ── 4b. The Diamond's own thread, which is the SECOND of three sinks ────────
249	//
250	// A daimon's conversation is durable and it is the surface this app is developed
251	// from, so a steer that quietly did nothing is exactly as invisible there as it
252	// was in a chat — and the reader is the same person. Same line, same rule.
253	await p.evaluate(() => document.querySelector('#diamond-list [data-id]').click());
254	await sleep(900);
255	await p.$('#dview-chat').then(el => el && el.click({ force: true }));
256	await sleep(400);
257	await say('@text I will rewrite lines 43 through 49.');
258	const daimon = await endings();
259	check('4e the DAIMON’S OWN THREAD closes the same way a chat’s does',
260	daimon.length >= 1 && /no tools used/i.test(daimon[daimon.length - 1].text),
261	JSON.stringify(daimon));
262	check('4f and it is furniture there too',
263	daimon.length >= 1 && daimon[daimon.length - 1].notice === false,
264	daimon.length ? ('notice=' + daimon[daimon.length - 1].notice) : '(no line)');
265	await shot(s, 'ending-4b-daimon');
266
267	// ── 4c. A worker's tile, which is the THIRD sink and NOT a thread ───────────
268	//
269	// A worker has no transcript to close: `run.text` is what it SAID, gathered and
270	// handed back to the daimon that dispatched it, and the app's audit written into
271	// that would be delivered to another model as if the worker had written it. So
272	// the ending rides on the TILE, beside the vision note — the other fact the app
273	// states about a run in its own voice.
274	await newChat(s);
275	await p.evaluate(() => window.DaimondPanels.show('agents'));
276	await say('@tool spawn_agent {"name":"scribe","task":"@text I will fix it."}');
277	const untilRun = Date.now() + 45000;
278	let tiles = [];
279	while (Date.now() < untilRun) {
280	tiles = await p.evaluate(() => [...document.querySelectorAll('#panel-agents .acard')].map(c => ({
281	pill: ((c.querySelector('.pill') \|\| {}).textContent \|\| ''),
282	ended: ((c.querySelector('.aended') \|\| {}).textContent \|\| ''),
283	notice: !!c.querySelector('.aended.ended-notice'),
284	})));
285	if (tiles.length && tiles.every(t => ['done', 'error', 'stopped'].includes(t.pill))) break;
286	await sleep(400);
287	}
288	check('4g A WORKER’S TILE SAYS HOW ITS TURN ENDED',
289	tiles.length >= 1 && /no tools used/i.test(tiles[0].ended), JSON.stringify(tiles));
290	check('4h and a worker that simply answered is not badged as a notice either',
291	tiles.length >= 1 && tiles[0].notice === false, JSON.stringify(tiles));
292	await shot(s, 'ending-4c-worker');
293
294	// ── 5. A TOOLLESS TURN DRAWS NOTHING AT ALL ─────────────────────────────────
295	//
296	// `offered === 0` is the pure-chat path, and the app reaches it for real: `cfg.tools`
297	// is read from the stored config (`loadCfg`) and handed to `DaimondApp`'s
298	// `enable_tools`. So the fixture is the product's own setting rather than an
299	// invented one, and what it produces is a genuine `Ended { offered: 0 }`.
300	await p.evaluate(() => {
301	const raw = JSON.parse(localStorage.getItem('daimond-byok') \|\| '{}');
302	raw.tools = false;
303	localStorage.setItem('daimond-byok', JSON.stringify(raw));
304	});
305	await reboot();
306	await newChat(s);
307	await say('@text a plain answer with no tools anywhere near it');
308	const quiet = await endings();
309	check('5a A TOOLLESS TURN DRAWS NOTHING AT ALL',
310	quiet.length === 0, JSON.stringify(quiet));
311	// And the turn really happened, so 5a is not passing because nothing ran.
312	const said = await p.evaluate(() => (document.getElementById('chat-output') \|\| {}).innerText \|\| '');
313	check('5b …and the turn really ran, so 5a is not green for want of a turn',
314	/plain answer\|no tools anywhere/i.test(said) \|\| said.trim().length > 0,
315	said.slice(0, 80));
316	await shot(s, 'ending-5-toolless');
317
318	// ── The console ─────────────────────────────────────────────────────────────
319	const errs = errors(s).filter(e => !/502\|Account service\|429/.test(e));
320	check('6 drawing the ending raised nothing in the console', errs.length === 0,
321	errs.slice(0, 2).join(' \| '));
322
323	await s.close();
324
325	console.log(`\n${ok.length} passed, ${bad.length} failed`);
326	if (BREAK) {
327	console.log(bad.length ? `--break ${BREAK}: reddened ${bad.length} check(s), as it must`
328	: `--break ${BREAK}: CHANGED NOTHING — the check it names is not testing what it says`);
329	process.exit(bad.length ? 0 : 1);
330	}
331	process.exit(bad.length ? 1 : 0);