Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_sweep_used.mjs

13.7 KiB, 1 run

created by r2519314175:713, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_sweep_used.mjs — the same questions, asked of an app that has been used.
2//
3// THIS IS THE HALF THE SUITE DID NOT HAVE, and it is the half that would have
4// caught the defects nobody could see in the code. Every fixture in this tree
5// seeds a world and looks at it. The owner does not: he opens the app, works in
6// it for an afternoon, and the faults he reports are the ones that ACCUMULATE.
7//
8// "The hamburger stops working after modestly long chat use."
9// "When a daimon initiates a new agent worker, the Agents panel should be
10// toggled visible; currently this is not happening." -- and it did happen,
11// exactly once, on the first fan-out that browser ever ran, remembered in
12// `localStorage` under `daimond-agents-revealed`. Nobody could see that from
13// the code: the call was there, on the right line, in the right function. A
14// fresh profile passes it every time, for ever.
15// Four "Daimond Optimiser" tiles, which arrive through a MERGE -- a second
16// device's pair, under minted ids -- and not through a first boot.
17//
18// So this file drives the app into a used state first: several chats, a dozen
19// turns, panels opened and closed and opened again, a second device's records
20// merged in, and a reload standing in for the app being left and come back to.
21// Then it asks the four families of dev/sweepkit.mjs all over again, and asks
22// three behaviours that only mean anything the SECOND time.
23//
24// The rule, stated once so it can be argued with: A CHECK THAT ONLY EVER RUNS
25// AGAINST A NEWLY SEEDED WORLD CANNOT FIND A FAULT THAT ACCUMULATES.
26//
27// node dev/verify_sweep_used.mjs # the run
28// node dev/verify_sweep_used.mjs --quick # fewer turns
29// node dev/verify_sweep_used.mjs --headed # under a real display
30//
31// Needs dev/serve.mjs (DAIMOND_PORT, default 8777) and dev/mockllm.mjs
32// (DAIMOND_MOCK_PORT, default 9099). No gateway.
33//
34// LIMITS. A reload is not an app switch on a phone, where the tab may be
35// discarded and the process killed; a Chromium at 390x844 is not iOS Safari;
36// and "modestly long use" here is a dozen turns of a mock provider in a few
37// minutes, not an afternoon of a real one. What accumulates in wasm heap or in
38// a service worker over hours is out of reach of this file and is not claimed.
39
40import fs from 'node:fs';
41import path from 'node:path';
42import { fileURLToPath } from 'node:url';
43import { open, chat, newChat, signInAs, shot, errors } from './harness.mjs';
44import { audit, showPanels, openPanels, seedNotes } from './sweepkit.mjs';
45
46const HERE = path.dirname(fileURLToPath(import.meta.url));
47const SHOTS = path.join(HERE, 'shots', 'nsweep');
48fs.mkdirSync(SHOTS, { recursive: true });
49
50const QUICK = process.argv.includes('--quick');
51const HEADED = process.argv.includes('--headed');
52const TURNS = QUICK ? 2 : 4; // per chat
53const CHATS = QUICK ? 2 : 3;
54
55const ok = [], bad = [];
56const check = (name, pass, detail) => {
57 (pass ? ok : bad).push(name + (detail ? ' — ' + detail : ''));
58 console.log((pass ? ' ok ' : ' FAIL ') + name + (detail ? ' — ' + detail : ''));
59};
60const say = (fs_) => { for (const f of fs_) console.log(` ${f.family} ${f.what} @ ${f.where} — ${f.detail}`); };
61const hard = (found) => found.filter((f) => f.family !== 'BEHIND');
62
63const s = await open({ name: 'nsweepused', headed: HEADED });
64const p = s.page;
65const findings = [];
66
67const agentsOpen = () => p.evaluate(() => { try { return !!window.DaimondPanels.isOpen('agents'); } catch (e) { return false; } });
68const quiet = async (ms = 60000) => {
69 const t0 = Date.now();
70 while (Date.now() - t0 < ms) {
71 const busy = await p.evaluate(() => { try { return window.DaimondCore.busy(); } catch (e) { return false; } });
72 if (!busy) return true;
73 await p.waitForTimeout(300);
74 }
75 return false;
76};
77
78// ── 1. A control, taken before the app has been used ──────────────────────
79//
80// Everything below is a comparison, and a comparison needs the other end of it.
81// Without this a file that found the fresh app broken would report it as a
82// fault of use.
83console.log('\n1. the app as the rest of the suite meets it\n');
84await seedNotes(p, 20);
85await p.reload({ waitUntil: 'domcontentloaded' });
86await signInAs(s, s.name);
87await p.waitForTimeout(800);
88await showPanels(p, ['social', 'trash']);
89{
90 const fresh = hard(await audit(p));
91 check('fresh: nothing hidden, stranded, covered or doubled', fresh.length === 0, `${fresh.length} finding(s)`);
92 if (fresh.length) { say(fresh); findings.push(...fresh.map((f) => ({ ...f, when: 'fresh' }))); }
93}
94
95// ── 2. Use it ─────────────────────────────────────────────────────────────
96console.log(`\n2. ${CHATS} chats, ${TURNS} turns each, panels opened and closed\n`);
97const PANEL_CYCLE = ['web', 'doc', 'tools', 'graph', 'social', 'trash', 'work', 'spend'];
98for (let c = 0; c < CHATS; c++) {
99 await newChat(s);
100 for (let t = 0; t < TURNS; t++) {
101 await chat(s, `Turn ${t + 1} of chat ${c + 1}: say something back.`, { timeout: 40000 });
102 }
103 // A panel opened and closed between turns, which is what a person does and
104 // what no fixture in this suite does twice.
105 const id = PANEL_CYCLE[c % PANEL_CYCLE.length];
106 await showPanels(p, ['ai', id]);
107 await p.evaluate((id) => { const b = document.querySelector(`[data-close="${id}"]`); if (b) b.click(); }, id);
108 await p.waitForTimeout(300);
109}
110check('the turns landed', (await p.evaluate(() => document.querySelectorAll('#chat-output .msg, #chat-output .chat-msg').length)) > 0,
111 `${await p.evaluate(() => document.querySelectorAll('#chat-output .msg, #chat-output .chat-msg').length)} message node(s)`);
112
113// A SECOND DEVICE'S RECORDS, arriving through the merge. This is how the four
114// tiles were made: a per-device seed flag plus a minted id, so each device made
115// its own pair and the merge -- knowing nothing about names -- kept them all.
116const minted = await p.evaluate(async () => {
117 const m = await import('/pkg/oxedyne_daimond.js');
118 const app = new m.DaimondApp('http://127.0.0.1/v1/chat/completions', '', 'none', 4096, '', true);
119 return [await app.create_diamond('Daimond Help'), await app.create_diamond('Daimond Optimiser')];
120});
121check('a second device\'s default Diamonds were merged in', minted.length === 2, minted.join(', '));
122
123// ── 3. Left, and come back to ─────────────────────────────────────────────
124console.log('\n3. the app left and come back to\n');
125await p.reload({ waitUntil: 'domcontentloaded' });
126await signInAs(s, s.name);
127await p.waitForTimeout(2500);
128await showPanels(p, ['social', 'trash']);
129{
130 const used = await audit(p);
131 const h = hard(used);
132 check('used: nothing hidden, stranded, covered or doubled', h.length === 0, `${h.length} finding(s)`);
133 if (h.length) { say(h); findings.push(...h.map((f) => ({ ...f, when: 'used' }))); }
134 await p.screenshot({ path: path.join(SHOTS, 'used-desktop.png') }).catch(() => {});
135}
136
137// THE RAIL, COUNTED. Two default Diamonds is the offer; four is the defect, and
138// it survives a reload because they are four distinct records. This is asked
139// after the merge and after the boot, which is the only place it can be asked.
140{
141 const names = await p.evaluate(() => {
142 const list = document.getElementById('diamond-list');
143 if (!list) return {};
144 const n = {};
145 for (const tile of list.children) {
146 if (tile.classList.contains('rail-note')) continue;
147 const t = (tile.textContent || '').trim().split('\n')[0].trim();
148 if (t) n[t] = (n[t] || 0) + 1;
149 }
150 return n;
151 });
152 const doubled = Object.keys(names).filter((k) => names[k] > 1);
153 check('no rail tile name is drawn twice after a merge and a reload',
154 doubled.length === 0, doubled.map((k) => `"${k}" x${names[k]}`).join(', '));
155 console.log(' rail: ' + JSON.stringify(names));
156 await p.screenshot({ path: path.join(SHOTS, 'used-rail.png'),
157 clip: await p.evaluate(() => {
158 const r = document.getElementById('panel-rail').getBoundingClientRect();
159 return { x: Math.round(r.left), y: 40, width: Math.round(r.width), height: 420 };
160 }) }).catch(() => {});
161}
162
163// ── 4. The behaviours that only mean anything the SECOND time ─────────────
164console.log('\n4. twice, because once is what the defect looked like\n');
165{
166 await p.evaluate(() => window.DaimondPanels.hide('agents'));
167 await p.waitForTimeout(300);
168 check('the control: the Agents panel is shut', (await agentsOpen()) === false);
169
170 await chat(s, '@tool spawn_agent {"name":"sweep1","task":"@slow 2000"}', { timeout: 45000 });
171 await p.waitForTimeout(1200);
172 check('the first fan-out opens the Agents panel', (await agentsOpen()) === true);
173
174 // Shut by hand with nothing running, which is a tidy-up and not a refusal.
175 await quiet();
176 await p.evaluate(() => { const b = document.querySelector('[data-close="agents"]'); if (b) b.click(); else window.DaimondPanels.hide('agents'); });
177 await p.waitForTimeout(400);
178 check('the control: it is shut again, by hand, with nothing running', (await agentsOpen()) === false);
179
180 await chat(s, '@tool spawn_agent {"name":"sweep2","task":"@slow 2000"}', { timeout: 45000 });
181 await p.waitForTimeout(1200);
182 const twice = await agentsOpen();
183 check('THE SECOND fan-out opens it too', twice === true,
184 twice ? '' : 'it opened once per browser lifetime and never again');
185 if (!twice) findings.push({ family: 'USED', what: 'a panel that reveals itself once per browser',
186 where: '#panel-agents', detail: 'the second fan-out left it shut', when: 'used' });
187 await quiet();
188}
189
190// A PANEL OPENED AND CLOSED TEN TIMES still draws what it holds. Nothing in the
191// suite opens one twice, and a surface that is built on show and not torn down
192// on hide is the shape of half the faults in this file's header.
193{
194 await showPanels(p, ['social', 'trash']);
195 let worst = null;
196 for (let i = 0; i < 10; i++) {
197 await p.evaluate(() => { const b = document.querySelector('[data-close="social"]'); if (b) b.click(); });
198 await p.waitForTimeout(120);
199 await showPanels(p, ['social', 'trash']);
200 }
201 const found = hard(await audit(p));
202 check('the Social panel, opened and closed ten times, still shows its notes',
203 found.length === 0, found.length ? `${found.length} finding(s)` : '');
204 if (found.length) { say(found); findings.push(...found.map((f) => ({ ...f, when: 'ten cycles' }))); }
205 void worst;
206}
207
208// ── 5. On a phone, after all of that ──────────────────────────────────────
209//
210// The hamburger is the owner's own report -- "it stops working after modestly
211// long chat use" -- and it is only a control on a phone. Pressed for real, and
212// the drawer asked whether it actually came out.
213console.log('\n5. a phone, after the same use\n');
214{
215 await p.setViewportSize({ width: 390, height: 844 });
216 await p.waitForTimeout(700);
217 const btn = await p.$('#drawer-btn');
218 const shown = await p.evaluate(() => {
219 const b = document.getElementById('drawer-btn');
220 if (!b) return false;
221 const r = b.getBoundingClientRect();
222 return r.width > 0 && r.height > 0;
223 });
224 check('the hamburger is drawn on a phone', shown);
225 if (shown) {
226 // The point is hit-tested before it is pressed: a button under a stale
227 // overlay takes a forced click and does nothing, which is exactly what
228 // "it stops working" looks like from the outside.
229 const atPoint = await p.evaluate(() => {
230 const b = document.getElementById('drawer-btn');
231 const r = b.getBoundingClientRect();
232 const e = document.elementFromPoint(Math.round(r.left + r.width / 2), Math.round(r.top + r.height / 2));
233 return e ? (b.contains(e) || e.contains(b) ? 'itself' : (e.id || e.className || e.tagName)) : 'nothing';
234 });
235 check('the point you would press belongs to the hamburger', atPoint === 'itself', String(atPoint));
236 await btn.click({ force: true });
237 await p.waitForTimeout(600);
238 const drawer = await p.evaluate(() => {
239 const r = document.getElementById('panel-rail');
240 if (!r) return { out: false, why: 'no rail' };
241 const b = r.getBoundingClientRect();
242 return { out: b.width > 40 && b.left > -40 && b.right > 0, left: Math.round(b.left), width: Math.round(b.width) };
243 });
244 check('the rail drawer comes out after all that use', drawer.out === true, JSON.stringify(drawer));
245 await p.screenshot({ path: path.join(SHOTS, 'used-iphone-drawer.png') }).catch(() => {});
246 }
247 const phone = hard(await audit(p));
248 check('phone, used: nothing hidden, stranded, covered or doubled', phone.length === 0, `${phone.length} finding(s)`);
249 if (phone.length) { say(phone); findings.push(...phone.map((f) => ({ ...f, when: 'phone, used' }))); }
250 await p.screenshot({ path: path.join(SHOTS, 'used-iphone.png') }).catch(() => {});
251}
252
253// ── The report ────────────────────────────────────────────────────────────
254const REPORT = path.join(HERE, 'sweep_used_report.md');
255let md = '# What a person would see, after using it\n\n'
256 + `Written by \`dev/verify_sweep_used.mjs\`, ${new Date().toISOString().slice(0, 10)}.\n\n`;
257md += findings.length
258 ? '| when | where | what | detail |\n|---|---|---|---|\n'
259 + findings.map((f) => `| ${f.when || ''} | \`${f.where}\` | ${f.what} | ${f.detail} |`).join('\n') + '\n'
260 : 'Nothing found.\n';
261fs.writeFileSync(REPORT, md);
262console.log(`\nreport: ${REPORT}`);
263
264const errs = errors(s).filter((e) => !(/status of 502/.test(e) && /\/api\//.test(e)));
265check('nothing threw in the page while it was used', errs.length === 0, errs.slice(0, 2).join(' | '));
266await shot(s, 'nsweep-used-last');
267await s.close();
268
269console.log(`\n${ok.length} passed, ${bad.length} failed`);
270if (bad.length) { for (const b of bad) console.log(' FAIL ' + b); process.exit(1); }