Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_webwatch.mjs

12.3 KiB, 1 run

created by r2519314175:809, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_webwatch.mjs — a slow human's approval still lands in the panel.
2//
3// Opening an unapproved site puts a question to a person, and the panel stops
4// waiting at 45s so the model is not left hanging on a spinner. The EXTENSION
5// does not stop: the grant window is still on screen, and when the user finally
6// clicks Allow the tab really opens and really is driven. The reply that comes
7// back after our wait used to be dropped, so the user watched Daimond drive a
8// page the panel said was never approved.
9//
10// This drives the real DaimondWeb through a FAKE extension bridge: a stubbed
11// chrome.runtime that holds every `open` message until the test answers it, so
12// "the human answered late" is an ordinary function call. Five cases:
13//
14// A a late approval adopts the page and says so;
15// B a late refusal replaces the stale note with the real reason;
16// C a page opened in the meantime is never stomped by a late answer;
17// D a retry while the question is up sends no SECOND open (which would be a
18// second grant window), and collects the answer as its own tool result;
19// E after the late window there is no watcher left: the reply changes nothing.
20//
21// Needs dev/serve.mjs (DAIMOND_PORT, default 8777). No gateway, no model, no extension.
22//
23// node dev/verify_webwatch.mjs # all cases
24// WEBWATCH_CASES=AB node dev/verify_webwatch.mjs
25import { open, shot, errors } from './harness.mjs';
26
27const CASES = (process.env.WEBWATCH_CASES || 'ABCDE').toUpperCase();
28const ok = [], bad = [];
29const check = (name, pass, detail) => {
30 (pass ? ok : bad).push(name);
31 console.log((pass ? ' ok ' : ' FAIL ') + name + (detail ? ' — ' + detail : ''));
32};
33
34const s = await open({ name: 'webwatch' });
35const page = s.page;
36
37// ── The fake hands ──────────────────────────────────────────────────
38// Everything but `open` answers at once; `open` is held, and the test decides
39// when — and whether — the human said yes.
40const installed = await page.evaluate(() => {
41 try {
42 if (!window.chrome) window.chrome = {};
43 window.__ext = { sent: [], pend: [] };
44 window.chrome.runtime = {
45 id: 'mockhands',
46 lastError: undefined,
47 sendMessage: function (id, msg, cb) {
48 window.__ext.sent.push(msg);
49 if (msg.cmd === 'open') { window.__ext.pend.push({ msg: msg, cb: cb }); return; }
50 // status is polled by the mirror once a page is adopted; answering
51 // with nothing to say leaves the panel's own state alone.
52 setTimeout(function () { cb({ ok: true, mode: 'agent' }); }, 0);
53 },
54 };
55 document.documentElement.dataset.daimondHands = 'mockhands';
56 window.dispatchEvent(new CustomEvent('daimond-hands', { detail: { id: 'mockhands' } }));
57 return true;
58 } catch (e) { return String(e); }
59});
60check('the fake extension bridge installs', installed === true, String(installed));
61await page.waitForTimeout(300);
62check('the panel adopts it as its driver', await page.evaluate(() => window.DaimondWeb.hasHands()));
63
64// The rule is minutes long by design; a test cannot spend minutes proving it.
65// Without the hook (an unfixed build) the real waits are used, which is slow but
66// still measures the right thing.
67const hook = await page.evaluate(() => !!window.DaimondWeb._setWaitsForTest);
68console.log(hook ? ' (timing hook present)' : ' (NO timing hook — using the real 45s/4min waits)');
69const OPENW = hook ? 1200 : 45000;
70const LATEW = hook ? 20000 : 240000;
71if (hook) await page.evaluate((w) => window.DaimondWeb._setWaitsForTest(w), { open: OPENW, late: LATEW });
72
73const clear = () => page.evaluate(() => { window.__ext.sent = []; window.__ext.pend = []; });
74const noteText = () => page.evaluate(() => (document.getElementById('web-note') || {}).innerText || '');
75const status = () => page.evaluate(() => window.DaimondWeb.status());
76const opens = () => page.evaluate(() => window.__ext.sent.filter((m) => m.cmd === 'open').map((m) => m.url));
77/// Answer a held `open` as the extension would. `match` picks which one, when
78/// more than one question is outstanding; the oldest otherwise.
79const answer = (v, match) => page.evaluate(({ r, m }) => {
80 const i = m ? window.__ext.pend.findIndex((p) => String(p.msg.url).includes(m)) : 0;
81 if (i < 0 || !window.__ext.pend[i]) return false;
82 window.__ext.pend.splice(i, 1)[0].cb(r);
83 return true;
84}, { r: v, m: match || '' });
85
86/// Start an open the way a tool call does — without awaiting it, because it
87/// blocks until a human answers. What it finally settles to is stashed for us.
88const ask = (url, tag) => page.evaluate(({ url, tag }) => {
89 window.__res = window.__res || {};
90 window.__res[tag] = { done: false };
91 window.DaimondWeb.open(url).then(
92 (r) => { window.__res[tag] = { done: true, ok: true, r: r }; },
93 (e) => { window.__res[tag] = { done: true, ok: false, err: String((e && e.message) || e) }; });
94}, { url, tag });
95
96const result = (tag) => page.evaluate((t) => (window.__res || {})[t] || { done: false }, tag);
97
98/// Wait for a tool call to settle — the panel's own wait plus slack, never a
99/// bare sleep, so a loaded machine does not read as a failure.
100async function settled(tag, extra = 6000) {
101 const until = Date.now() + OPENW + extra;
102 for (;;) {
103 const r = await result(tag);
104 if (r.done) return r;
105 if (Date.now() > until) return r;
106 await page.waitForTimeout(150);
107 }
108}
109
110// ── A: a late approval adopts the page ──────────────────────────────
111if (CASES.includes('A')) {
112 console.log('\nA — the user approves after the panel has given up');
113 await clear();
114 await ask('https://slow.example/one', 'A');
115 const timedOut = await settled('A');
116 check('the wait ends, and the model is told to try again',
117 timedOut.done && !timedOut.ok && /try web_open|approve this site/i.test(timedOut.err || ''), timedOut.err);
118 const before = await noteText();
119 check('the panel says the site was not approved', /was not approved/i.test(before), before.replace(/\n/g, ' / '));
120
121 check('the answer arrives late', await answer({ ok: true, url: 'https://slow.example/one', title: 'Slow', mode: 'agent' }));
122 await page.waitForTimeout(900);
123 const st = await status();
124 const after = await noteText();
125 check('a late approval adopts the page', st.driver === 'ext' && /slow\.example/.test(st.url) && st.mode === 'agent',
126 JSON.stringify(st));
127 check('the panel says the site was approved after all', /was approved/i.test(after), after.replace(/\n/g, ' / '));
128 check('and it no longer claims the site was refused', !/was not approved/i.test(after), after.replace(/\n/g, ' / '));
129 check('the driving note is there too', /driving/i.test(after), after.replace(/\n/g, ' / '));
130 // Adoption is not just words: the panel starts mirroring the tab, which is
131 // the one behaviour a discarded reply cannot fake.
132 const polls = await page.evaluate(() => window.__ext.sent.filter((m) => m.cmd === 'status').length);
133 check('the panel starts following the adopted tab', polls > 0, polls + ' status polls');
134 await shot(s, 'webwatch-late-approved');
135}
136
137// ── B: a late refusal replaces the stale note ───────────────────────
138if (CASES.includes('B')) {
139 console.log('\nB — the user refuses after the panel has given up');
140 await page.evaluate(() => window.DaimondWeb.close());
141 await clear();
142 await ask('https://nope.example/two', 'B');
143 const timedOut = await settled('B');
144 check('the wait ends first', timedOut.done && !timedOut.ok, JSON.stringify(timedOut));
145 const before = await noteText();
146
147 check('the refusal arrives late', await answer({ ok: false,
148 error: 'The user declined: Daimond may not operate nope.example. Do not ask for it again.' }));
149 await page.waitForTimeout(700);
150 const after = await noteText();
151 check('a late refusal rewrites the stale note', after !== before,
152 'before: ' + before.replace(/\n/g, ' / '));
153 check('it says the answer came after the wait', /after Daimond had/i.test(after), after.replace(/\n/g, ' / '));
154 check('it still says the site was not approved', /was not approved/i.test(after), '');
155 check('and nothing was adopted', !/driving/i.test(after), after.replace(/\n/g, ' / '));
156}
157
158// ── C: a page opened since must win ─────────────────────────────────
159if (CASES.includes('C')) {
160 console.log('\nC — a page opened in the meantime is not stomped');
161 await page.evaluate(() => window.DaimondWeb.close());
162 await clear();
163 await ask('https://stomp.example/three', 'C');
164 const timedOut = await settled('C');
165 check('the wait ends first', timedOut.done && !timedOut.ok, JSON.stringify(timedOut));
166
167 // The user asks for somewhere else, and that site is already approved, so it
168 // opens at once. This is now the page the panel is showing.
169 await ask('https://fresh.example/', 'C2');
170 await page.waitForTimeout(200);
171 await answer({ ok: true, url: 'https://fresh.example/', title: 'Fresh', mode: 'agent' }, 'fresh.example');
172 const fresh = await settled('C2', 2000);
173 check('a new page opens while the old question is still out', fresh.done && fresh.ok === true, JSON.stringify(fresh));
174
175 check('the old approval arrives late', await answer({ ok: true, url: 'https://stomp.example/three', title: 'Stomp', mode: 'agent' }, 'stomp.example'));
176 await page.waitForTimeout(900);
177 const st = await status();
178 check('the late answer does not stomp the newer page', /fresh\.example/.test(st.url) && !/stomp/.test(st.url),
179 JSON.stringify(st));
180 const after = await noteText();
181 check('and the panel still names the newer page', /fresh\.example/.test(after) && !/stomp\.example/.test(after),
182 after.replace(/\n/g, ' / '));
183}
184
185// ── D: the retry path ───────────────────────────────────────────────
186if (CASES.includes('D')) {
187 console.log('\nD — the model retries while the question is still on screen');
188 await page.evaluate(() => window.DaimondWeb.close());
189 await clear();
190 await ask('https://retry.example/four', 'D1');
191 const first = await settled('D1');
192 check('the first call gives up', first.done && !first.ok, JSON.stringify(first));
193
194 await ask('https://retry.example/four', 'D2');
195 await page.waitForTimeout(600);
196 const sent = await opens();
197 check('the retry sends no second open — no second grant window', sent.length === 1, JSON.stringify(sent));
198
199 check('the approval arrives late', await answer({ ok: true, url: 'https://retry.example/four', title: 'Retry', mode: 'agent' }));
200 const second = await settled('D2', 2000);
201 check('the retry collects the answer as its own result', second.done && second.ok === true, JSON.stringify(second));
202 const st = await status();
203 check('and the panel is driving the page', st.driver === 'ext' && /retry\.example/.test(st.url), JSON.stringify(st));
204}
205
206// ── E: the late window really does close ────────────────────────────
207if (CASES.includes('E')) {
208 if (!hook) {
209 console.log('\nE — skipped: without the timing hook this case waits out the whole late window');
210 } else {
211 console.log('\nE — after the late window, nothing is listening');
212 await page.evaluate(() => window.DaimondWeb.close());
213 await page.evaluate((w) => window.DaimondWeb._setWaitsForTest(w), { open: 600, late: 1200 });
214 await clear();
215 await ask('https://gone.example/five', 'E');
216 for (let i = 0; i < 40; i++) { if ((await result('E')).done) break; await page.waitForTimeout(150); }
217 const before = await noteText();
218 await page.waitForTimeout(2600); // past 600 + 1200, with slack
219 check('a reply after the late window is still delivered by the browser',
220 await answer({ ok: true, url: 'https://gone.example/five', title: 'Gone', mode: 'agent' }));
221 await page.waitForTimeout(700);
222 const after = await noteText();
223 check('it adopts nothing', !/was approved/i.test(after) && !/driving/i.test(after), after.replace(/\n/g, ' / '));
224 check('and the panel still says what it said', /was not approved/i.test(after), after.replace(/\n/g, ' / '));
225 }
226}
227
228await page.evaluate(() => window.DaimondWeb.close()).catch(() => {});
229await shot(s, 'webwatch-end');
230
231const errs = errors(s).filter((e) => !/Failed to load resource.*\b(401|402|404|502|503)\b/.test(e));
232check('the app logged no console errors', errs.length === 0, errs.join(' | '));
233
234console.log('\n' + ok.length + ' ok, ' + bad.length + ' failed');
235await s.close();
236process.exit(bad.length ? 1 : 0);