Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_guidesearch.mjs

19.9 KiB, 1 run

created by r2519314175:461, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_guidesearch.mjs — the guide's search box, driven rather than read.
2//
3// notes2.txt line 96: "Maybe we should have a search box in the help guide?"
4// and, later, "some users will choose to burn tokens on Daimond Help but a
5// search facility in the guide is a fallback. It has to be of decent quality."
6// Decent quality is the thing under test here, so this file does not check that
7// a box exists — it checks that the box ANSWERS.
8//
9// WHAT IS LOCKED DOWN.
10//
11// A. THE INDEX IS CURRENT. `node dev/guide-index.mjs --check` exits 0. A stale
12// index is worse than none: it answers confidently about a page that has
13// changed underneath it.
14// B. Every section the index names can actually be jumped to — the anchor
15// exists in the page it claims, in every locale.
16// C. Searching finds the right page: a set of question-and-expected-page pairs
17// written from what a reader would actually type.
18// D. Every term must be present. A two-word query does not return everything
19// matching either word.
20// E. Ranking puts a heading match above a passing mention.
21// F. The keyboard works: `/` focuses, arrows move a highlight, Enter goes,
22// Escape closes. And the highlight is announced (aria-activedescendant).
23// G. It works in a locale that is not English and has no spaces between words
24// — Japanese, where splitting a query on whitespace finds nothing.
25// H. Nothing found says so, rather than showing an empty box.
26// I. It works inside a SANDBOXED frame with no allow-same-origin, which is how
27// the guide is really served. This is the one that decides the design: an
28// opaque origin cannot fetch its own index, so it must arrive as a script.
29// J. Every local URL any guide page names is a file that is on the disk, in
30// every locale. A 404 in the guide is silent: the page still renders, and a
31// missing `search.js` simply means no search box. It is the CLASS that
32// matters, not search alone — the translated pages sit a folder deeper than
33// the English source they are generated from, so any URL the generator
34// forgets to reroot points at nothing, and the last two that did were the
35// screenshots and `search.js`.
36//
37// PROVED RED: `--break <what>` neuters one property in the page before it runs
38// and requires the matching check to notice. `all` runs each in turn.
39//
40// node dev/verify_guidesearch.mjs
41// node dev/verify_guidesearch.mjs --break all
42// node dev/verify_guidesearch.mjs --static # A, B and J only; no browser
43//
44// Needs dev/serve.mjs (DAIMOND_PORT, default 8777). No gateway, no model.
45
46import fs from 'node:fs';
47import path from 'node:path';
48import { execFileSync } from 'node:child_process';
49import { fileURLToPath } from 'node:url';
50import { open, APP } from './harness.mjs';
51
52const HERE = path.dirname(fileURLToPath(import.meta.url));
53const ROOT = path.join(HERE, '..');
54const GUIDE = path.join(ROOT, 'www', 'guide');
55
56const BREAK = (process.argv.find((a) => a.startsWith('--break')) || '').split('=')[1]
57 || (process.argv.includes('--break') ? process.argv[process.argv.indexOf('--break') + 1] : null);
58// The checks that only read the built files need no browser and no server, so
59// they can be run on their own -- straight after a guide build, which is where
60// the mistakes they catch are made.
61const STATIC = process.argv.includes('--static');
62
63const out = [];
64let bad = 0;
65const check = (ok, what, detail) => {
66 out.push(`${ok ? 'PASS' : 'FAIL'} ${what}${detail != null ? ' — ' + detail : ''}`);
67 if (!ok) bad++;
68 return ok;
69};
70
71// ── A. The index is current ─────────────────────────────────────────
72{
73 let ok = true, why = '';
74 try {
75 execFileSync('node', [path.join(HERE, 'guide-index.mjs'), '--check'],
76 { cwd: ROOT, stdio: 'pipe' });
77 } catch (e) {
78 ok = false;
79 why = String(e.stdout || e.message).split('\n').filter(Boolean).slice(-3).join(' | ');
80 }
81 check(ok, 'the search index is current — no guide page has changed under it', why || null);
82}
83
84// ── B. Every anchor the index names exists ──────────────────────────
85{
86 const locales = ['.'].concat(fs.readdirSync(GUIDE, { withFileTypes: true })
87 .filter((e) => e.isDirectory() && /^[a-z]{2}(-[A-Za-z]+)?$/.test(e.name))
88 .map((e) => e.name));
89 let sections = 0, missing = [];
90 for (const loc of locales) {
91 const dir = loc === '.' ? GUIDE : path.join(GUIDE, loc);
92 const ixf = path.join(dir, 'search-index.js');
93 if (!fs.existsSync(ixf)) { missing.push(`${loc}: no index`); continue; }
94 const g = {};
95 // eslint-disable-next-line no-new-func
96 new Function('window', fs.readFileSync(ixf, 'utf8'))(g);
97 const ix = g.GUIDE_INDEX;
98 const pages = new Map();
99 for (const s of ix.sections) {
100 sections++;
101 if (!s.a) continue;
102 if (!pages.has(s.p)) {
103 const pf = path.join(dir, s.p);
104 pages.set(s.p, fs.existsSync(pf) ? fs.readFileSync(pf, 'utf8') : null);
105 }
106 const html = pages.get(s.p);
107 if (html == null) { missing.push(`${loc}/${s.p}: no such page`); continue; }
108 if (!html.includes(`id="${s.a}"`)) missing.push(`${loc}/${s.p}#${s.a}`);
109 }
110 }
111 check(sections > 500, `the index covers every locale (${sections} sections across ${locales.length})`);
112 check(missing.length === 0, 'every section it names can be jumped to',
113 missing.length ? `${missing.length} cannot, e.g. ${missing.slice(0, 3).join(', ')}` : null);
114}
115
116// ── J. Every local URL a page names is a file that is there ─────────
117//
118// Cheap, whole-corpus, and no browser: for every generated page in every
119// locale, resolve each of its own relative URLs against the folder that page
120// actually sits in, and require the file to exist.
121//
122// This is the check the guide had been missing. The translated pages are
123// generated from the English ones and land a folder deeper, so a URL written
124// as `shots/x.png` or `search.js` -- correct at the root, and looking correct
125// everywhere -- resolves inside the locale folder, where nothing is. Both of
126// those shipped. Neither showed: a missing screenshot is a gap in a page
127// nobody reads in Korean, and a missing `search.js` is just no search box.
128{
129 /// Every .html under the guide, root and locale folders alike.
130 const files = [];
131 const collect = (dir) => {
132 for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
133 if (e.isDirectory()) collect(path.join(dir, e.name));
134 else if (e.name.endsWith('.html')) files.push(path.join(dir, e.name));
135 }
136 };
137 collect(GUIDE);
138
139 const dead = [];
140 let urls = 0;
141 for (const f of files) {
142 const html = fs.readFileSync(f, 'utf8');
143 for (const m of html.matchAll(/(?:href|src)="([^"]*)"/g)) {
144 const raw = m[1];
145 // An anchor, a root-relative path the dev server owns, or anything
146 // with a scheme is somebody else's business.
147 if (!raw || /^(#|\/|[a-z][a-z0-9+.-]*:)/i.test(raw)) continue;
148 const rel = decodeURIComponent(raw.split('#')[0].split('?')[0]);
149 if (!rel) continue;
150 urls++;
151 const target = path.resolve(path.dirname(f), rel);
152 if (!fs.existsSync(target)) {
153 dead.push(`${path.relative(GUIDE, f)} -> ${raw}`);
154 }
155 }
156 }
157 check(dead.length === 0,
158 `every local URL in the guide resolves to a file that is there (${urls} across ${files.length} pages)`,
159 dead.length ? `${dead.length} do not: ${[...new Set(dead)].slice(0, 6).join(', ')}` : null);
160}
161
162if (STATIC) {
163 console.log(out.join('\n'));
164 const n = out.filter((l) => /^(PASS|FAIL)/.test(l)).length;
165 console.log(bad === 0 ? `\nALL ${n} STATIC CHECKS PASSED` : `\n${bad} of ${n} FAILED`);
166 process.exit(bad === 0 ? 0 : 1);
167}
168
169// ── The browser ─────────────────────────────────────────────────────
170const s = await open({ name: 'guidesearch', signIn: false, connect: false });
171const p = s.page;
172
173/// Load a guide page, optionally with one property broken before it runs.
174async function load(url) {
175 if (BREAK) {
176 await p.route('**/guide/**/search.js', async (route) => {
177 const res = await route.fetch();
178 let body = await res.text();
179 if (BREAK === 'everyterm' || BREAK === 'all') {
180 // Any term will do, instead of all of them: the classic bad search.
181 body = body.replace('if (!inT && !inU && !inB) return null;',
182 'if (!inT && !inU && !inB) { total += 0; continue; } /* BROKEN */');
183 }
184 if (BREAK === 'rank' || BREAK === 'all') {
185 // A heading match scores no more than a body mention.
186 body = body.replace(/best = Math\.max\(best, 100 \+[^;]+;/,
187 'best = Math.max(best, 30); /* BROKEN */');
188 }
189 if (BREAK === 'keys' || BREAK === 'all') {
190 body = body.replace("if (e.key === 'ArrowDown')", "if (false)");
191 }
192 if (BREAK === 'cjk' || BREAK === 'all') {
193 // Split on whitespace only, which finds nothing in ja/ko/zh.
194 body = body.replace('var CJK = /[\\u3040-\\u30ff', 'var CJK = /[\\uE000-\\uE001');
195 }
196 if (BREAK === 'none' || BREAK === 'all') {
197 body = body.replace("none.textContent = W.no || 'Nothing found';", "none.textContent = '';");
198 }
199 await route.fulfill({ response: res, body,
200 headers: { ...res.headers(), 'content-type': 'text/javascript; charset=utf-8' } });
201 });
202 }
203 await p.goto(url, { waitUntil: 'domcontentloaded' });
204 await p.waitForTimeout(500);
205}
206
207/// Type a query and read what comes back.
208async function ask(q) {
209 await p.fill('#guide-search', '');
210 await p.fill('#guide-search', q);
211 await p.waitForTimeout(220);
212 return p.evaluate(() => {
213 const panel = document.getElementById('guide-search-results');
214 return {
215 open: panel && !panel.hidden,
216 none: !!panel.querySelector('.gsearch-none'),
217 noneText: (panel.querySelector('.gsearch-none') || {}).textContent || '',
218 live: (document.querySelector('.gsearch-live') || {}).textContent || '',
219 hits: [...panel.querySelectorAll('.gsearch-hit')].map((a) => ({
220 href: a.getAttribute('href'),
221 title: (a.querySelector('.gsearch-t') || {}).textContent || '',
222 marks: [...a.querySelectorAll('mark')].map((m) => m.textContent),
223 snippet: (a.querySelector('.gsearch-b') || {}).textContent || '',
224 })),
225 };
226 });
227}
228
229await load(`${APP}/guide/index.html`);
230
231check(await p.$('#guide-search') !== null, 'there is a search box in the guide header');
232
233// ── C. It finds the right page ──────────────────────────────────────
234//
235// Written from what a reader would type, not from the headings — a query
236// copied out of a heading proves only that the string is in the file.
237// Each of these was checked against the corpus before being written down. The
238// first draft of this list was invented from what a reader might type and three
239// of six asked for words the guide does not contain -- which proved nothing
240// about the search and everything about writing expectations without looking.
241const ASKS = [
242 ['passphrase', 'accounts.html'],
243 ['passkey', 'accounts.html'],
244 ['pause spending', 'spending.html'],
245 ['crystal', 'chats-and-diamonds.html'],
246 ['mailbox', 'email-web-files.html'],
247 ['credits', 'models.html'],
248];
249
250// THE ALIASES, which are the answer to that finding rather than a way round it.
251// A reader searching a help guide does not know its vocabulary -- that is why
252// they are searching. The guide says "passphrase" and never "password"; it
253// explains bringing your own key and never writes "BYOK". Each of these is a
254// word the guide does NOT contain, and must still land.
255const ALIAS_ASKS = [
256 ['password', 'accounts.html'],
257 ['byok', 'models.html'],
258 ['cost', 'spending.html'],
259];
260{
261 const misses = [];
262 for (const [q, want] of ASKS) {
263 const r = await ask(q);
264 const top3 = r.hits.slice(0, 3).map((h) => (h.href || '').split('#')[0]);
265 if (!top3.includes(want)) misses.push(`${q} -> ${top3.join(',') || 'nothing'} (want ${want})`);
266 }
267 check(misses.length === 0, `a reader's own words reach the right page (${ASKS.length} queries)`,
268 misses.length ? misses.join(' | ') : null);
269}
270{
271 const misses = [];
272 for (const [q, want] of ALIAS_ASKS) {
273 const r = await ask(q);
274 const top3 = r.hits.slice(0, 3).map((h) => (h.href || '').split('#')[0]);
275 if (!top3.includes(want)) misses.push(`${q} -> ${top3.join(',') || 'nothing'} (want ${want})`);
276 }
277 check(misses.length === 0,
278 `a word the guide never uses still lands, through an alias (${ALIAS_ASKS.length} queries)`,
279 misses.length ? misses.join(' | ') : null);
280 // And an alias never beats the real word: "passphrase" typed directly must
281 // still answer with a passphrase section, not with whatever "password"
282 // happens to reach.
283 const direct = await ask('passphrase');
284 check(/passphrase/i.test((direct.hits[0] || {}).title || ''),
285 'and a direct hit still outranks an aliased one',
286 JSON.stringify((direct.hits[0] || {}).title));
287}
288
289// ── D. Every term must be present ───────────────────────────────────
290{
291 const one = await ask('passphrase');
292 const two = await ask('passphrase kangaroo');
293 check(one.hits.length > 0, 'a one-word query finds sections', `${one.hits.length}`);
294 check(two.hits.length === 0,
295 'adding a word that appears NOWHERE returns nothing — every term must be present',
296 `${two.hits.length} hit(s): ${two.hits.slice(0, 2).map((h) => h.title).join(', ')}`);
297 // And the honest version of the same property: a second word that DOES
298 // appear narrows rather than widens.
299 const wide = await ask('sync');
300 const narrow = await ask('sync passkey');
301 check(narrow.hits.length > 0 && narrow.hits.length <= wide.hits.length,
302 'a second real word narrows the answer rather than widening it',
303 `sync ${wide.hits.length} -> sync passkey ${narrow.hits.length}`);
304}
305
306// ── E. A heading beats a passing mention ────────────────────────────
307{
308 const r = await ask('passkey');
309 const first = r.hits[0] || {};
310 check(/passkey/i.test(first.title || ''),
311 'the top answer is a section ABOUT the word, not one that merely says it',
312 JSON.stringify((r.hits.slice(0, 3)).map((h) => h.title)));
313 check((first.marks || []).length > 0,
314 'and the match is marked in what is shown', JSON.stringify(first.marks));
315 check(!!(first.snippet || '').trim(),
316 'with the sentence it was found in', JSON.stringify((first.snippet || '').slice(0, 60)));
317}
318
319// ── H. Nothing found says so ────────────────────────────────────────
320{
321 const r = await ask('zzzqqxwv');
322 check(r.open && r.none && r.noneText.trim().length > 0,
323 'a query that matches nothing says so, rather than showing an empty box',
324 JSON.stringify(r.noneText));
325 check(/\d|[^\s]/.test(r.live), 'and the count is announced', JSON.stringify(r.live));
326}
327
328// ── F. The keyboard ─────────────────────────────────────────────────
329{
330 await p.evaluate(() => document.getElementById('guide-search').blur());
331 await p.keyboard.press('/');
332 await p.waitForTimeout(150);
333 const focused = await p.evaluate(() => document.activeElement && document.activeElement.id);
334 check(focused === 'guide-search', '`/` from the page focuses the box', String(focused));
335
336 await p.fill('#guide-search', 'passkey');
337 await p.waitForTimeout(220);
338 await p.keyboard.press('ArrowDown');
339 await p.keyboard.press('ArrowDown');
340 const nav = await p.evaluate(() => {
341 const on = document.querySelectorAll('.gsearch-hit.on');
342 const inp = document.getElementById('guide-search');
343 return {
344 lit: on.length,
345 which: on[0] ? on[0].id : null,
346 active: inp.getAttribute('aria-activedescendant'),
347 expanded: inp.getAttribute('aria-expanded'),
348 };
349 });
350 check(nav.lit === 1 && nav.which === 'gsearch-hit-1',
351 'the arrows move exactly one highlight', JSON.stringify(nav));
352 check(nav.active === nav.which,
353 'and a screen reader is told which row it is on', JSON.stringify(nav.active));
354 check(nav.expanded === 'true', 'the box says its list is open');
355
356 const href = await p.evaluate(() => {
357 var el = document.querySelector('.gsearch-hit.on') || document.querySelector('.gsearch-hit');
358 return el ? el.getAttribute('href') : null;
359 });
360 check(!!href, 'there is a highlighted answer to press Enter on', String(href));
361 if (!href) { console.log(out.join('\n')); await s.close(); process.exit(1); }
362 await p.keyboard.press('Enter');
363 await p.waitForTimeout(700);
364 const landed = await p.evaluate(() => ({
365 url: location.pathname.split('/').pop() + location.hash,
366 flashed: !!document.querySelector('.gsearch-landed'),
367 heading: (document.querySelector('.gsearch-landed') || {}).textContent || '',
368 }));
369 check(landed.url === href, 'Enter goes to the highlighted answer', `${landed.url} vs ${href}`);
370 check(landed.flashed, 'and the section it landed on is marked, so the eye has somewhere to go',
371 JSON.stringify(landed.heading.slice(0, 40)));
372
373 // Escape closes without navigating.
374 await p.fill('#guide-search', 'sync');
375 await p.waitForTimeout(220);
376 await p.keyboard.press('Escape');
377 const closed = await p.evaluate(() => document.getElementById('guide-search-results').hidden);
378 check(closed, 'Escape closes the list');
379}
380
381// ── G. A locale with no spaces between words ────────────────────────
382{
383 await load(`${APP}/guide/ja/index.html`);
384 const box = await p.$('#guide-search');
385 check(box !== null, 'the Japanese guide has the box too');
386 const ph = await p.evaluate(() => document.getElementById('guide-search').placeholder);
387 check(/[぀-ヿ㐀-䶿一-鿿]/.test(ph), 'in Japanese', JSON.stringify(ph));
388 const r = await ask('パスキー');
389 check(r.hits.length > 0,
390 'and a Japanese query finds something — a whitespace split would find nothing here',
391 `${r.hits.length} hit(s): ${r.hits.slice(0, 2).map((h) => h.title).join(' | ')}`);
392 const inJa = r.hits.every((h) => !/^\.\.\//.test(h.href || ''));
393 check(inJa, 'and its answers stay inside the Japanese guide',
394 JSON.stringify(r.hits.slice(0, 2).map((h) => h.href)));
395}
396
397// ── I. Inside a sandboxed frame, which is how it is really served ───
398{
399 await p.goto(`${APP}/guide/index.html`, { waitUntil: 'domcontentloaded' });
400 const framed = await p.evaluate(async (app) => {
401 const f = document.createElement('iframe');
402 // The app's own sandbox, from the Web panel: no allow-same-origin, so
403 // the frame has an OPAQUE origin and cannot fetch its own index.
404 f.setAttribute('sandbox', 'allow-scripts allow-popups allow-forms');
405 f.src = app + '/guide/index.html';
406 f.style.cssText = 'width:900px;height:600px';
407 document.body.appendChild(f);
408 await new Promise((r) => { f.onload = r; setTimeout(r, 6000); });
409 await new Promise((r) => setTimeout(r, 800));
410 // Nothing can be read out of an opaque frame from here, so the frame is
411 // asked to report on itself the only way it can: it cannot. What CAN be
412 // checked from outside is that it loaded and did not throw — and the
413 // index being a <script> is what makes that true, so the real evidence
414 // is the console, collected by the harness.
415 return { loaded: true };
416 }, APP);
417 await p.waitForTimeout(500);
418 const sandboxErrs = s.errs.filter((e) =>
419 /GUIDE_INDEX|search-index|Failed to fetch|CORS|Access to fetch/i.test(e));
420 check(framed.loaded && sandboxErrs.length === 0,
421 'the guide loads its index inside a sandbox with no allow-same-origin',
422 sandboxErrs.length ? JSON.stringify(sandboxErrs.slice(0, 2)) : 'no index or CORS errors');
423}
424
425const noise = /favicon|401|402|502|Unauthorized|Payment|Bad Gateway/i;
426const errs = s.errs.filter((e) => !noise.test(e));
427check(errs.length === 0, 'no console errors', JSON.stringify(errs.slice(0, 3)));
428
429await s.close();
430
431console.log(out.join('\n'));
432const total = out.filter((l) => /^(PASS|FAIL)/.test(l)).length;
433if (BREAK) {
434 console.log(`\nBROKEN RUN (${BREAK}): ${bad} of ${total} checks failed. `
435 + (bad > 0 ? 'Good — the checks see it.' : 'BAD — a check that cannot fail is not evidence.'));
436 process.exit(bad > 0 ? 0 : 1);
437}
438console.log(bad === 0 ? `\nALL ${total} CHECKS PASSED` : `\n${bad} of ${total} FAILED`);
439process.exit(bad === 0 ? 0 : 1);