Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_errnames.mjs

10.0 KiB, 1 run

created by r2519314175:399, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_errnames.mjs — when a form refuses, it says WHICH box is wrong.
2//
3// The fourth property search, after reversible, escapable and focus. A form that
4// answers "that didn't work" has told the user nothing they did not already
5// know; a form that names a box they did not touch is worse, because they will
6// go and change it. The property is narrow and checkable: submit a form with
7// exactly ONE thing wrong, and the message that comes back has to mention that
8// thing.
9//
10// It is a search rather than a script because the interesting failures are the
11// ones nobody wrote a case for. The BYOK form validated that a base URL was
12// PRESENT but never that it was a URL, so "http:/typo" sailed past, the model
13// list could not load from it, and the refusal that finally came back read
14// "Choose a model, or wait a moment for the list to load" — naming the one box
15// the user had filled in correctly.
16//
17// node dev/verify_errnames.mjs
18// node dev/verify_errnames.mjs byok-badurl
19//
20// Needs dev/serve.mjs (DAIMOND_PORT, default 8777). No gateway.
21
22import fs from 'node:fs';
23import { open, scratch, MOCK } from './harness.mjs';
24
25let failures = 0, skips = 0;
26const skipped = [];
27const check = (cond, msg, detail) => {
28 console.log((cond ? ' ok ' : ' FAIL ') + msg + (detail != null ? ' — ' + detail : ''));
29 if (!cond) failures++;
30};
31const skip = (name, why) => {
32 console.log(' SKIP ' + name + ' — ' + why);
33 skipped.push(name + ': ' + why);
34 skips++;
35};
36
37async function press(page, sel) {
38 await page.waitForSelector(sel, { timeout: 10000 });
39 await page.evaluate((s) => { const e = document.querySelector(s); if (e) e.click(); }, sel);
40 await page.waitForTimeout(350);
41}
42
43/// Press the control inside `rootSel` whose text is exactly `text`.
44async function pressLabel(page, rootSel, text) {
45 await page.waitForSelector(rootSel, { timeout: 10000 });
46 const hit = await page.evaluate(({ rootSel, text }) => {
47 const root = document.querySelector(rootSel);
48 if (!root) return false;
49 const b = [...root.querySelectorAll('button')].find((x) => (x.textContent || '').trim() === text);
50 if (!b) return false;
51 b.click();
52 return true;
53 }, { rootSel, text });
54 if (!hit) throw new Error(`no control labelled "${text}" in ${rootSel}`);
55 await page.waitForTimeout(450);
56}
57
58/// Set a field the way a person does — the app listens for `input`, and a value
59/// assigned without one is a value it never hears about.
60async function fill(page, sel, value) {
61 await page.evaluate(({ sel, value }) => {
62 const el = document.querySelector(sel);
63 if (!el) throw new Error('no field ' + sel);
64 if (el.tagName === 'SELECT') { el.value = value; el.dispatchEvent(new Event('change', { bubbles: true })); return; }
65 el.value = value;
66 el.dispatchEvent(new Event('input', { bubbles: true }));
67 el.dispatchEvent(new Event('change', { bubbles: true }));
68 }, { sel, value });
69 await page.waitForTimeout(120);
70}
71
72// ── One wrong box at a time ─────────────────────────────────────────────
73//
74// `field` is the box deliberately made wrong; `wants` is what the message has
75// to mention for the user to know where to look. `settle` is for forms that go
76// and ask a provider something before they answer.
77const CASES = [
78 {
79 id: 'create-noname',
80 name: 'Create account, no name',
81 field: 'the name',
82 open: { connect: false, signIn: false },
83 reach: async () => {},
84 act: async (page) => {
85 await fill(page, '#id-name', '');
86 // Tick it the way a person does. Setting `.checked` alone leaves the
87 // create button disabled -- it is enabled by the `change` the tick
88 // fires -- and a disabled button swallows the click, so the form looks
89 // silent when it was never asked anything.
90 await page.evaluate(() => {
91 const w = document.getElementById('id-wrote');
92 if (w && !w.checked) { w.checked = true; w.dispatchEvent(new Event('change', { bubbles: true })); }
93 });
94 await page.evaluate(() => document.getElementById('id-primary').click());
95 },
96 err: '#id-error',
97 wants: /\bname\b/i,
98 },
99 {
100 id: 'create-shortpass',
101 name: 'Create account, passphrase too short',
102 field: 'the passphrase',
103 open: { connect: false, signIn: false },
104 reach: async (page) => { await press(page, '#id-choose'); }, // type my own
105 act: async (page) => {
106 await fill(page, '#id-name', 'Someone');
107 await fill(page, '#id-pass', 'abc');
108 await fill(page, '#id-pass2', 'abc');
109 await page.evaluate(() => document.getElementById('id-primary').click());
110 },
111 err: '#id-error',
112 wants: /passphrase/i,
113 },
114 {
115 id: 'create-mismatch',
116 name: 'Create account, confirmation does not match',
117 field: 'the passphrase',
118 open: { connect: false, signIn: false },
119 reach: async (page) => { await press(page, '#id-choose'); },
120 act: async (page) => {
121 await fill(page, '#id-name', 'Someone');
122 await fill(page, '#id-pass', 'correcthorsebattery');
123 await fill(page, '#id-pass2', 'correcthorsebatteryX');
124 await page.evaluate(() => document.getElementById('id-primary').click());
125 },
126 err: '#id-error',
127 wants: /passphrase/i,
128 },
129 {
130 id: 'byok-noprovider',
131 name: 'Models, no provider chosen',
132 field: 'the provider',
133 open: { connect: false },
134 reach: openByok,
135 act: async (page) => { await pressLabel(page, '#admin-models', 'Save & start'); },
136 err: '#byok-note',
137 wants: /provider/i,
138 },
139 {
140 id: 'byok-nourl',
141 name: 'Models, custom provider with no base URL',
142 field: 'the base URL',
143 open: { connect: false },
144 reach: openByok,
145 act: async (page) => {
146 await fill(page, '#cfg-provider', 'custom');
147 await fill(page, '#cfg-base-url', '');
148 await pressLabel(page, '#admin-models', 'Save & start');
149 },
150 err: '#byok-note',
151 wants: /\burl\b|address/i,
152 },
153 {
154 id: 'byok-nokey',
155 name: 'Models, no API key',
156 field: 'the API key',
157 open: { connect: false },
158 reach: openByok,
159 act: async (page) => {
160 await fill(page, '#cfg-provider', 'custom');
161 await fill(page, '#cfg-base-url', MOCK);
162 await fill(page, '#cfg-api-key', '');
163 await pressLabel(page, '#admin-models', 'Save & start');
164 },
165 err: '#byok-note',
166 wants: /\bkey\b/i,
167 },
168 {
169 // The one the exploration found. Everything else is filled in correctly.
170 id: 'byok-badurl',
171 name: 'Models, base URL that is not a URL',
172 field: 'the base URL',
173 open: { connect: false },
174 reach: openByok,
175 act: async (page) => {
176 await fill(page, '#cfg-provider', 'custom');
177 await fill(page, '#cfg-base-url', 'not a url');
178 await fill(page, '#cfg-api-key', 'some-key');
179 await page.waitForTimeout(1200); // it goes off to list models first
180 await pressLabel(page, '#admin-models', 'Save & start');
181 await page.waitForTimeout(600);
182 },
183 err: '#byok-note',
184 wants: /\burl\b|address/i,
185 },
186 {
187 id: 'mail-bademail',
188 name: 'Add a mailbox, address that is not one',
189 field: 'the email address',
190 open: { connect: false },
191 reach: async (page) => {
192 await page.evaluate(() => window.DaimondPanels && DaimondPanels.show('mail'));
193 await page.waitForTimeout(300);
194 await press(page, '#panel-mail [data-act="mail-add"]');
195 await page.waitForSelector('#admin-form .dlg-input', { timeout: 8000 });
196 },
197 act: async (page) => {
198 await fill(page, '#admin-form .dlg-input', 'definitely-not-an-address');
199 await pressLabel(page, '#admin-form', 'Add and sync');
200 },
201 err: '#admin-form .dlg-err',
202 wants: /e-?mail|address/i,
203 },
204 {
205 id: 'pass-wrongcurrent',
206 name: 'Change passphrase, wrong current one',
207 field: 'the current passphrase',
208 open: { connect: false },
209 reach: async (page) => {
210 await press(page, '#user-row');
211 await pressLabel(page, '#admin-home', 'Change passphrase…');
212 await page.waitForSelector('.dlg-card', { timeout: 8000 });
213 },
214 act: async (page) => {
215 await fill(page, '.dlg .dlg-input', 'not-the-passphrase');
216 await page.evaluate(() => { const i = document.querySelector('.dlg .dlg-input'); if (i) i.focus(); });
217 await page.keyboard.type('x');
218 await page.keyboard.press('Backspace');
219 await pressLabel(page, '.modal.dlg', 'Next');
220 await page.waitForTimeout(900); // deriving a key takes a moment
221 },
222 err: '.modal.dlg .dlg-err',
223 wants: /passphrase/i,
224 },
225];
226
227/// Open the Models view with the add-a-provider form showing. Several cases
228/// start here, so the walk lives once.
229async function openByok(page) {
230 await press(page, '#astat-model');
231 await page.waitForSelector('#admin-models', { timeout: 8000 });
232 await page.evaluate(() => { const f = document.getElementById('byok-form'); if (f) f.style.display = ''; });
233 await page.waitForTimeout(250);
234}
235
236const only = process.argv[2] || '';
237
238for (const c of CASES) {
239 if (only && c.id !== only) continue;
240 console.log(`\n── ${c.name}`);
241 const dir = scratch('pw', 'err-' + Math.random().toString(36).slice(2, 10));
242 let s = null;
243 try {
244 s = await open({ ...c.open, name: 'errnames', profile: dir });
245 await c.reach(s.page);
246 await s.page.waitForTimeout(300);
247 await c.act(s.page);
248 await s.page.waitForTimeout(500);
249 const msg = (await s.page.evaluate((sel) => {
250 const el = document.querySelector(sel);
251 return el ? (el.textContent || '').trim() : '';
252 }, c.err)) || '';
253 if (!msg) {
254 check(false, `refusing names ${c.field}`, 'it said NOTHING AT ALL — the form simply did not respond');
255 } else {
256 check(c.wants.test(msg), `refusing names ${c.field}`, JSON.stringify(msg));
257 }
258 } catch (e) {
259 skip(c.name, String(e && e.message ? e.message : e).split('\n')[0]);
260 } finally {
261 if (s) { try { await s.close(); } catch (e) { /* already gone */ } }
262 try { fs.rmSync(dir, { recursive: true, force: true }); } catch (e) { /* gone */ }
263 }
264}
265
266if (skipped.length) console.log('\nskipped: ' + skipped.join('; '));
267console.log(failures === 0
268 ? `\nerrnames: every refusal names the box that caused it${skips ? ` (${skips} SKIPPED)` : ''}.`
269 : `\nerrnames: ${failures} message(s) that do not name the failing field${skips ? `, ${skips} SKIPPED` : ''}.`);
270process.exit(failures === 0 && skips === 0 ? 0 : 1);