Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_favourites.mjs

8.9 KiB, 1 run

created by r2519314175:407, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_favourites.mjs — the models you use are at the top, and they are the same
2// models as the ones further down.
3//
4// A working setup reaches a dozen models across four providers, listed in the order
5// they are BILLED, which is not the order anybody wants them in. The favourites
6// group is a shortcut to a row that already exists — so the two things it has to
7// get right are that the shortlist is genuinely the most-used, and that a shortcut
8// says exactly what the row it stands for says. A shortcut that read differently
9// from the real row would be worse than no shortcut, because the economy marking
10// (whose money a turn spends) is the part that matters most in this pulldown.
11//
12// The list is built from USE, not from selection: a model chosen in a pulldown and
13// never run is not one anybody uses. So the checks drive `noteUse`, which is what
14// the three commit points in daimond.js call.
15//
16// node dev/verify_favourites.mjs
17//
18// Needs dev/serve.mjs (DAIMOND_PORT, default 8777). No gateway and no model: the
19// provider list is seeded directly, and no turn is run.
20import fs from 'node:fs';
21import { open, errors, scratch } from './harness.mjs';
22
23const PROFILE = scratch('pw', 'favourites');
24fs.rmSync(PROFILE, { recursive: true, force: true });
25
26let bad = 0;
27const check = (pass, name, detail) => {
28 if (!pass) bad++;
29 console.log((pass ? ' ok ' : ' FAIL ') + name + (detail ? ' — ' + detail : ''));
30};
31
32const s = await open({ name: 'favourites', profile: PROFILE, connect: false });
33const { page } = s;
34
35/// Seed enough providers and models that the list is genuinely long, then read the
36/// pulldown back as a structure. Everything below is the real `fillSelect`.
37const build = (uses) => page.evaluate(async (u) => {
38 localStorage.setItem('daimond-models-v2', JSON.stringify({
39 v: 2,
40 def: { provider: 'alpha', model: 'alpha/one' },
41 providers: {
42 alpha: { name: 'Alpha', url: 'https://a/v1/chat/completions', key: 'k',
43 models: ['alpha/one', 'alpha/two', 'alpha/three', 'alpha/four'] },
44 beta: { name: 'Beta', url: 'https://b/v1/chat/completions', key: 'k',
45 models: ['beta/one', 'beta/two', 'beta/three', 'beta/four'] },
46 gamma: { name: 'Gamma', url: 'https://g/v1/chat/completions', key: 'k',
47 models: ['gamma/one', 'gamma/two', 'gamma/three'] },
48 },
49 }));
50 localStorage.removeItem('daimond-model-use');
51 DaimondModels.init({});
52 for (const [prov, model, times] of u) {
53 for (let i = 0; i < times; i++) DaimondModels.noteUse(prov, model);
54 }
55 // Then push the timestamps APART, oldest for the most-used.
56 //
57 // Without this the fixture cannot tell the two orderings apart: the loop above
58 // runs inside one millisecond, so every entry carries the same `t`, a
59 // recency sort is stable, and it happens to return insertion order — which is
60 // the order the counts would have given anyway. A break that ranked by recency
61 // instead of by use passed against this file until the stamps were separated.
62 // The most-used model is now the LEAST recent, so the two orderings disagree
63 // and only the right one satisfies the check.
64 const raw = JSON.parse(localStorage.getItem('daimond-model-use') || '{}');
65 const byCount = Object.keys(raw).sort((a, b) => raw[b].n - raw[a].n);
66 byCount.forEach((k, i) => { raw[k].t = 1000 + i * 1000; });
67 localStorage.setItem('daimond-model-use', JSON.stringify(raw));
68 const sel = document.createElement('select');
69 DaimondModels.fillSelect(sel, '', '');
70 const groups = Array.from(sel.children).map((g) => ({
71 label: g.label || '',
72 options: Array.from(g.children).map((o) => ({
73 providerName: (g.label || '').split(' (')[0].split(' · ')[0],
74 value: o.value, text: o.textContent, title: o.title,
75 provider: o.dataset.provider || '', paid: o.dataset.paid || '',
76 fav: o.dataset.fav || '', disabled: o.disabled, selected: o.selected,
77 })),
78 }));
79 return { groups, total: sel.querySelectorAll('option').length,
80 picked: DaimondModels.pick(sel) };
81}, uses);
82
83try {
84 await page.waitForFunction(() => !!(window.DaimondModels && DaimondModels.fillSelect),
85 null, { timeout: 20000 });
86 await page.waitForTimeout(500);
87
88 console.log('with nothing used yet');
89 const cold = await build([]);
90 check(cold.groups.length > 0 && !/favourite/i.test(cold.groups[0].label),
91 'a new account gets no shortlist — there is nothing to shorten',
92 JSON.stringify(cold.groups.map((g) => g.label)));
93
94 console.log('once some models have been run');
95 const uses = [['beta', 'beta/three', 5], ['alpha', 'alpha/two', 3], ['gamma', 'gamma/one', 1]];
96 const warm = await build(uses);
97 const first = warm.groups[0];
98 check(/favourite|favorit|常用|자주|よく/i.test(first.label),
99 'the shortlist is the FIRST group, so it needs no scrolling', first.label);
100 check(first.options.map((o) => o.value).join(',') === 'beta/three,alpha/two,gamma/one',
101 'ordered by how much each is used, most first — and NOT by recency, which the '
102 + 'fixture deliberately puts in the opposite order',
103 first.options.map((o) => `${o.value}(${o.provider})`).join(', '));
104
105 console.log('a shortcut says what the row says');
106 const rows = {};
107 warm.groups.slice(1).forEach((g) => g.options.forEach((o) => { rows[o.provider + ' ' + o.value] = o; }));
108 let mismatched = [];
109 for (const f of first.options) {
110 const r = rows[f.provider + ' ' + f.value];
111 if (!r) { mismatched.push(`${f.value}: no row below`); continue; }
112 // The favourite carries the row's text PLUS the provider's name. Not an
113 // inconsistency: a row's full meaning includes the group heading above it,
114 // and under "Favourites" that heading is gone — so a shortcut that
115 // reproduced only the row's own text would lose which provider it is, and
116 // two providers offering the same model under the user's own key would
117 // draw two identical shortcuts.
118 const rowStem = r.text.replace(/\s+★$/, '');
119 if (!f.text.startsWith(rowStem)) mismatched.push(`${f.value}: "${f.text}" does not begin with "${rowStem}"`);
120 if (!f.text.includes(r.providerName || '') && r.providerName)
121 mismatched.push(`${f.value}: does not name its provider`);
122 if (r.title !== f.title) mismatched.push(`${f.value}: title differs`);
123 if (r.paid !== f.paid) mismatched.push(`${f.value}: economy marking differs`);
124 }
125 check(mismatched.length === 0,
126 'every favourite is word-for-word the row it stands for, marking included',
127 mismatched.join(' | '));
128 check(first.options.every((o) => o.fav === '1'),
129 'and each is marked as the shortcut rather than a second listing');
130 check(first.options.every((o) => /·\s*(Alpha|Beta|Gamma)\b/.test(o.text)),
131 'and names its provider, which the group heading no longer does',
132 first.options.map((o) => o.text).join(' | '));
133
134 console.log('and it is a shortcut, not a category');
135 check(warm.total === cold.total + first.options.length,
136 'nothing was MOVED out of its provider — the rows are all still there',
137 `cold ${cold.total}, warm ${warm.total}, shortlist ${first.options.length}`);
138 check(Object.keys(rows).length === cold.total,
139 'so a model is still findable under who bills for it', String(Object.keys(rows).length));
140
141 console.log('what pick() answers');
142 check(warm.picked.model === 'alpha/one' && warm.picked.provider === 'alpha',
143 'the starred default is still what an unasked pulldown points at',
144 JSON.stringify(warm.picked));
145 const asked = await page.evaluate(() => {
146 const sel = document.createElement('select');
147 DaimondModels.fillSelect(sel, 'beta', 'beta/three');
148 const p = DaimondModels.pick(sel);
149 const o = sel.selectedOptions[0];
150 return { pick: p, fromFav: o.dataset.fav === '1' };
151 });
152 check(asked.pick.provider === 'beta' && asked.pick.model === 'beta/three',
153 'asking for a model still selects that model, and pick() names its provider',
154 JSON.stringify(asked.pick));
155 check(asked.fromFav === true,
156 'and the selected one is the shortcut copy, so it shows without scrolling');
157
158 console.log('a shortlist of one is not a shortlist');
159 const thin = await build([['beta', 'beta/three', 4]]);
160 check(!/favourite|favorit|常用|자주|よく/i.test(thin.groups[0].label),
161 'one used model draws no group', thin.groups[0].label);
162
163 console.log('a model whose provider went away');
164 const gone = await page.evaluate(() => {
165 const store = JSON.parse(localStorage.getItem('daimond-models-v2'));
166 delete store.providers.beta;
167 localStorage.setItem('daimond-models-v2', JSON.stringify(store));
168 DaimondModels.init({});
169 const sel = document.createElement('select');
170 DaimondModels.fillSelect(sel, '', '');
171 return Array.from(sel.querySelectorAll('option')).map((o) => o.value);
172 });
173 check(!gone.some((v) => v === 'beta/three'),
174 'is not still offered from the top of the list', JSON.stringify(gone.slice(0, 4)));
175
176 const errs = errors(s).filter((e) => !/50[23]|Bad Gateway|Failed to load resource/i.test(e));
177 check(errs.length === 0, 'no console errors beyond the offline gateway',
178 JSON.stringify(errs.slice(0, 2)));
179} finally {
180 await s.close();
181}
182
183console.log(bad ? `\n${bad} failed` : '\nall checks passed');
184process.exit(bad ? 1 : 0);