Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_search_i18n.mjs

19.3 KiB, 1 run

created by r2519314175:667, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_search_i18n.mjs — the search tool's strings, in all eight languages.
2//
3// `verify_i18n.mjs` already diffs whole key sets against en.js. This one is
4// narrower and louder: it is about the keys SEARCH_CONTRACT.md §11 names, and
5// when it fails it says which locale and which key, because "de: 3 missing
6// keys" is a census and "de is missing search.no_key" is a fix.
7//
8// Four properties, and the last two exist because of defects that got through:
9//
10// 1. Every locale carries every key in §11. A key present only in English
11// falls back silently in the app, so nothing on screen looks wrong.
12// 2. Every {placeholder} in the English survives into every translation. A
13// translation that drops {engine} renders a sentence with a hole in it,
14// and no screenshot would show a hole -- it reads as a finished sentence.
15// 3. The SUBSTITUTED string is clean: no double space, no space at either
16// end, no `{` left standing. Japanese once emitted a stray space before
17// its particle and Chinese once glued a translated word onto an
18// untranslated one, and NEITHER was visible in the source strings --
19// both appeared only once the placeholder was filled in.
20// 4. The two terms this release could have invented a second word for --
21// the API key and Daimond credits -- are word for word what the locale
22// already calls them at `models.api_key` and `money.daimond_credits`.
23//
24// No browser, no server, no gateway: the tables are read from disk.
25//
26// node dev/verify_search_i18n.mjs
27import fs from 'node:fs';
28import path from 'node:path';
29import { fileURLToPath } from 'node:url';
30
31const HERE = path.dirname(fileURLToPath(import.meta.url));
32const I18NDIR = path.join(HERE, '..', 'www', 'i18n');
33
34const ok = [], bad = [];
35const check = (name, pass, detail) => {
36 (pass ? ok : bad).push(name);
37 console.log((pass ? ' ok ' : ' FAIL ') + name + (detail ? ' — ' + detail : ''));
38};
39
40/// One locale table, out of its IIFE and into a plain object.
41const loadTable = (code) => {
42 let table = null;
43 const src = fs.readFileSync(path.join(I18NDIR, code + '.js'), 'utf8');
44 new Function('window', src)({ DaimondI18n: { register: (c, t) => { table = t; } } });
45 return table;
46};
47
48// ── What §11 names ───────────────────────────────────────────────────
49//
50// Written out rather than derived from en.js: this file is the second reader
51// of the contract, and a key quietly dropped from en.js should fail here
52// rather than shrink the list being checked.
53const KEYS = [
54 'search.head', 'search.engine', 'search.engine_note', 'search.credits',
55 'search.key', 'search.key_note', 'search.no_key',
56 'search.kind_web', 'search.kind_news', 'search.kind_academic',
57 'search.refused_serper', 'search.free_month',
58 'web.pause', 'pause.refused.web',
59 'egress.search_title', 'egress.search_body', 'egress.search_ok',
60];
61
62/// The consent dialog. Its own name, because several checks are only about it:
63/// it is the one string a user reads before a query leaves their device.
64const BODY = 'egress.search_body';
65
66const LOCALES = ['en', 'de', 'es', 'fr', 'ja', 'ko', 'pt-BR', 'zh-Hans'];
67
68/// The five engine ids of §3, by the name a user sees. Proper nouns in Latin
69/// script in every language, which is the whole reason the spacing around
70/// `{engine}` has to be read back.
71const ENGINES = ['Brave', 'Exa', 'Tavily', 'Serper'];
72
73/// What each placeholder holds when the app fills it in. Every engine is tried
74/// rather than one: `search.refused_serper` is only ever shown for Serper, and
75/// a check that only ever composed "Brave" would not be reading the line the
76/// user gets. `{node}` is a pause node's human name, and the app passes the
77/// TRANSLATED one.
78/// Engine DISPLAY names, as the egress dialog and the allowance line receive
79/// them: sometimes a proper noun, sometimes a translated phrase. The second
80/// kind is why the CJK spacing has to be read back with both.
81const DISPLAY = (t) => ['Brave Search', 'Serper', t['search.credits'] || 'Daimond credits'];
82
83const FILL = {
84 engine: (t) => ENGINES.concat(DISPLAY(t)),
85 node: (t) => [t['pause.web'] || 'Web'],
86 // Already grouped by toLocaleString() before it arrives — three groupings
87 // a locale might use, so a separator added by hand shows up as a double.
88 n: () => ['1,000', '1 000', '1.000'],
89 // The user's own text, truncated to 300 with an ellipsis. Two shapes that
90 // have broken a dialog before: one carrying quotes of its own, and one
91 // already ending in the ellipsis.
92 query: () => ['best "cold brew" ratio', 'a very long question…'],
93};
94
95/// Every way a string can be filled in: one per engine, times one per query,
96/// times one per grouping of {n}. Small enough to enumerate, and enumerating
97/// is the point — the defects this file exists for appear in ONE of the fills
98/// and not the others.
99const fills = (code, s) => {
100 let out = [String(s)];
101 for (const p of new Set((String(s).match(/\{(\w+)\}/g) || []).map((m) => m.slice(1, -1)))) {
102 if (!FILL[p]) continue;
103 const vals = FILL[p](table[code]);
104 out = out.flatMap((one) => vals.map((v) => one.split(`{${p}}`).join(v)));
105 }
106 return out;
107};
108
109const table = {};
110for (const code of LOCALES) {
111 try {
112 table[code] = loadTable(code);
113 } catch (e) {
114 table[code] = null;
115 check(`${code}: the table loads`, false, String(e.message));
116 }
117}
118check('the eight locale files load', LOCALES.every((c) => table[c]), LOCALES.join(' '));
119
120const EN = table.en || {};
121
122// ── 1. Every locale carries every key ────────────────────────────────
123{
124 const gaps = [];
125 for (const code of LOCALES) {
126 if (!table[code]) continue;
127 for (const k of KEYS) {
128 const v = table[code][k];
129 if (typeof v !== 'string') gaps.push(`${code} is missing ${k}`);
130 else if (!v.trim()) gaps.push(`${code} has ${k} empty`);
131 }
132 }
133 check('every locale carries every key §11 names', gaps.length === 0, gaps.join('; '));
134}
135
136// ── 2. Every placeholder survives translation ────────────────────────
137//
138// Order is a translator's business and is not checked; presence is not.
139{
140 const holes = [];
141 const names = (s) => (String(s).match(/\{(\w+)\}/g) || []).map((m) => m.slice(1, -1)).sort();
142 for (const k of KEYS) {
143 const want = names(EN[k]);
144 if (!want.length) continue;
145 for (const code of LOCALES) {
146 if (!table[code] || typeof table[code][k] !== 'string') continue;
147 const got = names(table[code][k]);
148 for (const p of new Set(want)) {
149 if (!got.includes(p)) holes.push(`${code}/${k} drops {${p}}`);
150 }
151 for (const p of new Set(got)) {
152 if (!want.includes(p)) holes.push(`${code}/${k} invents {${p}}`);
153 }
154 }
155 }
156 check('every placeholder in the English survives into every translation',
157 holes.length === 0, holes.join('; '));
158}
159
160// ── 3. The substituted string, read back ─────────────────────────────
161//
162// Both halves matter. The RAW value catches a double space a translator typed;
163// the SUBSTITUTED one catches what only appears once the placeholder is gone.
164{
165 const dirty = [];
166 for (const code of LOCALES) {
167 if (!table[code]) continue;
168 for (const k of KEYS) {
169 const raw = table[code][k];
170 if (typeof raw !== 'string') continue;
171 for (const [what, s] of [['source', raw]].concat(fills(code, raw).map((f) => ['substituted', f]))) {
172 if (/ {2}/.test(s)) dirty.push(`${code}/${k} ${what}: double space — ${JSON.stringify(s)}`);
173 if (/^\s|\s$/.test(s)) dirty.push(`${code}/${k} ${what}: space at an end — ${JSON.stringify(s)}`);
174 if (what === 'substituted' && /[{}]/.test(s))
175 dirty.push(`${code}/${k}: a brace survived substitution — ${JSON.stringify(s)}`);
176 }
177 }
178 }
179 check('no double space, no space at either end, no brace left standing',
180 dirty.length === 0, dirty.join('; '));
181}
182
183// ── 3b. CJK spacing around a placeholder in Latin script ─────────────
184//
185// Two placeholders arrive in Latin script whatever the language: `{engine}`
186// holds a proper noun or a display name, `{n}` holds digits. Each of the three
187// files has a settled convention for what goes beside them, and each is
188// different, so the rule is read off the file rather than off the language.
189//
190// ja spaces both — `API キー`, `{provider} 経由`, `{n} 件`, `{n} 分前`.
191// zh-Hans spaces both — `Daimond 额度`, `移除 {provider}`, `{n} 个`, `{n} 封`.
192// ko spaces a following NOUN but glues a PARTICLE onto {engine}
193// (`{host}에`, `{provider}을(를)`), and glues the counter onto {n}
194// (`{n}개`, `{n}번`, `{n}줄`) — 24 of 24 in that file.
195//
196// Punctuation beside a placeholder is not spacing and is not checked: a
197// full-width comma or colon takes no space in front of it in any of the three.
198{
199 const CJK = /[぀-ヿ㐀-䶿一-鿿]/;
200 const HANGUL = /[가-힯]/;
201 // The particles this file already glues to a placeholder.
202 const JOSA = /^(을\(를\)|이\(가\)|와\(과\)|은\(는\)|\(으\)로|으?로|에서|에게|까지|부터|을|를|이|가|은|는|와|과|의|에|도|만)/;
203 const wrong = [];
204 for (const code of ['ja', 'zh-Hans', 'ko']) {
205 if (!table[code]) continue;
206 for (const k of KEYS) {
207 const s = table[code][k];
208 if (typeof s !== 'string') continue;
209 for (const m of s.matchAll(/\{(engine|n)\}/g)) {
210 const ph = m[0], which = m[1];
211 const after = s.slice(m.index + ph.length);
212 const prev = s.slice(0, m.index).slice(-1), next = after.slice(0, 1);
213 if (prev && (CJK.test(prev) || HANGUL.test(prev)))
214 wrong.push(`${code}/${k}: ${ph} is glued to “${prev}” before it`);
215 if (!next) continue;
216 if (code === 'ko' && which === 'engine') {
217 if (HANGUL.test(next) && !JOSA.test(after))
218 wrong.push(`${code}/${k}: ${ph} runs into “${after.slice(0, 4)}” with neither a space nor a particle`);
219 } else if (code === 'ko') {
220 // The counter is glued, so a SPACE is what goes wrong here.
221 if (/\s/.test(next) && HANGUL.test(after.trim().slice(0, 1)))
222 wrong.push(`${code}/${k}: ${ph} is spaced off its counter “${after.trim().slice(0, 2)}”`);
223 } else if (CJK.test(next)) {
224 wrong.push(`${code}/${k}: ${ph} is glued to “${next}” after it`);
225 }
226 }
227 }
228 }
229 check('ja and zh-Hans space a Latin placeholder; ko takes a particle on {engine} and glues the counter to {n}',
230 wrong.length === 0, wrong.join('; '));
231}
232
233// ── 3c. {n} is not grouped twice ─────────────────────────────────────
234//
235// The count arrives already grouped by toLocaleString(), so the reader gets
236// 1,000 or 1 000 or 1.000 as their own locale wants it. A separator written
237// into the string on top of that reads as a real, larger figure rather than as
238// a bug -- "1,000,000 searches a month free" is a promise nobody can keep --
239// and it would be wrong in all eight locales at once, not just the CJK three,
240// so this is not part of the spacing check above.
241{
242 const doubled = [];
243 for (const code of LOCALES) {
244 if (!table[code]) continue;
245 for (const k of KEYS) {
246 const s = table[code][k];
247 if (typeof s !== 'string') continue;
248 for (const m of s.matchAll(/\{n\}/g)) {
249 const after = s.slice(m.index + m[0].length);
250 if (/^[,.\u00a0\u202f ]\d/.test(after))
251 doubled.push(`${code}/${k}: {n} is followed by \u201c${after.slice(0, 4)}\u201d \u2014 a digit separator on top of the one toLocaleString() already wrote`);
252 }
253 }
254 }
255 check('{n} carries no digit grouping of our own', doubled.length === 0, doubled.join('; '));
256}
257
258// ── 3d. No space between two Japanese or two Chinese characters ──────
259//
260// This is the check the other spacing one cannot be. 3b reads the TEMPLATE and
261// can only ask whether a space sits beside the placeholder; whether that space
262// is right depends on what the placeholder turns out to hold. `{engine}` holds
263// a proper noun most of the time and a TRANSLATED phrase when the tier is
264// credits, and one template cannot be correct for both:
265//
266// 検索には {engine} を使います
267// → 検索には Brave Search を使います ← right, Latin on one side
268// → 検索には Daimond クレジット を使います ← wrong, a stray space before を
269//
270// That second line is the recorded Japanese defect exactly, and it is
271// invisible in the source string. The rule that separates the two cases is not
272// about particles at all: a space belongs between Latin and Japanese, and
273// never between two Japanese characters. So compose, then look for a space
274// with CJK on BOTH sides. Same in zh-Hans, where the same trap is a space
275// between two Chinese words.
276{
277 const CJK = /[぀-ヿ㐀-䶿一-鿿]/;
278 const wrong = [];
279 for (const code of ['ja', 'zh-Hans']) {
280 if (!table[code]) continue;
281 for (const k of KEYS) {
282 const raw = table[code][k];
283 if (typeof raw !== 'string') continue;
284 for (const s of fills(code, raw)) {
285 for (const m of s.matchAll(/(.) (.)/gs)) {
286 if (CJK.test(m[1]) && CJK.test(m[2]))
287 wrong.push(`${code}/${k}: “${m[1]} ${m[2]}” — a space between two ${code === 'ja' ? 'Japanese' : 'Chinese'} characters, in ${JSON.stringify(s.slice(Math.max(0, m.index - 14), m.index + 16))}`);
288 }
289 }
290 }
291 }
292 check('once filled in, no space sits between two Japanese or two Chinese characters',
293 wrong.length === 0, [...new Set(wrong)].join('; '));
294}
295
296// ── 3c. The consent dialog keeps its shape ───────────────────────────
297//
298// `egress.search_body` is five paragraphs and four `\n\n`. A translation that
299// runs them together passes every other check in this file and produces a wall
300// of text in a permission dialog, which is a dialog nobody reads. And {query}
301// — the user's own words, the thing that is about to leave — stands alone in
302// its own paragraph: never wrapped in quotation marks, because the query may
303// carry quotes of its own and a mismatched pair reads as corruption.
304{
305 const runs = (s) => (String(s).match(/\n+/g) || []).join('|');
306 const want = runs(EN[BODY]);
307 const broke = [], loose = [];
308 for (const code of LOCALES) {
309 const s = table[code] && table[code][BODY];
310 if (typeof s !== 'string') continue;
311 if (runs(s) !== want)
312 broke.push(`${code}: ${JSON.stringify(runs(s))} not ${JSON.stringify(want)}`);
313 // Its own paragraph: \n\n on both sides, and nothing between those
314 // breaks but the placeholder itself.
315 if (!/(^|\n\n)\{query\}(\n\n|$)/.test(s))
316 loose.push(`${code}: ${JSON.stringify((s.match(/.{0,12}\{query\}.{0,12}/s) || ['(absent)'])[0])}`);
317 }
318 check(`${BODY} keeps every paragraph break (${want.split('|').length} of them)`,
319 broke.length === 0, broke.join('; '));
320 check('the query stands alone in its own paragraph, unquoted', loose.length === 0, loose.join('; '));
321}
322
323// ── 4. One word per thing ────────────────────────────────────────────
324//
325// The two terms this change could have coined a second word for. `search.key`
326// is the same kind of thing as a provider key for inference, and the credits
327// tier is the same money as everywhere else; a user must not meet two words
328// for one thing because two lanes named it on different days.
329//
330// The credits group is three keys deep, and pt-BR had drifted: `Créditos
331// Daimond` at `models.credits_row` against `Créditos do Daimond` at
332// `money.daimond_credits`, in a file that takes the article for the product
333// everywhere else. Seven locales agreed word for word and one did not, which
334// is a typo with a long life rather than a translation somebody chose. Holding
335// all three equal is what stops a fourth key arriving to three spellings.
336for (const [what, group] of [
337 ['the API key', ['search.key', 'models.api_key']],
338 ['Daimond credits', ['search.credits', 'money.daimond_credits', 'models.credits_row']],
339]) {
340 const [first, ...rest] = group;
341 const off = [];
342 for (const code of LOCALES) {
343 if (!table[code]) continue;
344 for (const k of rest) {
345 if (table[code][k] !== table[code][first])
346 off.push(`${code}: ${k} ${JSON.stringify(table[code][k])} vs ${first} ${JSON.stringify(table[code][first])}`);
347 }
348 }
349 check(`${what} is one word per locale (${group.join(' = ')})`, off.length === 0, off.join('; '));
350}
351
352// ── 4b. No engine is written into a string ───────────────────────────
353//
354// `search.refused_serper` used to say "That engine", which pointed at nothing
355// in two of the three places it is reached from. It names the engine now, and
356// it does so through `{engine}` — writing "Serper" into thirteen strings
357// across eight files is how a fifth engine arrives and eight files disagree.
358{
359 const written = [];
360 for (const code of LOCALES) {
361 if (!table[code]) continue;
362 for (const k of KEYS) {
363 const v = table[code][k];
364 if (typeof v !== 'string') continue;
365 for (const e of ENGINES) {
366 if (new RegExp(`\\b${e}\\b`).test(v)) written.push(`${code}/${k} names ${e} outright`);
367 }
368 }
369 }
370 check('no engine is written into a string; {engine} carries the name', written.length === 0, written.join('; '));
371}
372
373// ── 5. Product nouns are left alone ──────────────────────────────────
374{
375 const lost = [];
376 for (const k of KEYS) {
377 if (!String(EN[k] || '').includes('Daimond')) continue;
378 for (const code of LOCALES) {
379 const v = table[code] && table[code][k];
380 if (typeof v === 'string' && !v.includes('Daimond')) lost.push(`${code}/${k}`);
381 }
382 }
383 check('"Daimond" is left untranslated wherever the English says it', lost.length === 0, lost.join(', '));
384}
385
386// ── 6. Every refusal says where the control is ───────────────────────
387//
388// This check has been both ways round, and the reason is worth keeping. The web
389// refusal used to end "Press play on it to resume" and sent the reader into a
390// dead end, because `root/web` had no control anywhere in the app: it was the
391// one leaf nobody had given a widget. So the clause was stripped and this
392// asserted its ABSENCE.
393//
394// The Web panel header has the control now. The clause is true again, and its
395// three siblings never stopped saying it -- so a reader who is told how to
396// resume a held mailbox and not a held page would reasonably conclude the page
397// cannot be resumed at all. Consistency across the four is the property; the
398// absence never was.
399{
400 const NAMES_A_CONTROL = [
401 /press play/i, /auf Play/i, /Pulsa play/i, /pulsar play/i, /appuyez sur lecture/i,
402 /Pressione play/i, /再生を押/, /재생을 누/, /播放键/, /播放按钮/,
403 ];
404 // All four, in every language -- not the English alone. A clause dropped
405 // from one locale is exactly the kind of thing a census of keys cannot see,
406 // since the key is present and the sentence is merely poorer.
407 const KINDS = ['turn', 'dispatch', 'web', 'mail'];
408 const silent = [];
409 for (const c of LOCALES) {
410 for (const n of KINDS) {
411 const v = table[c] && table[c]['pause.refused.' + n];
412 if (typeof v !== 'string') continue;
413 if (!NAMES_A_CONTROL.some((re) => re.test(v))) silent.push(`${c}/${n}`);
414 }
415 }
416 check('every pause refusal says how to resume, in every language',
417 silent.length === 0, silent.join(', '));
418}
419
420console.log('\n' + ok.length + ' ok, ' + bad.length + ' failed');
421process.exit(bad.length ? 1 : 0);