Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_thinking.mjs

14.7 KiB, 1 run

created by r2519314175:739, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// The model's working, shown while it is being done — 2026-08-28.
2//
3// The finding this is built on, measured against OpenRouter that morning: Daimond read
4// the `content` of an OpenAI-dialect delta and nothing else, so every reasoning token
5// that GLM and DeepSeek streamed was thrown away. One DeepSeek round took 84.7 seconds
6// and pulled 1,814,168 bytes down the wire in 1,378 chunks, and what reached the page
7// was one tool call and about fifty characters. A GLM round measured here spent 230 of
8// its 300 output tokens reasoning and was then cut off mid-thought. The user was billed
9// for all of it and watched a blank spinner throughout.
10//
11// So the engine now reads `reasoning` (OpenRouter's spelling) and `reasoning_content`
12// (DeepSeek's own), and hands them to the page as they arrive, tagged as working rather
13// than as answer. Which leaves this file with two things to establish, and they pull in
14// opposite directions:
15//
16// THE WORKING IS SHOWN, WHILE IT IS STILL BEING DONE. Not after the round, which is
17// the one moment it no longer explains the wait.
18//
19// AND IT IS NEVER THE ANSWER. The reply is what gets written into the transcript and
20// handed back to the model next turn as its own words. A round's working is mostly
21// wrong turns, and quoting those back to a model as its own conclusions is a data
22// defect, not a cosmetic one.
23//
24// node dev/verify_thinking.mjs
25// node dev/verify_thinking.mjs --break perdelta # a tile per delta, as it was before
26// node dev/verify_thinking.mjs --break shut # the tile drawn already collapsed
27// node dev/verify_thinking.mjs --break nocollapse # and never folded away afterwards
28// node dev/verify_thinking.mjs --break asanswer # the working recorded as the reply
29
30import fs from 'node:fs';
31import path from 'node:path';
32import { fileURLToPath } from 'node:url';
33
34const HERE = path.dirname(fileURLToPath(import.meta.url));
35const ROOT = path.dirname(HERE);
36const WWW = path.join(ROOT, 'www');
37const H = await import(path.join(HERE, 'harness.mjs'));
38const { open, newChat, chat, transcript, shot, connectMock, signInAs, scratch } = H;
39
40const BREAK = (() => {
41 const i = process.argv.indexOf('--break');
42 return i === -1 ? '' : (process.argv[i + 1] || '');
43})();
44const NAME = 'thinking';
45
46let failures = 0;
47const log = (...a) => console.log(...a);
48const line = (t) => log('\n════════ ' + t + ' ════════');
49const check = (ok, what, detail = '') => {
50 log((ok ? ' PASS ' : ' FAIL ') + what + (detail ? ' -- ' + detail : ''));
51 if (!ok) failures++;
52};
53
54const BREAKS = {
55 // The drawing exactly as it was before this round: every delta its own tile, its own
56 // record, and no growing. On a real round that is four hundred disclosures.
57 perdelta: [{
58 file: 'js/daimond.js',
59 find: "\t\tif (live && liveThink && liveThink.parentNode) {",
60 with: "\t\tif (false && liveThink && liveThink.parentNode) {",
61 }, {
62 file: 'js/daimond.js',
63 find: "\t\tif (last && last.role === 'think_log') { last.content = (last.content || '') + text; return; }",
64 with: "\t\tif (false) { return; }",
65 }],
66 // Drawn, but collapsed — which is what the tile did before, and which leaves a reader
67 // with a closed box and a spinner rather than something to watch.
68 shut: [{
69 file: 'js/daimond.js',
70 find: "\t\t\td.open = true;",
71 with: "\t\t\td.open = false;",
72 }],
73 // Grown and never closed off, so the answer arrives underneath a wall of working.
74 nocollapse: [{
75 file: 'js/daimond.js',
76 find: "\t\t\tif (!liveThink._held) liveThink.open = false;",
77 with: "\t\t\tif (false) liveThink.open = false;",
78 }],
79 // THE ONE THAT MATTERS. The working written into the record as what the model said.
80 asanswer: [{
81 file: 'js/daimond.js',
82 find: "\t\tmsgs.push({ role: 'think_log', content: text, mid: newMid(), ts: Date.now() });",
83 with: "\t\tmsgs.push({ role: 'assistant', content: text, mid: newMid(), ts: Date.now() });",
84 }],
85};
86if (BREAK && !BREAKS[BREAK]) {
87 console.error(`unknown break '${BREAK}'; one of: ${Object.keys(BREAKS).join(', ')}`);
88 process.exit(2);
89}
90function damaged(src, spec) {
91 const n = src.split(spec.find).length - 1;
92 if (n !== 1) {
93 console.error(`break '${BREAK}': the anchor appears ${n} times in ${spec.file}, `
94 + 'so nothing was broken and the run below would prove nothing.');
95 process.exit(2);
96 }
97 return src.replace(spec.find, spec.with);
98}
99const bodies = new Map();
100for (const spec of (BREAKS[BREAK] || [])) {
101 const src = bodies.has(spec.file) ? bodies.get(spec.file)
102 : fs.readFileSync(path.join(WWW, spec.file), 'utf8');
103 bodies.set(spec.file, damaged(src, spec));
104}
105async function serveBreaks(page) {
106 for (const [file, body] of bodies) {
107 await page.route('**/' + file, r => r.fulfill({
108 status: 200, contentType: 'application/javascript', body }));
109 }
110}
111
112// THE FIXTURE. Every word distinct, so a doubled stream is a doubled word rather than a
113// longer one, and so a count of occurrences means something.
114const WORDS = ['WORKING-01', 'WORKING-02', 'WORKING-03', 'WORKING-04', 'WORKING-05',
115 'WORKING-06', 'WORKING-07', 'WORKING-08', 'WORKING-09', 'WORKING-10',
116 'WORKING-11', 'WORKING-12', 'WORKING-13', 'WORKING-14', 'WORKING-15'];
117const THINK = WORDS.join(' ');
118const ANSWER = 'ANSWER-MARKER the number is 391.';
119
120const s = await open({ name: NAME, connect: false,
121 profile: scratch('pw', 'thinking' + (BREAK ? '-' + BREAK : '')),
122 route: serveBreaks });
123await connectMock(s);
124// The profile outlives a run, and section 3 counts records across the whole store: a
125// second run would otherwise be counting the first one's rounds and calling it a defect.
126await H.clearChats(s);
127await newChat(s);
128
129// ── The round, watched while it runs ────────────────────────────────────────
130//
131// The turn is started and NOT awaited, because the whole question is what the page shows
132// during it. `@reasonslow` paces the working at a real round's pace so there is something
133// to look at.
134line('1. the working is on the screen while the model is still doing it');
135await s.page.fill('#chat-input', `@reasonslow ${THINK} ;; ${ANSWER}`);
136await s.page.click('#chat-send', { force: true });
137
138// Poll until a thinking tile appears, and record how far into the round that was and how
139// much of the answer had arrived by then.
140const t0 = Date.now();
141let firstSight = null;
142while (Date.now() - t0 < 20000) {
143 const seen = await s.page.evaluate(() => {
144 const tiles = Array.from(document.querySelectorAll('.chat-msg-thinking'));
145 if (!tiles.length) return null;
146 const b = tiles[0].querySelector('.chat-thinking-body');
147 const asst = document.querySelector('.chat-msg-assistant');
148 return {
149 tiles: tiles.length,
150 open: tiles[0].open,
151 body: b ? String(b.textContent || '') : '',
152 answerYet: asst ? String(asst.textContent || '') : '',
153 };
154 });
155 if (seen) { firstSight = { ...seen, atMs: Date.now() - t0 }; break; }
156 await s.page.waitForTimeout(60);
157}
158check(!!firstSight, 'a thinking tile appeared during the round',
159 firstSight ? `after ${firstSight.atMs} ms` : 'nothing in 20 s');
160if (firstSight) {
161 check(firstSight.body.includes('WORKING-01'),
162 'and it already had the model\'s first words in it', JSON.stringify(firstSight.body.slice(0, 40)));
163 check(!firstSight.answerYet.includes('ANSWER-MARKER'),
164 'shown BEFORE the answer, which is the whole point of showing it',
165 `answer so far: ${JSON.stringify(firstSight.answerYet.slice(0, 40))}`);
166 check(firstSight.open === true,
167 'open, because at that moment it is the only thing there is to show');
168}
169
170// And it GROWS: a second look, still mid-round, must find more in the same tile.
171let grew = null;
172for (let i = 0; i < 60; i++) {
173 await s.page.waitForTimeout(200);
174 const now = await s.page.evaluate(() => {
175 const tiles = Array.from(document.querySelectorAll('.chat-msg-thinking'));
176 const b = tiles.length ? tiles[0].querySelector('.chat-thinking-body') : null;
177 return { tiles: tiles.length, body: b ? String(b.textContent || '') : '' };
178 });
179 if (firstSight && now.body.length > firstSight.body.length) { grew = now; break; }
180}
181check(!!grew && grew.body.length > (firstSight ? firstSight.body.length : 0),
182 'the tile FILLS as the model thinks rather than appearing whole at the end',
183 grew ? `${firstSight.body.length} → ${grew.body.length} characters` : 'never grew');
184check(!!grew && grew.tiles === 1,
185 'and it is one tile, not one per delta',
186 grew ? `${grew.tiles} tile(s) partway through` : '');
187await shot(s, 'thinking-midround');
188
189// ── The round finishes ──────────────────────────────────────────────────────
190line('2. and then it gets out of the answer\'s way');
191const t1 = Date.now();
192while (Date.now() - t1 < 30000) {
193 const busy = await s.page.evaluate(() => {
194 const b = document.getElementById('chat-send');
195 if (!b) return false;
196 const t = (b.getAttribute('title') || '') + (b.className || '');
197 return /stop/i.test(t) || b.disabled;
198 });
199 if (!busy) break;
200 await s.page.waitForTimeout(200);
201}
202await s.page.waitForTimeout(600);
203
204const done = await s.page.evaluate((words) => {
205 const tiles = Array.from(document.querySelectorAll('.chat-msg-thinking'));
206 const body = tiles.length
207 ? String(tiles[0].querySelector('.chat-thinking-body').textContent || '') : '';
208 const counts = {};
209 for (const w of words) counts[w] = (body.match(new RegExp(w, 'g')) || []).length;
210 const out = document.getElementById('chat-output');
211 return {
212 tiles: tiles.length,
213 open: tiles.length ? tiles[0].open : null,
214 body,
215 counts,
216 text: out ? out.innerText : '',
217 };
218}, WORDS);
219
220check(done.tiles === 1, 'the whole round is ONE tile', `${done.tiles}`);
221check(done.open === false,
222 'shut once there is an answer, so the reply is not buried under the working');
223check(WORDS.every(w => done.body.includes(w)),
224 'and it holds every word the model thought',
225 `${done.body.length} characters`);
226// `reasoning_details` repeats the same words verbatim on the wire. A reader that took
227// both fields would put each of them on the page twice.
228const wrong = WORDS.filter(w => done.counts[w] !== 1);
229check(wrong.length === 0,
230 'each word in the tile exactly once, though the provider sends it twice '
231 + '(`reasoning` and `reasoning_details` carry the same text)',
232 wrong.length
233 ? wrong.map(w => `${w}×${done.counts[w]}`).join(' ')
234 : `all ${WORDS.length} exactly once`);
235check(done.text.includes('ANSWER-MARKER'), 'the answer is on screen');
236await shot(s, 'thinking-after');
237
238// ── THE ONE THAT MUST NOT BREAK ─────────────────────────────────────────────
239line('3. the working is not the answer, in the record as well as on the screen');
240const stored = await s.page.evaluate(() => new Promise((res) => {
241 const req = indexedDB.open('daimond-chats', 1);
242 req.onsuccess = () => {
243 const db = req.result; let t;
244 try { t = db.transaction('chats', 'readonly'); } catch (e) { res([]); return; }
245 const out = []; const cur = t.objectStore('chats').openCursor();
246 cur.onsuccess = () => { const c = cur.result; if (c) { out.push(c.value); c.continue(); } else res(out); };
247 cur.onerror = () => res(out);
248 };
249 req.onerror = () => res([]);
250}));
251const msgs = stored.flatMap(c => c.messages || []);
252const asst = msgs.filter(m => m.role === 'assistant');
253const think = msgs.filter(m => m.role === 'think_log');
254check(asst.length > 0 && asst.some(m => String(m.content || '').includes('ANSWER-MARKER')),
255 'the answer is stored as the assistant\'s message', `${asst.length} assistant message(s)`);
256check(!asst.some(m => WORDS.some(w => String(m.content || '').includes(w))),
257 'AND NONE OF THE WORKING IS IN IT -- reasoning stored as the reply would be handed '
258 + 'back to the model next turn as its own conclusions',
259 asst.map(m => JSON.stringify(String(m.content || '').slice(0, 60))).join(' '));
260check(think.length === 1,
261 'the working is one record of its own kind, not one per delta', `${think.length} think_log(s)`);
262check(think.length > 0 && WORDS.every(w => String(think[0].content || '').includes(w)),
263 'and it is all there, so a reload can redraw it');
264
265// ── DeepSeek's own spelling ─────────────────────────────────────────────────
266line('4. and the same when the provider calls the field something else');
267await chat(s, '@reasonc DEEPSEEK-WORKING considered ;; DEEPSEEK-ANSWER done.', { timeout: 30000 });
268const two = await s.page.evaluate(() => {
269 const tiles = Array.from(document.querySelectorAll('.chat-msg-thinking'));
270 const last = tiles.length ? tiles[tiles.length - 1] : null;
271 const out = document.getElementById('chat-output');
272 return {
273 body: last ? String(last.querySelector('.chat-thinking-body').textContent || '') : '',
274 text: out ? out.innerText : '',
275 };
276});
277check(two.body.includes('DEEPSEEK-WORKING'),
278 '`reasoning_content` is read as well as `reasoning`', JSON.stringify(two.body.slice(0, 50)));
279check(two.text.includes('DEEPSEEK-ANSWER'), 'and the answer still arrives');
280
281// ── A reload ────────────────────────────────────────────────────────────────
282line('5. a reload draws it back, and draws it shut');
283await s.page.reload({ waitUntil: 'domcontentloaded' });
284await s.page.waitForTimeout(1200);
285await signInAs(s, NAME);
286await s.page.waitForTimeout(1500);
287await s.page.evaluate(() => {
288 const b = document.querySelector('#session-list .chat-box.active')
289 || document.querySelector('#session-list .chat-box');
290 if (b) b.click();
291});
292await s.page.waitForTimeout(1200);
293const back = await s.page.evaluate(() => {
294 const tiles = Array.from(document.querySelectorAll('.chat-msg-thinking'));
295 const out = document.getElementById('chat-output');
296 return {
297 tiles: tiles.length,
298 open: tiles.map(t => t.open),
299 body: tiles.map(t => String(t.querySelector('.chat-thinking-body').textContent || '')).join(' '),
300 text: out ? out.innerText : '',
301 };
302});
303check(back.body.includes('WORKING-01') && back.body.includes('WORKING-15'),
304 'the working survived the reload', `${back.tiles} tile(s)`);
305check(back.open.every(o => o === false),
306 'and comes back closed: those rounds are over, so it is working again and not news',
307 JSON.stringify(back.open));
308check(back.text.includes('ANSWER-MARKER'), 'and so did the answer');
309
310// This world asked for no gateway, so `/api` answers 502 by design and says so --
311// see the note in dev/world.sh. That is the world, not the page.
312const errs = H.errors(s).filter(e => !/favicon/i.test(e) && !/\/api\/\S*\b502\b|502 \(Bad Gateway\)/.test(e));
313check(errs.length === 0, 'no console errors', errs.slice(0, 3).join(' | '));
314
315log('\n' + (failures ? `FAILED: ${failures} check(s)` : 'all checks passed'));
316await s.close();
317process.exit(failures ? 1 : 0);