Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_working.mjs

9.7 KiB, 1 run

created by r2519314175:823, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// The model's own working, after it has been demoted — note 05 of the 2026-08-27 round.
2//
3// Verbatim: *"Chats are losing responses from models. I see some response text from a model,
4// then later it is superceded by a final answer and I can't see what was there before."*
5//
6// Nothing is lost, and that was measured before anything was changed. A run of prose
7// followed by a tool call is demoted by `demoteToWorking` (www/js/daimond.js) into a
8// collapsed control, exactly as designed on 2026-08-23 -- and the WHOLE run, working and
9// answer both, is stored as one assistant message, so a reload shows every word of it.
10//
11// What the tester met is presentational, and in two parts. A paragraph he had just watched
12// arrive was replaced by one grey sentence with no sign that there was anything behind it;
13// and that sentence was cut at 160 characters wherever that landed, mid-word, in the branch
14// the code's own comment says never happens. Both are checked here, and the second is
15// checked with a narration deliberately built to have no sentence end inside 160 characters,
16// because that is the only input that reaches the broken branch.
17//
18// AND THERE WAS A THIRD PART, which this file could not see because it never touched the
19// switch. The 2026-08-23 design also hid the demoted tile with the tool steps, so a reader
20// with Steps off was shown no working at all -- zero by zero, nothing to open, the model's
21// prose gone from the screen entirely. The owner reversed that on 2026-08-28 and
22// `dev/verify_visible.mjs` is where the switch is now driven; this file runs with it in its
23// default position and is unchanged in what it asks.
24//
25// node dev/verify_working.mjs
26// node dev/verify_working.mjs --break midword # the cut put back where it was
27// node dev/verify_working.mjs --break nocount # the control with nothing to say it opens
28
29import fs from 'node:fs';
30import path from 'node:path';
31import { fileURLToPath } from 'node:url';
32
33const HERE = path.dirname(fileURLToPath(import.meta.url));
34const ROOT = path.dirname(HERE);
35const WWW = path.join(ROOT, 'www');
36const H = await import(path.join(HERE, 'harness.mjs'));
37const { open, newChat, chat, transcript, shot, connectMock, signInAs, scratch } = H;
38
39const BREAK = (() => {
40 const i = process.argv.indexOf('--break');
41 return i === -1 ? '' : (process.argv[i + 1] || '');
42})();
43const NAME = 'working';
44
45let failures = 0;
46const log = (...a) => console.log(...a);
47const line = (t) => log('\n════════ ' + t + ' ════════');
48const check = (ok, what, detail = '') => {
49 log((ok ? ' PASS ' : ' FAIL ') + what + (detail ? ' -- ' + detail : ''));
50 if (!ok) failures++;
51};
52
53const BREAKS = {
54 // The cut put back exactly as it was: 160 characters, wherever that lands.
55 midword: [{
56 file: 'js/daimond.js',
57 find: "\t\t\tvar sp = head.lastIndexOf(' ');\n\t\t\tif (sp > 40) head = head.slice(0, sp);",
58 with: "\t\t\tvar sp = head.lastIndexOf(' ');\n\t\t\tif (false) head = head.slice(0, sp);",
59 }],
60 // The control, with nothing on it to say there is anything to open.
61 nocount: [{
62 file: 'js/daimond.js',
63 find: "\t\tsum.appendChild(more);",
64 with: "\t\tif (false) sum.appendChild(more);",
65 }],
66};
67if (BREAK && !BREAKS[BREAK]) {
68 console.error(`unknown break '${BREAK}'; one of: ${Object.keys(BREAKS).join(', ')}`);
69 process.exit(2);
70}
71function damaged(src, spec) {
72 const n = src.split(spec.find).length - 1;
73 if (n !== 1) {
74 console.error(`break '${BREAK}': the anchor appears ${n} times in ${spec.file}, `
75 + 'so nothing was broken and the run below would prove nothing.');
76 process.exit(2);
77 }
78 return src.replace(spec.find, spec.with);
79}
80const bodies = new Map();
81for (const spec of (BREAKS[BREAK] || [])) {
82 const src = bodies.has(spec.file) ? bodies.get(spec.file)
83 : fs.readFileSync(path.join(WWW, spec.file), 'utf8');
84 bodies.set(spec.file, damaged(src, spec));
85}
86async function serveBreaks(page) {
87 for (const [file, body] of bodies) {
88 await page.route('**/' + file, r => r.fulfill({
89 status: 200, contentType: 'application/javascript', body }));
90 }
91}
92
93// THE FIXTURE IS THE INPUT THAT REACHES THE BROKEN BRANCH, and it is built here rather than
94// written out, so a later hand cannot shorten it without noticing what it was for. No full
95// stop until well past 160 characters, which is what sends `demoteToWorking` down the arm
96// that used to cut wherever 160 fell.
97const LONG = 'INTERIM-PROSE-MARKER and then the model keeps going in one long clause about '
98 + 'the line numbers it means to pin down and the files it means to open before it would '
99 + 'ever commit to anything at all, still with no full stop anywhere in sight, and only '
100 + 'here does it finally stop. Then a second sentence, which is the part that goes behind '
101 + 'the control and must be findable.';
102// THE 160th CHARACTER MUST LAND INSIDE A WORD, or the broken branch cuts cleanly by luck and
103// `--break midword` reddens nothing. Asserted rather than trusted: the first draft of this
104// fixture put a space there, the break passed every check, and the proof was worthless.
105if (/\s/.test(LONG[160]) || LONG.search(/[.!?](\s|$)/) < 160) {
106 console.error('the fixture no longer reaches the branch under test: character 160 is '
107 + JSON.stringify(LONG[160]) + ' and the first sentence ends at '
108 + LONG.search(/[.!?](\s|$)/));
109 process.exit(2);
110}
111
112const s = await open({ name: NAME, connect: false,
113 profile: scratch('pw', 'working' + (BREAK ? '-' + BREAK : '')),
114 route: serveBreaks });
115await connectMock(s);
116await newChat(s);
117
118const dir = await s.page.evaluate(() => {
119 const f = window.DaimondAttach.focus();
120 return f && f.id ? window.DaimondAttach.chatScratch(f.id) : '';
121});
122check(!!dir, 'the chat has a scratch folder, so the tool call has somewhere legal to point', dir);
123await chat(s, `@narrate ${LONG} ;; file_list {"path":"${dir}"}`, { timeout: 60000 });
124
125line('1. the words are still there');
126const live = await transcript(s);
127check(live.includes('INTERIM-PROSE-MARKER'),
128 'the prose the reader watched arrive is still on screen after the answer superseded it');
129check(live.includes('Narration done'), 'and so is the answer');
130
131line('2. and the control says there is something to open');
132const seen = await s.page.evaluate(() => {
133 const w = document.querySelector('.chat-msg-working');
134 if (!w) return { there: false };
135 const d = w.querySelector('details');
136 const sum = d && d.querySelector('summary');
137 const more = sum && sum.querySelector('.chat-working-more');
138 const body = d && d.querySelector('.chat-thinking-body');
139 // The summary's own words, without the count that sits at the end of it.
140 const label = sum ? String(sum.textContent || '')
141 .replace(more ? String(more.textContent || '') : '', '').trim() : '';
142 return {
143 there: true,
144 label: label,
145 more: more ? String(more.textContent || '').trim() : '',
146 moreBox: more ? more.getBoundingClientRect().width : 0,
147 body: body ? String(body.textContent || '') : '',
148 open: d ? d.open : null,
149 };
150});
151check(seen.there, 'the run of prose was demoted to working, which is the shape this is about');
152check((seen.body || '').length > 0,
153 'the rest of it is behind the control rather than gone', `${(seen.body || '').length} characters`);
154
155// THE CUT, which is the defect. The code's own comment says "never mid-word"; the branch this
156// fixture reaches did exactly that. A label that ends inside a word is the failure, and it is
157// asked of the label rather than of the source.
158const ends = (seen.label || '').slice(-1);
159const wholeWord = !!seen.label && (
160 /[.!?]$/.test(seen.label) // cut on a sentence end
161 || !LONG.slice(seen.label.length, seen.label.length + 1).match(/\S/));
162check(wholeWord,
163 'the summary never ends in the middle of a word',
164 `ends "${(seen.label || '').slice(-28)}" (next character in the source: `
165 + JSON.stringify(LONG.slice((seen.label || '').length, (seen.label || '').length + 1)) + ')');
166check(/\d/.test(seen.more || '') && seen.moreBox > 0,
167 'and it says how much went behind it, so the control reads as one',
168 `"${seen.more}" at ${seen.moreBox}px`);
169check(seen.open === false,
170 'it starts closed, because the working is not the answer');
171
172line('3. nothing was lost, which is what the store has to show');
173const stored = await s.page.evaluate(() => new Promise((res) => {
174 const req = indexedDB.open('daimond-chats', 1);
175 req.onsuccess = () => {
176 const db = req.result; let t;
177 try { t = db.transaction('chats', 'readonly'); } catch (e) { res([]); return; }
178 const out = []; const cur = t.objectStore('chats').openCursor();
179 cur.onsuccess = () => { const c = cur.result; if (c) { out.push(c.value); c.continue(); } else res(out); };
180 cur.onerror = () => res(out);
181 };
182 req.onerror = () => res([]);
183}));
184const asst = stored.flatMap(c => c.messages || []).filter(m => m.role === 'assistant');
185check(asst.some(m => String(m.content || '').includes('INTERIM-PROSE-MARKER')),
186 'the working is stored as part of the turn, so it was never the app\'s to lose',
187 `${asst.length} assistant message(s) stored`);
188
189await s.page.reload({ waitUntil: 'domcontentloaded' });
190await s.page.waitForTimeout(1200);
191await signInAs(s, NAME);
192await s.page.waitForTimeout(1500);
193await s.page.evaluate(() => {
194 const b = document.querySelector('#session-list .chat-box.active')
195 || document.querySelector('#session-list .chat-box');
196 if (b) b.click();
197});
198await s.page.waitForTimeout(1200);
199const after = await transcript(s);
200check(after.includes('INTERIM-PROSE-MARKER'),
201 'and a reader who comes back tomorrow can still read it');
202
203await shot(s, 'working-final');
204const errs = s.errs.filter(e => !/favicon|manifest|502|Bad Gateway/i.test(e));
205check(errs.length === 0, 'no unexpected console errors', errs.slice(0, 3).join(' | '));
206await s.close();
207
208log(`\n${failures === 0 ? 'ALL CHECKS PASSED' : failures + ' CHECK(S) FAILED'}`);
209process.exit(failures === 0 ? 0 : 1);