Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_ctxwhole.mjs

17.7 KiB, 1 run

created by r2519314175:339, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// Is the whole conversation fed back to the model, and is the user told when it is not?
2//
3// Note 06 of the 2026-08-27 tester round, verbatim: *"Serious: all of a conversation is not
4// being fed back into context in long chats. Easily tested, just ask a model if it is
5// receiving some of its first response in a long transcript."*
6//
7// That is a test, so it is run here rather than reasoned about. A first reply carrying a
8// unique word, an ordinary conversation on top of it, and then the MOCK'S OWN LOG read back
9// to see whether the word is still in the request the model was sent. Nothing here asks the
10// app what it did.
11//
12// What the answer turned out to be, and why the checks are shaped as they are. Nothing was
13// dropping messages: twenty turns reach the model whole, and so do twenty turns reloaded from
14// the store. A long enough conversation FOLDS, which is `src/compact.rs` working as designed
15// and paid for, and the app announced it in one sentence at the bottom of the thread -- while
16// leaving every folded message on screen above it, at full fidelity, drawn exactly like the
17// ones the model still has. So the reader scrolls up, reads their first answer, asks about
18// it, is told it is gone, and concludes the app is losing the conversation.
19//
20// The defect is therefore the disclosure and not the fold, and these checks are about the
21// LINE: it is drawn, it says which side of it the model still has, it survives a reload, and
22// it does not dim the words above it.
23//
24// node dev/verify_ctxwhole.mjs
25// node dev/verify_ctxwhole.mjs --break noline # the notice with no boundary
26// node dev/verify_ctxwhole.mjs --break nomark # the boundary with nothing marked above
27// node dev/verify_ctxwhole.mjs --break dimmed # the mark, but the ink taken out of it
28// node dev/verify_ctxwhole.mjs --break oldfold # the fold fraction back at 0.8
29
30import fs from 'node:fs';
31import path from 'node:path';
32import { fileURLToPath } from 'node:url';
33
34const HERE = path.dirname(fileURLToPath(import.meta.url));
35const ROOT = path.dirname(HERE);
36const H = await import(path.join(HERE, 'harness.mjs'));
37const { open, newChat, chat, transcript, shot, connectMock, mockLog, clearMockLog,
38 contentText, signInAs } = H;
39
40const BREAK = (() => {
41 const i = process.argv.indexOf('--break');
42 return i === -1 ? '' : (process.argv[i + 1] || '');
43})();
44const NAME = 'ctxwhole';
45const MARK = 'FIRSTREPLY-Q7X2';
46const WWW = path.join(ROOT, 'www');
47
48// ── Every check is proved against broken code first ──────────────────────────
49//
50// A break serves ONE damaged file to the browser -- or, for the two figures read off disk,
51// damages the text this file reads -- and the run is then expected to FAIL. A break whose
52// anchor is gone stops the run rather than passing quietly, because a break that breaks
53// nothing proves nothing.
54const BREAKS = {
55 // The notice goes back to being one more message at the bottom of the thread.
56 noline: [{
57 file: 'js/daimond.js',
58 find: '\t\tvar isFold = (folded | 0) > 0;',
59 with: '\t\tvar isFold = false;',
60 }],
61 // The line is drawn and nothing above it is marked, so the region it names is unfindable.
62 nomark: [{
63 file: 'js/daimond.js',
64 find: "\t\t\tn.classList.add('chat-above-fold');",
65 with: '\t\t\tif (false) n.classList.add(\'chat-above-fold\');',
66 }],
67 // The mark, with the ink taken out of the words it marks.
68 dimmed: [{
69 file: 'css/app.css',
70 find: '.chat-above-fold {\n\tborder-left: 2px solid var(--border-2);',
71 with: '.chat-above-fold {\n\topacity: 0.45;\n\tborder-left: 2px solid var(--border-2);',
72 }],
73 // The engine folds where it used to.
74 oldfold: [{
75 file: '../src/compact.rs',
76 find: 'pub const FOLD_AT: f64 = 0.65;',
77 with: 'pub const FOLD_AT: f64 = 0.8;',
78 }],
79 // The browser's copy of the figure drifts from the engine's, so the meter marks the fold
80 // in a place the engine does not fold at.
81 jsdrift: [{
82 file: 'js/daimond.js',
83 find: 'var DEFAULT_FOLD_AT = 0.65;',
84 with: 'var DEFAULT_FOLD_AT = 0.8;',
85 }],
86};
87if (BREAK && !BREAKS[BREAK]) {
88 console.error(`unknown break '${BREAK}'; one of: ${Object.keys(BREAKS).join(', ')}`);
89 process.exit(2);
90}
91
92/// `src` with `spec` applied, or a hard stop.
93function damaged(src, spec) {
94 const n = src.split(spec.find).length - 1;
95 if (n !== 1) {
96 console.error(`break '${BREAK}': the anchor appears ${n} times in ${spec.file}, `
97 + 'so nothing was broken and the run below would prove nothing.');
98 process.exit(2);
99 }
100 return src.replace(spec.find, spec.with);
101}
102
103const _bodies = new Map();
104for (const spec of (BREAKS[BREAK] || [])) {
105 const src = _bodies.has(spec.file) ? _bodies.get(spec.file)
106 : fs.readFileSync(path.join(WWW, spec.file), 'utf8');
107 _bodies.set(spec.file, damaged(src, spec));
108}
109
110/// A file as the browser will see it, damaged where a break says so.
111const srcOf = (f) => _bodies.has(f) ? _bodies.get(f)
112 : fs.readFileSync(path.join(WWW, f), 'utf8');
113
114const TYPE = { '.html': 'text/html', '.css': 'text/css', '.js': 'application/javascript' };
115async function serveBreaks(page) {
116 for (const [file, body] of _bodies) {
117 if (file.startsWith('..')) continue; // not served; read off disk above
118 const type = TYPE[path.extname(file)] || 'text/plain';
119 await page.route('**/' + file, r => r.fulfill({ status: 200, contentType: type, body }));
120 }
121}
122
123let failures = 0;
124const log = (...a) => console.log(...a);
125const line = (t) => log('\n════════ ' + t + ' ════════');
126const check = (ok, what, detail = '') => {
127 log((ok ? ' PASS ' : ' FAIL ') + what + (detail ? ' -- ' + detail : ''));
128 if (!ok) failures++;
129};
130
131/// The last thing the model was actually sent.
132const lastSent = () => {
133 const rows = mockLog();
134 return (rows[rows.length - 1] || {}).messages || [];
135};
136const carries = (msgs, role, word) =>
137 msgs.some(m => m.role === role && contentText(m.content).includes(word));
138
139// ── The two figures, read from the source rather than written down twice ──────
140//
141// `compact::FOLD_AT` is the engine's authority and `DEFAULT_FOLD_AT` is the browser's copy of
142// it, used to draw the meter's mark before an agent exists. A test that pinned only one of
143// them would pass while the meter marked a place the engine does not fold at.
144const rustFoldAt = (() => {
145 const m = /pub const FOLD_AT:\s*f64\s*=\s*([0-9.]+)/.exec(srcOf('../src/compact.rs'));
146 return m ? Number(m[1]) : NaN;
147})();
148const jsFoldAt = (() => {
149 const m = /var DEFAULT_FOLD_AT = ([0-9.]+)/.exec(srcOf('js/daimond.js'));
150 return m ? Number(m[1]) : NaN;
151})();
152
153line('0. where the conversation folds');
154check(rustFoldAt === 0.65,
155 'the engine folds at the owner\'s figure', `compact::FOLD_AT = ${rustFoldAt}`);
156check(jsFoldAt === rustFoldAt,
157 'and the browser draws the mark in the same place',
158 `DEFAULT_FOLD_AT = ${jsFoldAt} against compact::FOLD_AT = ${rustFoldAt}`);
159
160clearMockLog();
161const s = await open({ name: NAME, connect: false,
162 profile: H.scratch('pw', 'ctxwhole' + (BREAK ? '-' + BREAK : '')),
163 route: serveBreaks });
164await connectMock(s);
165await newChat(s);
166
167// ── 1. The owner's own test ──────────────────────────────────────────────────
168line('1. a long conversation, under the window');
169await chat(s, `@text ${MARK} this is the very first answer in the conversation.`);
170for (let i = 1; i <= 12; i++) await chat(s, `@text turn ${i} answer, nothing special.`);
171
172let sent = lastSent();
173check(carries(sent, 'user', MARK),
174 'the first thing the user said is still in the request', `${sent.length} messages sent`);
175check(carries(sent, 'assistant', MARK),
176 'AND SO IS THE MODEL\'S FIRST REPLY, which is the question the tester asked',
177 `${sent.length} messages sent`);
178check(!/[Ff]olded \d+ earlier/.test(await transcript(s)),
179 'a conversation this size did not need folding, so nothing was folded');
180
181// ── 2. And it survives a reload ──────────────────────────────────────────────
182line('2. after a reload');
183await s.page.reload({ waitUntil: 'domcontentloaded' });
184await s.page.waitForTimeout(1200);
185await signInAs(s, NAME);
186await s.page.waitForTimeout(1500);
187await s.page.evaluate(() => {
188 const b = document.querySelector('#session-list .chat-box.active')
189 || document.querySelector('#session-list .chat-box');
190 if (b) b.click();
191});
192await s.page.waitForTimeout(1200);
193await chat(s, '@text after the reload.');
194sent = lastSent();
195check(carries(sent, 'assistant', MARK),
196 'the model\'s first reply survives a reload and is sent again',
197 `${sent.length} messages sent`);
198
199// ── 3. Force a real fold, and read the line it draws ─────────────────────────
200line('3. a conversation big enough to fold');
201const BULK = 'lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod. ';
202const chunk = BULK.repeat(900); // about 63 KB of user text per turn
203let folded = false;
204for (let i = 1; i <= 12 && !folded; i++) {
205 await chat(s, `bulk ${i}: ${chunk}`, { timeout: 90000 });
206 folded = /[Ff]olded \d+ earlier/.test(await transcript(s));
207 log(` bulk ${i}: ${lastSent().length} messages sent, folded=${folded}`);
208}
209check(folded, 'the conversation folded, which is the state this is about');
210
211sent = lastSent();
212check(sent.some(m => /Daimond folded/.test(contentText(m.content))),
213 'the fold notice reached the model, so the summary really is standing in');
214check(!carries(sent, 'assistant', MARK),
215 'and the first reply is genuinely no longer on the wire — the fold is real, not cosmetic');
216
217const seen = await transcript(s);
218check(seen.includes(MARK),
219 'while the SCREEN still shows it, which is exactly why the line has to be drawn');
220
221// One ordinary turn AFTER the fold, so there is a message on each side of the line and the
222// ink above it can be compared with the ink below it. Without it the fold is the last thing
223// in the thread and the comparison has nothing to compare against.
224await chat(s, '@text a short turn after the fold.', { timeout: 60000 });
225
226// ── 4. The line ──────────────────────────────────────────────────────────────
227line('4. what the reader is shown');
228const drawn = await s.page.evaluate(() => {
229 const out = document.getElementById('chat-output');
230 // THE LAST LINE, not the first. A conversation this size folds again on the turn after,
231 // and everything between two folds is above the SECOND one -- correctly marked, and below
232 // the first. Measuring from the first read that as the mark running past its own line.
233 const lines = [...out.querySelectorAll('.chat-msg-compacted.chat-fold-line')];
234 const fold = lines[lines.length - 1] || null;
235 const says = fold && fold.querySelector('.chat-fold-boundary');
236 const above = [...out.querySelectorAll('.chat-above-fold')];
237 // The rule itself is a ::before, so it is measured rather than looked for in the markup.
238 const rule = fold ? getComputedStyle(fold, '::before').borderTopWidth : '';
239 // WHAT THE MARK DOES TO THE INK, measured by taking the mark off and putting it back
240 // rather than by finding a second message to compare against. A conversation that folded
241 // on its last turn has nothing below the line to compare with, and a check that needs one
242 // is a check that goes blank exactly when the fold is freshest.
243 const ink = (n) => {
244 if (!n) return null;
245 const cs = getComputedStyle(n);
246 return { colour: cs.color, opacity: cs.opacity };
247 };
248 const aboveMsg = out.querySelector('.chat-above-fold.chat-msg-user');
249 let unmarkedInk = null;
250 if (aboveMsg) {
251 aboveMsg.classList.remove('chat-above-fold');
252 unmarkedInk = ink(aboveMsg);
253 aboveMsg.classList.add('chat-above-fold');
254 }
255 let belowMsg = null, markedBelow = 0;
256 if (fold) {
257 let n = fold.nextElementSibling;
258 while (n) {
259 // The queue box and the turn indicator sit permanently below the line and are
260 // furniture rather than conversation, so they are not counted either way.
261 if (n.id !== 'chat-queued' && !n.classList.contains('chat-spinner')
262 && n.classList.contains('chat-above-fold')) markedBelow++;
263 if (!belowMsg && n.classList.contains('chat-msg-user')) belowMsg = n;
264 n = n.nextElementSibling;
265 }
266 }
267 return {
268 hasFold: !!fold,
269 says: says ? (says.textContent || '') : '',
270 above: above.length,
271 rule: rule,
272 aboveInk: ink(aboveMsg),
273 belowInk: ink(belowMsg) || unmarkedInk,
274 markedBelow,
275 foldBox: fold ? fold.getBoundingClientRect().height : 0,
276 lines: lines.length,
277 };
278});
279check(drawn.hasFold, 'the fold notice is drawn as a boundary, not as one more message');
280check(drawn.foldBox > 0, 'and it is actually on screen', `height ${drawn.foldBox}`);
281check(/^\d/.test(drawn.rule) && parseFloat(drawn.rule) > 0,
282 'a rule is drawn across the thread at the fold', `border-top-width ${drawn.rule || '(none)'}`);
283check(drawn.says.length > 40 && /above this line/i.test(drawn.says),
284 'and it says which side of the line the model still has',
285 drawn.says.slice(0, 140));
286check(drawn.above > 0,
287 'the messages the model no longer holds verbatim are marked',
288 `${drawn.above} marked`);
289// AND THE MARK STOPS AT THE LINE. A mark that ran on past it would be the app saying the
290// model has lost something it is holding, which is the same fault as the silence, pointed the
291// other way.
292check(drawn.markedBelow === 0,
293 'and nothing below the line is marked, because the model still has all of it',
294 `${drawn.markedBelow} marked below, ${drawn.lines} fold line(s) in the thread`);
295// THE INK IS NOT TAKEN OUT. Dimming the words above the line would be the app hiding a
296// conversation it has just been accused of losing, so the mark is a rule down the side and
297// the colour is measured to prove it.
298check(!!drawn.aboveInk && !!drawn.belowInk
299 && drawn.aboveInk.colour === drawn.belowInk.colour
300 && drawn.aboveInk.opacity === drawn.belowInk.opacity,
301 'and they are marked without being dimmed — the words stay as readable as the rest',
302 JSON.stringify({ above: drawn.aboveInk, below: drawn.belowInk }));
303
304// ── 5. And the line survives a reload ────────────────────────────────────────
305line('5. the line after a reload');
306await s.page.reload({ waitUntil: 'domcontentloaded' });
307await s.page.waitForTimeout(1200);
308await signInAs(s, NAME);
309await s.page.waitForTimeout(1500);
310await s.page.evaluate(() => {
311 const b = document.querySelector('#session-list .chat-box.active')
312 || document.querySelector('#session-list .chat-box');
313 if (b) b.click();
314});
315await s.page.waitForTimeout(1500);
316const again = await s.page.evaluate(() => {
317 const out = document.getElementById('chat-output');
318 return {
319 fold: !!out.querySelector('.chat-msg-compacted.chat-fold-line'),
320 says: (out.querySelector('.chat-fold-boundary') || {}).textContent || '',
321 above: out.querySelectorAll('.chat-above-fold').length,
322 };
323});
324check(again.fold && again.above > 0 && /above this line/i.test(again.says),
325 'a reader who comes back tomorrow is shown the same line in the same place',
326 JSON.stringify({ fold: again.fold, above: again.above }));
327
328// ── 6. The setting ───────────────────────────────────────────────────────────
329line('6. the fold point is the user\'s to set');
330const row = await s.page.evaluate(() => {
331 const sel = document.getElementById('cfg-fold-at');
332 if (!sel) return { there: false };
333 return {
334 there: true,
335 options: [...sel.options].map(o => o.value + ':' + o.textContent),
336 value: sel.value,
337 };
338});
339check(row.there, 'there is a control for it, beside the round limit', JSON.stringify(row.options || []));
340if (row.there) {
341 check((row.options[0] || '').indexOf(String(Math.round(jsFoldAt * 100)) + '%') !== -1,
342 'whose Default row names the figure actually in force', row.options[0]);
343 // Chosen the way a user chooses, and read back from where it is kept.
344 const stored = await s.page.evaluate(() => {
345 const sel = document.getElementById('cfg-fold-at');
346 sel.value = '50';
347 sel.dispatchEvent(new Event('change'));
348 return localStorage.getItem('daimond-byok');
349 });
350 let f = null;
351 try { f = JSON.parse(stored).foldAt; } catch (e) { /* unreadable is a fail below */ }
352 check(f === 0.5, 'and a choice is kept as the fraction the engine takes', `foldAt = ${f}`);
353}
354// The engine's own end of it, since a setting nothing can apply is not a setting.
355const engine = await s.page.evaluate(async () => {
356 const w = await import('/pkg/oxedyne_daimond.js');
357 if (typeof w.DaimondApp !== 'function') return { built: false };
358 const a = new w.DaimondApp('http://127.0.0.1/v1/chat/completions', '', 'none', 256, 'x', true);
359 const before = a.fold_at;
360 if (typeof a.set_fold_at !== 'function') return { built: true, setter: false, before };
361 a.set_fold_at(0.5);
362 const after = a.fold_at;
363 a.set_fold_at(0); // zero is "the user has not chosen"
364 return { built: true, setter: true, before, after, afterZero: a.fold_at };
365});
366check(engine.setter === true, 'the engine takes a fold fraction from the browser',
367 JSON.stringify(engine));
368check(Math.abs((engine.before || 0) - jsFoldAt) < 1e-9,
369 'a fresh agent folds where the browser says it does', `${engine.before} against ${jsFoldAt}`);
370check(engine.after === 0.5 && engine.afterZero === 0.5,
371 'a chosen fraction is taken, and a zero leaves it alone rather than folding to nothing',
372 JSON.stringify(engine));
373
374await shot(s, 'ctxwhole-final');
375const errs = s.errs.filter(e => !/favicon|manifest|502|Bad Gateway/i.test(e));
376check(errs.length === 0, 'no unexpected console errors', errs.slice(0, 3).join(' | '));
377await s.close();
378
379log(`\n${failures === 0 ? 'ALL CHECKS PASSED' : failures + ' CHECK(S) FAILED'}`);
380process.exit(failures === 0 ? 0 : 1);