Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_concise.mjs

7.8 KiB, 1 run

created by r2519314175:305, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_concise.mjs — the chip changes what the MODEL is sent.
2//
3// The chip in the chat header is a standing toggle that puts the `/concise`
4// skill in front of every turn the chat sends. Whether the input box changed
5// proves nothing: the property is about the request, so every assertion here
6// reads the mock provider's own log (`DAIMOND_MOCK_LOG`) and looks at the user
7// message the provider actually received.
8//
9// Four properties:
10// 1. Off, the turn carries exactly what was typed.
11// 2. On, the turn carries the SKILL'S OWN TEXT — not the string "/concise",
12// which would mean the wasm resolved nothing and the model was handed a
13// slash it cannot act on.
14// 3. The chip is per chat and survives a reload.
15// 4. A message the user began with their own `/command` is left alone.
16//
17// Gated: the skill file has to be on disk, and the mock has to have been
18// reached at all, before any of the above can mean anything.
19//
20// node dev/verify_concise.mjs
21//
22// Needs dev/serve.mjs and dev/mockllm.mjs (dev/world.sh N --up gives both).
23import fs from 'node:fs';
24import path from 'node:path';
25import os from 'node:os';
26import { open, newChat, scratch, mockLog, clearMockLog } from './harness.mjs';
27
28const OUT = path.join(os.homedir(), '.cache/daimond/concise-shots');
29fs.mkdirSync(OUT, { recursive: true });
30
31let failures = 0;
32const check = (cond, msg, detail) => {
33 console.log((cond ? ' ok ' : ' FAIL ') + msg + (detail != null ? ' — ' + detail : ''));
34 if (!cond) failures++;
35};
36
37/// The last user message the provider was actually sent.
38function lastUserSent() {
39 const rows = mockLog();
40 for (let i = rows.length - 1; i >= 0; i--) {
41 const msgs = rows[i].messages || [];
42 for (let j = msgs.length - 1; j >= 0; j--) {
43 if (msgs[j].role === 'user') return String(msgs[j].content || '');
44 }
45 }
46 return null;
47}
48
49const s = await open({ name: 'concise', profile: scratch('pw', 'concise-' + process.pid) });
50const { page } = s;
51try {
52 await newChat(s);
53
54 // ── 0. The gate: the mock must be reachable, or every assertion below is
55 // about a file that was never written. ──
56 clearMockLog();
57 await page.fill('#chat-input', '@text hello');
58 await page.click('#chat-send');
59 await page.waitForTimeout(2500);
60 check(lastUserSent() !== null, 'the mock provider was reached',
61 lastUserSent() === null ? 'nothing in the mock log — nothing below can be judged' : 'yes');
62 if (lastUserSent() === null) { await s.close(); process.exit(1); }
63
64 // ── 1. Off: what was typed is what was sent ──
65 check(lastUserSent() === '@text hello', 'with the chip off, the turn is what was typed',
66 JSON.stringify(lastUserSent()));
67
68 const chipState = () => page.evaluate(() => {
69 const c = document.getElementById('concise-chip');
70 if (!c) return null;
71 return { shown: c.getClientRects().length > 0, pressed: c.getAttribute('aria-pressed') };
72 });
73 // Which chat this is, so the check below can come BACK to it rather than to
74 // whichever tile happens to be first once a second chat exists.
75 const firstChatId = await page.evaluate(() => {
76 const box = document.querySelector('#session-list .session-box');
77 return box ? box.dataset.id : null;
78 });
79
80 const before = await chipState();
81 check(before && before.shown, 'the chip is in the chat header', JSON.stringify(before));
82 check(before && before.pressed === 'false', 'and it starts off', before && before.pressed);
83
84 // ── 2. On: the chip lights, and the skill file appears ──
85 await page.evaluate(() => document.getElementById('concise-chip').click());
86 await page.waitForTimeout(1500);
87 const lit = await chipState();
88 check(lit && lit.pressed === 'true', 'pressing it lights it', lit && lit.pressed);
89
90 // Read the file the way the panel does, not the way the model does.
91 const skillText = await page.evaluate(async () => {
92 try {
93 const m = await import('./pkg/oxedyne_daimond.js');
94 return await m.read_file('.daimond/skills/concise.md');
95 } catch (e) { return 'ERR ' + String(e); }
96 });
97 check(typeof skillText === 'string' && /name:\s*concise/.test(skillText),
98 'the skill file is on disk', String(skillText).slice(0, 40).replace(/\n/g, '⏎'));
99
100 // ── 3. The turn carries the SKILL'S TEXT, not the slash ──
101 clearMockLog();
102 await page.fill('#chat-input', '@text second');
103 await page.click('#chat-send');
104 await page.waitForTimeout(3000);
105 const sent = lastUserSent();
106 check(sent !== null && sent !== '@text second',
107 'with the chip on, the turn is NOT what was typed', JSON.stringify(String(sent).slice(0, 60)));
108 check(sent !== null && /Skill 'concise'/.test(sent),
109 'the skill was resolved into the turn, not left as a slash',
110 JSON.stringify(String(sent).slice(0, 70)));
111 check(sent !== null && /as few words as carry the answer/i.test(sent),
112 'the skill FILE\'S OWN WORDS reached the model');
113 check(sent !== null && /@text second/.test(sent),
114 'and what the user typed is still in it');
115
116 // ── 4. The choice survives a reload ──
117 await page.reload({ waitUntil: 'domcontentloaded' });
118 await page.waitForSelector('#id-primary', { timeout: 15000 }).catch(() => {});
119 if (await page.$('#id-pass')) {
120 await page.fill('#id-pass', 'testpass1234');
121 await page.evaluate(() => document.getElementById('id-primary').click());
122 await page.waitForSelector('#identity-modal', { state: 'hidden', timeout: 15000 }).catch(() => {});
123 }
124 await page.waitForTimeout(2500);
125 const afterReload = await chipState();
126 check(afterReload && afterReload.pressed === 'true',
127 'the chip is still lit after a reload', afterReload && JSON.stringify(afterReload));
128
129 // ── 5. It is per CHAT, not per app ──
130 await page.evaluate(() => { const b = document.getElementById('admin-close'); if (b) b.click(); });
131 await page.waitForTimeout(200);
132 await page.evaluate(() => document.getElementById('new-session-btn').click());
133 await page.waitForTimeout(600);
134 await page.evaluate(() => {
135 const start = [...document.querySelectorAll('.tile-start')].pop();
136 if (start) start.click();
137 });
138 await page.waitForTimeout(1200);
139 const onNewChat = await chipState();
140 check(onNewChat && onNewChat.pressed === 'false',
141 'a different chat does not inherit it', onNewChat && JSON.stringify(onNewChat));
142
143 // ── 6. The user's own command wins ──
144 // Back to the lit chat, then type a slash command of their own.
145 await page.evaluate((id) => {
146 const box = document.querySelector('#session-list .session-box[data-id="' + id + '"]');
147 if (box) box.click();
148 }, firstChatId);
149 await page.waitForTimeout(800);
150 const backLit = await chipState();
151 check(backLit && backLit.pressed === 'true', 'and the first chat still has it lit',
152 backLit && JSON.stringify(backLit));
153 clearMockLog();
154 await page.fill('#chat-input', '/nosuchskill please');
155 await page.click('#chat-send');
156 await page.waitForTimeout(2500);
157 const afterSlash = lastUserSent();
158 // The engine refuses an unknown skill BEFORE the provider is reached, so the
159 // property is that nothing new was sent — and in particular that `/concise`
160 // was not stapled in front of the user's own command.
161 check(afterSlash === null || !/Skill 'concise'/.test(afterSlash),
162 'a message the user began with a slash is left alone',
163 afterSlash === null ? 'nothing reached the provider (the unknown skill was refused)'
164 : JSON.stringify(String(afterSlash).slice(0, 60)));
165
166 await page.screenshot({ path: path.join(OUT, 'chip-lit.png'),
167 clip: await page.evaluate(() => {
168 const h = document.querySelector('#panel-ai .chead');
169 const r = h.getBoundingClientRect();
170 return { x: Math.round(r.x), y: Math.round(r.y), width: Math.round(r.width), height: Math.round(r.height) };
171 }) });
172 check(fs.existsSync(path.join(OUT, 'chip-lit.png')), 'the header screenshot is on disk');
173} finally {
174 await s.close();
175}
176
177console.log(failures === 0
178 ? `\nconcise: the chip changes the turn, not just the box. Shots in ${OUT}`
179 : `\nconcise: ${failures} failure(s). Shots in ${OUT}`);
180process.exit(failures === 0 ? 0 : 1);