Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_chatscope.mjs

15.6 KiB, 1 run

created by r2519314175:281, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_chatscope.mjs — an ordinary chat has a WORKSPACE, and it is a real
2// boundary that never interrupts anybody.
3//
4// On 2026-08-11 a daimon in an ordinary chat edited two files of the user's own
5// book — `thinking.typ` and `config.typ` — to work around a compiler
6// limitation, in a directory under no version control, and put them back only
7// because it chose to. No worker was involved, so the worker fence that existed
8// could not have helped: `chat_bounds` was applied to dispatched workers alone
9// and the conversation itself carried no bounds at all.
10//
11// The remedy the author asked for is a WORKING DIRECTORY and not a permission
12// dialog — Claude Code run on bypassed permissions, where the friction is paid
13// once at the `cd` and nothing inside interrupts you. So:
14//
15// * A CHAT HAS A WORKSPACE, the set of folders the user marked into it, and it
16// is fenced to it FOR WRITING AND RUNNING. Reading is free inside whatever
17// the user already opened (amended 2026-08-13; this file asserted the
18// opposite for a day, and `dev/verify_chatfence.mjs` asserted this one, and
19// they cannot both be right).
20// * THE MARK IS THE PERMISSION, and what it grants is WRITING. Inside the
21// workspace a write happens with nothing asked. A fence that also
22// interrupted would have missed the point, so the control below is as
23// load-bearing as the refusals.
24// * NOTE AND READ ARE NOT THE MARK. They are a cost decision about what is
25// quoted into the prompt — a path costs a few tokens, a file costs
26// thousands — and neither grants any reach. A path attached as Read and NOT
27// marked into the workspace is readable, as everything is, and cannot be
28// changed.
29// * AN EMPTY WORKSPACE IS THE CHAT'S OWN SCRATCH AND NOTHING ELSE TO WRITE IN,
30// which is what a Diamond with no attachment gets.
31//
32// EACH CHECK IS PROVED AGAINST BROKEN CODE FIRST.
33//
34// node dev/verify_chatscope.mjs --break unscoped # the state before this change
35// node dev/verify_chatscope.mjs --break allattached # Note becomes a grant
36// node dev/verify_chatscope.mjs --break readonlyrw # 'consult' becomes 'edit'
37// node dev/verify_chatscope.mjs --break nowrite # the workspace is read-only
38// node dev/verify_chatscope.mjs # and then, clean
39//
40// The breaks are applied to the SCOPE THE PAGE ASKS FOR, never to the engine:
41// the engine is the thing under test, and a break that damaged it would prove
42// only that a damaged engine misbehaves. Each one is a plausible caller mistake,
43// and `allattached` is the one this design turns on — it is what a caller does
44// who reads the paperclip's list as a permission list.
45import { open } from './harness.mjs';
46
47const BREAK = (() => {
48 const i = process.argv.indexOf('--break');
49 return i > 0 ? String(process.argv[i + 1] || '') : '';
50})();
51
52const ok = [], bad = [];
53const check = (name, pass, detail) => {
54 (pass ? ok : bad).push(name + (detail ? ' — ' + detail : ''));
55 console.log((pass ? ' ok ' : ' FAIL ') + name + (detail ? ' — ' + detail : ''));
56};
57
58const s = await open({ name: 'chatscope', signIn: true, connect: false });
59const { page } = s;
60await page.waitForFunction(() => !!window.DaimondCore, null, { timeout: 15000 }).catch(() => {});
61
62try {
63 // ── The instrument, installed before anything is driven ──
64 //
65 // "Nothing asked" is a claim about an ABSENCE, and an absence is what a
66 // broken detector reports too. So the counter is installed first, proved at
67 // the end against a dialog put up on purpose, and only then believed.
68 // `.modal.dlg` is what `confirmDialog` builds (daimond.js:5762); the other
69 // selectors are there so a future dialog of a different shape is still seen.
70 await page.evaluate(() => {
71 window.__asked = { dialogs: 0, confirms: 0, prompts: 0 };
72 window.confirm = function () { window.__asked.confirms++; return true; };
73 window.prompt = function () { window.__asked.prompts++; return ''; };
74 var sel = '.modal, .dlg, [role="dialog"], dialog';
75 var isDialog = function (n) {
76 if (!n || n.nodeType !== 1) return false;
77 return (n.matches && n.matches(sel)) || (n.querySelector && !!n.querySelector(sel));
78 };
79 new MutationObserver(function (ms) {
80 ms.forEach(function (m) {
81 Array.prototype.forEach.call(m.addedNodes, function (n) {
82 if (isDialog(n)) window.__asked.dialogs++;
83 });
84 });
85 }).observe(document.documentElement, { childList: true, subtree: true });
86 });
87 const asked = () => page.evaluate(() => JSON.parse(JSON.stringify(window.__asked)));
88 const resetAsked = () => page.evaluate(() => {
89 window.__asked.dialogs = 0; window.__asked.confirms = 0; window.__asked.prompts = 0;
90 });
91
92 // Lay down the user's files through an UNSCOPED app — which is what a chat
93 // was until this change, and which is also the reader used below to prove a
94 // refusal really left a file alone. `papers` is marked into the workspace;
95 // `books` and `elsewhere` are not; `refs` is marked read-only; `quoted` is
96 // attached as Read and marked into nothing.
97 await page.evaluate(async () => {
98 const mod = await import('../pkg/oxedyne_daimond.js');
99 const app = new mod.DaimondApp('http://127.0.0.1/v1/chat/completions', '', 'none', 256, '', true);
100 window.__free = app;
101 const put = (path, content) => app.run_tool('file_write', JSON.stringify({ path, content }));
102 await put('papers/spec.md', 'the spec as the user left it\n');
103 await put('books/thinking.typ', 'the user own chapter\n');
104 await put('elsewhere/notes.md', 'the user own note\n');
105 await put('refs/handbook.md', 'a reference to consult\n');
106 await put('quoted/passage.md', 'a passage quoted into the chat\n');
107 // Laid down so the deny below is refused for being DENIED rather than for
108 // being absent. A refusal and a missing file read the same to a check on
109 // the reply text, and only one of them is the rule under test.
110 await put('.daimond/config.json', '{"seeded":true}\n');
111 });
112
113 /// A chat's own app, scoped as the page will scope one.
114 ///
115 /// `ws` is what the user MARKED INTO THE WORKSPACE; `ro` is the read-only
116 /// part of it; `noted` is attached-but-not-marked — Note or Read, which are
117 /// about cost and not about reach. Only the first two may reach the engine.
118 const chatApp = async (ws, ro, noted, brk) => await page.evaluate(async (a) => {
119 const mod = await import('../pkg/oxedyne_daimond.js');
120 const app = new mod.DaimondApp('http://127.0.0.1/v1/chat/completions', '', 'none', 256, '', true);
121 window.__app = app;
122 let scratch = 'chats/c-scope/work';
123 let ws = a.ws.slice(), ro = a.ro.slice();
124 // The state this change replaced: the chat's own turn was never scoped at
125 // all. Nothing is set, and the app keeps the reach of the whole workspace.
126 if (a.brk === 'unscoped') return JSON.parse(app.diamond_scope() || '{}');
127 // The mistake this design turns on: reading the paperclip's whole list as a
128 // permission list, so a path attached only to be quoted becomes reachable.
129 if (a.brk === 'allattached') ws = ws.concat(a.noted);
130 // 'Consult, do not edit' handed over as ordinary workspace.
131 if (a.brk === 'readonlyrw') { ws = ws.concat(ro); ro = []; }
132 // And the opposite slip: the workspace handed over as read-only, so the
133 // chat can look at the folder it was given and change nothing in it.
134 if (a.brk === 'nowrite') { ro = ro.concat(ws); ws = []; }
135 app.set_chat_scope(scratch, JSON.stringify(ws), JSON.stringify(ro));
136 return JSON.parse(app.diamond_scope() || '{}');
137 }, { ws, ro, noted, brk });
138
139 const tool = (name, args) => page.evaluate(
140 ({ name, args }) => window.__app.run_tool(name, JSON.stringify(args)).then(String),
141 { name, args });
142 /// What is actually on disk, read by the unscoped app — so "the refusal is
143 /// real" is asserted against the file rather than against the reply text.
144 const onDisk = (path) => page.evaluate(
145 (p) => window.__free.run_tool('file_read', JSON.stringify({ path: p })).then(String), path);
146
147 // ── An empty workspace ──
148 const bare = await chatApp([], [], ['quoted'], BREAK);
149 check('a chat declares a workspace even when the user has marked nothing into it',
150 Array.isArray(bare.write_allow) && bare.write_allow.indexOf('chats/c-scope/work') >= 0
151 && (bare.allow || []).length === 0,
152 JSON.stringify(bare));
153
154 const bareRead = await tool('file_read', { path: 'books/thinking.typ' });
155 check('with an empty workspace the user\'s files are still readable',
156 /the user own chapter/.test(bareRead), bareRead.slice(0, 100));
157
158 const bareWrite = await tool('file_write', { path: 'books/thinking.typ', content: 'rewritten\n' });
159 check('and not writable — this is the file the incident was about',
160 /Refused/.test(bareWrite), bareWrite.slice(0, 100));
161 const bookAfter = await onDisk('books/thinking.typ');
162 check('and the refusal is real: the chapter is untouched on disk',
163 /the user own chapter/.test(bookAfter) && !/rewritten/.test(bookAfter),
164 bookAfter.slice(0, 100));
165
166 const bareScratch = await tool('file_write',
167 { path: 'chats/c-scope/work/draft.md', content: 'thinking out loud\n' });
168 check('a chat always has its own folder to work in, marked or not',
169 !/Refused/.test(bareScratch), bareScratch.slice(0, 100));
170
171 // ── A folder marked into the workspace ──
172 const held = await chatApp(['papers'], ['refs'], ['quoted'], BREAK);
173 check('a marked folder is in the workspace the engine holds',
174 (held.write_allow || []).indexOf('papers') >= 0, JSON.stringify(held.write_allow));
175
176 const readIn = await tool('file_read', { path: 'papers/spec.md' });
177 check('inside the workspace, reading works',
178 /the spec as the user left it/.test(readIn), readIn.slice(0, 100));
179
180 // THE CONTROL. A fence that also interrupts has failed the brief, so the
181 // counters are cleared, a real write is made, and nothing may have asked.
182 await resetAsked();
183 const writeIn = await tool('file_write',
184 { path: 'papers/from-the-chat.md', content: 'written with nobody asked\n' });
185 const quietWrite = await asked();
186 check('inside the workspace, writing works',
187 !/Refused/.test(writeIn), writeIn.slice(0, 100));
188 check('and it lands: the file is there afterwards',
189 /written with nobody asked/.test(await onDisk('papers/from-the-chat.md')));
190 check('and NOTHING ASKED — no dialog, no confirm, no prompt',
191 quietWrite.dialogs === 0 && quietWrite.confirms === 0 && quietWrite.prompts === 0,
192 JSON.stringify(quietWrite));
193
194 // An EDIT of a file that was already there, not merely a new file. Asserted on
195 // what the file says afterwards rather than on the reply text: `!/Refused/` is
196 // true of an exception as well as of a success, which is how a verifier passes
197 // for the wrong reason.
198 await tool('file_edit',
199 { path: 'papers/spec.md', old_string: 'as the user left it', new_string: 'as the chat left it' });
200 const spec = await onDisk('papers/spec.md');
201 check('and editing a file already in the workspace works, with nothing asked',
202 /as the chat left it/.test(spec) && (await asked()).dialogs === 0, spec.slice(0, 100));
203
204 // ── Outside it, one verb ──
205 const outRead = await tool('file_read', { path: 'elsewhere/notes.md' });
206 check('outside the workspace, reading works',
207 /the user own note/.test(outRead), outRead.slice(0, 110));
208
209 const outWrite = await tool('file_write', { path: 'elsewhere/notes.md', content: 'clobbered\n' });
210 check('outside the workspace, writing is refused', /Refused/.test(outWrite), outWrite.slice(0, 110));
211 check('and the refusal names the chat\'s workspace, not a Diamond\'s',
212 /chat/i.test(outWrite) && !/Diamond's workspace/.test(outWrite), outWrite.slice(0, 140));
213 check('and it says the read it was probably about is allowed',
214 /Reading is not fenced/.test(outWrite), outWrite.slice(0, 200));
215 const noteAfter = await onDisk('elsewhere/notes.md');
216 check('and that refusal is real too — the note is untouched',
217 /the user own note/.test(noteAfter) && !/clobbered/.test(noteAfter), noteAfter.slice(0, 100));
218
219 // A walk reaches paths it does not name, so the door is not the whole house:
220 // `file_list` re-asks per entry. It now lists outside the workspace, which is
221 // what "summarise these ten files" needs — and every entry it lists is still
222 // refused to every writing tool.
223 const listOut = await tool('file_list', { path: 'elsewhere' });
224 check('and a directory listing outside the workspace works',
225 /notes\.md/.test(listOut), listOut.slice(0, 110));
226
227 // ── Note and Read are not the mark ──
228 //
229 // The mark grants WRITING. Note and Read decide what is quoted into the prompt
230 // and grant nothing at all — which is now asserted as it always should have
231 // been: the quoted path is no more writable than any other, and no less
232 // readable.
233 const quotedRead = await tool('file_read', { path: 'quoted/passage.md' });
234 check('a path attached to be QUOTED is readable, as everything in the workspace is',
235 /a passage quoted into the chat/.test(quotedRead), quotedRead.slice(0, 110));
236 const quotedWrite = await tool('file_write',
237 { path: 'quoted/passage.md', content: 'rewritten by the chat\n' });
238 check('and attaching it did not make it writable — Note is not a grant',
239 /Refused/.test(quotedWrite), quotedWrite.slice(0, 110));
240 check('and it is untouched on disk',
241 /a passage quoted into the chat/.test(await onDisk('quoted/passage.md')));
242
243 // ── Consult, do not edit ──
244 const roRead = await tool('file_read', { path: 'refs/handbook.md' });
245 check('a read-only workspace path can be consulted',
246 /a reference to consult/.test(roRead), roRead.slice(0, 100));
247 const roWrite = await tool('file_write',
248 { path: 'refs/handbook.md', content: 'edited by the chat\n' });
249 check('and cannot be edited', /Refused/.test(roWrite), roWrite.slice(0, 110));
250 check('and is untouched on disk',
251 /a reference to consult/.test(await onDisk('refs/handbook.md')));
252
253 // Daimond's own directory holds the rules about what agents may do, and is
254 // out of a chat's workspace as it is out of a Diamond's.
255 const ownDir = await tool('file_read', { path: '.daimond/config.json' });
256 check('Daimond\'s own directory is not readable from a chat',
257 /Refused/.test(ownDir), ownDir.slice(0, 100));
258 const ownWrite = await tool('file_write',
259 { path: '.daimond/config.json', content: '{"owned":true}\n' });
260 check('nor writable — it holds the rules about what agents may do',
261 /Refused/.test(ownWrite), ownWrite.slice(0, 100));
262 check('and the control that makes that mean something: the file IS there',
263 /seeded/.test(await onDisk('.daimond/config.json')));
264
265 // ── The instrument, proved ──
266 //
267 // Everything above that says "nothing asked" is worth exactly as much as
268 // this. A dialog is put up deliberately and the counter must see it; a
269 // counter that cannot go up has been reporting silence, not quiet.
270 await resetAsked();
271 await page.evaluate(() => {
272 var back = document.createElement('div');
273 back.className = 'modal dlg';
274 var card = document.createElement('div');
275 card.className = 'modal-card dlg-card';
276 back.appendChild(card);
277 document.body.appendChild(back);
278 window.confirm('proving the counter');
279 });
280 await page.waitForTimeout(50);
281 const proof = await asked();
282 check('the instrument works: a dialog put up on purpose IS counted',
283 proof.dialogs > 0 && proof.confirms > 0, JSON.stringify(proof));
284 await page.evaluate(() => {
285 var n = document.querySelector('.modal.dlg');
286 if (n && n.parentNode) n.parentNode.removeChild(n);
287 });
288} catch (e) {
289 check('no exception during the run', false, String(e && e.message || e));
290} finally {
291 try { await s.browser.close(); } catch (e) { /* ignore */ }
292}
293
294console.log('\n' + ok.length + ' ok, ' + bad.length + ' failed');
295if (BREAK) {
296 console.log(bad.length
297 ? `\nbreak '${BREAK}' produced failures, as it must.`
298 : `\nBREAK '${BREAK}' CHANGED NOTHING — the check it targets is not proving anything.`);
299 process.exit(bad.length ? 0 : 1);
300}
301process.exit(bad.length ? 1 : 0);