Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_prompts.mjs

8.1 KiB, 1 run

created by r2519314175:605, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_prompts.mjs — the prompt each agent runs under is the user's to change.
2//
3// Four roles, four files in the workspace (prompts/<role>.md). What has to be
4// true is not "the file can be written" but that what the MODEL is sent changes
5// with it, so every assertion below reads the wire: the mock provider records
6// each request, and the system message on it is the thing under test.
7//
8// The two properties worth the most:
9//
10// * An absent file means the shipped default, so DELETING one restores the
11// original. A user who breaks a prompt must be able to get back.
12// * A user may write anything at all, and the safety rules still reach the
13// model. Page text is data, not instruction; nothing irreversible happens
14// unasked. Those survive a rewrite, or an editable prompt would be a way to
15// disarm the agent by accident.
16import { open, chat, mockLog, clearMockLog } from './harness.mjs';
17
18const ok = [], bad = [];
19const check = (name, pass, detail) => {
20 (pass ? ok : bad).push(name + (detail ? ' — ' + detail : ''));
21 console.log((pass ? ' ok ' : ' FAIL ') + name + (detail ? ' — ' + detail : ''));
22};
23
24const s = await open({ name: 'prompts' });
25const p = s.page;
26await p.waitForTimeout(1200);
27
28/// The system message of the most recent request the mock actually received.
29const systemSent = () => {
30 const reqs = mockLog();
31 for (let i = reqs.length - 1; i >= 0; i--) {
32 const m = (reqs[i].messages || []).find(x => x.role === 'system');
33 if (m) return m.content || '';
34 }
35 return '';
36};
37
38const write = (path, content) => p.evaluate(async ({ path, content }) => {
39 const mod = await import('../pkg/oxedyne_daimond.js');
40 const app = new mod.DaimondApp('http://127.0.0.1/v1/chat/completions', '', 'none', 256, '', true);
41 await app.run_tool('dir_create', JSON.stringify({ path: 'prompts' }));
42 return await app.run_tool('file_write', JSON.stringify({ path, content }));
43}, { path, content });
44
45const remove = (path) => p.evaluate(async (path) => {
46 const mod = await import('../pkg/oxedyne_daimond.js');
47 const app = new mod.DaimondApp('http://127.0.0.1/v1/chat/completions', '', 'none', 256, '', true);
48 return await app.run_tool('file_delete', JSON.stringify({ path }));
49}, path);
50
51// ── 1. The shipped default, with no file at all ─────────────────────────
52clearMockLog();
53await chat(s, 'hello there');
54const asShipped = systemSent();
55check('a chat with no prompt file runs on the shipped default',
56 /You are Daimond/.test(asShipped), asShipped.slice(0, 48));
57check('...which carries the rules that always apply',
58 /untrusted data/.test(asShipped) && /cannot undo/.test(asShipped));
59
60// ── 2. The user's own words reach the model ─────────────────────────────
61const MINE = 'You are Bartleby. You answer only in the fewest words possible.';
62await write('prompts/chat.md', MINE);
63await p.evaluate(() => window.DaimondPrompts.refresh());
64await p.waitForTimeout(600);
65clearMockLog();
66await chat(s, 'and hello again');
67const mine = systemSent();
68check('an edited prompt is what the model is sent', mine.includes('Bartleby'), mine.slice(0, 60));
69check('...and the shipped wording it replaced is gone',
70 !/helpful coding assistant/.test(mine));
71
72// ── 3. What an edit cannot take away ────────────────────────────────────
73check('the rules survive a prompt that does not mention them',
74 /untrusted data/.test(mine) && /cannot undo/.test(mine),
75 mine.slice(-90));
76check('...and they come AFTER the user’s text, so they are read last',
77 mine.indexOf('Bartleby') < mine.indexOf('untrusted data'));
78
79// ── 4. Deleting the file puts the original back ─────────────────────────
80await remove('prompts/chat.md');
81await p.evaluate(() => window.DaimondPrompts.refresh());
82await p.waitForTimeout(600);
83clearMockLog();
84await chat(s, 'once more');
85const restored = systemSent();
86check('deleting the file restores the shipped prompt',
87 /You are Daimond/.test(restored) && !/Bartleby/.test(restored), restored.slice(0, 48));
88
89// ── 5. The wasm agrees with the files about what a default is ───────────
90const roundTrip = await p.evaluate(async () => {
91 const mod = await import('../pkg/oxedyne_daimond.js');
92 const out = {};
93 for (const role of ['chat', 'conductor', 'worker', 'reducer']) {
94 out[role] = {
95 def: mod.default_prompt(role).slice(0, 40),
96 clause: mod.compose_prompt(role, '').includes('untrusted data'),
97 mineWins: mod.compose_prompt(role, 'ZZZ').includes('ZZZ'),
98 };
99 }
100 out.unknown = mod.default_prompt('wizard');
101 return out;
102});
103check('every role has a default of its own',
104 ['chat', 'conductor', 'worker', 'reducer'].every(r => roundTrip[r].def.length > 20)
105 && new Set(['chat', 'conductor', 'worker', 'reducer'].map(r => roundTrip[r].def)).size === 4);
106check('every role takes the user’s text over its default',
107 ['chat', 'conductor', 'worker', 'reducer'].every(r => roundTrip[r].mineWins));
108check('the tool-holding roles carry the rules, the tool-less reducer does not',
109 roundTrip.chat.clause && roundTrip.conductor.clause && roundTrip.worker.clause
110 && !roundTrip.reducer.clause,
111 JSON.stringify({ chat: roundTrip.chat.clause, conductor: roundTrip.conductor.clause,
112 worker: roundTrip.worker.clause, reducer: roundTrip.reducer.clause }));
113check('an unknown role yields nothing rather than a wrong prompt',
114 roundTrip.unknown === '');
115
116// ── 6. A worker is told the user's worker prompt, not the chat's ────────
117await write('prompts/worker.md', 'You are a WORKERMARK agent.');
118await p.evaluate(() => window.DaimondPrompts.refresh());
119await p.waitForTimeout(600);
120const workerSystem = await p.evaluate(() => window.DaimondPrompts.role('worker'));
121check('a worker runs on the worker file, not the chat one',
122 /WORKERMARK/.test(workerSystem) && !/Bartleby/.test(workerSystem),
123 workerSystem.slice(0, 50));
124check('...with the rules appended to it too', /untrusted data/.test(workerSystem));
125await remove('prompts/worker.md');
126
127// ── 7. The Admin panel offers each one, and opens it in the Doc panel ───
128// Through the control a user actually presses: the cog in the rail's status
129// strip, which is how the Admin panel is reached.
130await p.click('#settings-btn', { force: true });
131await p.waitForTimeout(900);
132const buttons = await p.$$eval('#admin-home .admin-item', els => els.map(e => e.textContent));
133check('the Admin panel offers a button per role',
134 // "daimon", not "conductor": the agent behind a Diamond was renamed and this
135 // list was not, so the check went on looking for a word the app stopped using.
136 ['chat', 'diamond daimon', 'dispatched worker', 'crystal fold']
137 .every(r => buttons.some(b => b.toLowerCase().includes(r))),
138 buttons.filter(b => /prompt/i.test(b)).join(' | '));
139
140await p.evaluate(() => {
141 const b = [...document.querySelectorAll('#admin-home .admin-item')]
142 .find(e => /chat prompt/i.test(e.textContent));
143 if (b) b.click();
144});
145await p.waitForTimeout(2500);
146const doc = await p.evaluate(() => ({
147 shown: !!(document.querySelector('#panel-doc') || {}).offsetParent,
148 name: (document.getElementById('doc-name') || {}).textContent || '',
149 body: (document.querySelector('.files-view-body') || {}).textContent || '',
150}));
151check('the button opens the prompt in the Doc panel', doc.shown && /prompts\/chat\.md/.test(doc.name),
152 doc.name);
153check('...seeded with the real shipped text, so there is something to edit from',
154 /You are Daimond/.test(doc.body), doc.body.slice(0, 48));
155// Seeding writes the file; leave the workspace as it was found.
156await remove('prompts/chat.md');
157
158// This walk needs no gateway, so /api calls fail: a 502 from dev/serve.mjs's
159// proxy, or a 401/402 where one is running without an entitled account. Neither
160// is anything to do with a prompt.
161const errs = s.errs.filter(e => !/favicon|404|401|402|502|net::ERR/.test(e));
162check('nothing throws while all this happens', errs.length === 0, errs.slice(0, 3).join(' | '));
163
164console.log(`\n${ok.length} passed, ${bad.length} failed`);
165if (bad.length) console.log('FAILED:\n ' + bad.join('\n '));
166await s.close();
167process.exit(bad.length ? 1 : 0);