Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/verify_signals.mjs

13.3 KiB, 1 run

created by r2519314175:679, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// verify_signals.mjs -- the local usage index, and the rule it is built around.
2//
3// notes2 #51 asks for the Optimiser to be given a scope over "a lot of things,
4// possibly everything, but as a minimum all LLM sessions". A scope cannot say
5// that: a scope is workspace folders, the ledger is localStorage and chats are
6// in IndexedDB. So what it needs is written into a folder instead, and the
7// folder is what it is granted.
8//
9// The properties, in the order they matter:
10//
11// 1. THE PRESENTATION RULE. The index may read irritation; nothing built on
12// it may ever mention it. A digest that says "you seem frustrated", or
13// prints a mood score, has misused the module -- so the digest is checked
14// for those words against input chosen to provoke them. This is the check
15// that stops the whole idea being obnoxious, so it is first.
16// 2. NO MESSAGE TEXT IS KEPT. The index holds counters and hashes. It cannot
17// leak what it does not hold, and that claim is testable: put a rare
18// string in, and it must not come out anywhere in the stored index.
19// 3. The composite is a composite. One signal is noise -- people swear at the
20// weather and write "again." meaning "once more please" -- so one does not
21// count and two do.
22// 4. Terseness is judged against THIS user's norm, not a fixed number. Six
23// words from someone who always writes six words is not a signal.
24// 5. A repeated ask is counted, and counted by MEANING rather than by exact
25// text, because nobody retypes a request the same way twice.
26// 6. Findings carry a number and something to do. A finding with no action is
27// a complaint; one without evidence is astrology.
28// 7. Silence is a valid answer. An index with nothing to report says so
29// rather than padding.
30//
31// The pure half runs in Node against www/js/signals.js with a localStorage
32// stub, so the scoring is tested without a browser. The browser half proves the
33// digest reaches system/usage/digest.md and that the Optimiser can read it.
34//
35// node dev/verify_signals.mjs
36// node dev/verify_signals.mjs --pure # skip the browser half
37//
38// Needs dev/serve.mjs for the browser half (dev/world.sh N --up).
39import fs from 'node:fs';
40import path from 'node:path';
41import { fileURLToPath } from 'node:url';
42
43const HERE = path.dirname(fileURLToPath(import.meta.url));
44const ROOT = path.join(HERE, '..');
45
46let failures = 0;
47const check = (cond, msg, detail) => {
48 if (!cond) failures++;
49 console.log((cond ? ' ok ' : ' FAIL ') + msg + (detail ? ' -- ' + detail : ''));
50};
51
52// ── A localStorage the module can use ───────────────────────
53const mem = new Map();
54globalThis.localStorage = {
55 getItem: k => (mem.has(k) ? mem.get(k) : null),
56 setItem: (k, v) => mem.set(k, String(v)),
57 removeItem: k => mem.delete(k),
58};
59const src = fs.readFileSync(path.join(ROOT, 'www/js/signals.js'), 'utf8');
60const mod = { exports: {} };
61new Function('module', 'localStorage', src)(mod, globalThis.localStorage);
62const S = mod.exports;
63
64// ══ 3. The composite ═══════════════════════════════════════════════
65// A norm long enough that terseness is available as a signal, so these test
66// what they say they test rather than accidentally scoring on length.
67const NORM = 120;
68// Long enough that terseness does NOT also fire: a four-letter swear is two
69// signals, quite correctly, and would prove nothing about strong language alone.
70const SWORE = 'this is a fucking long message about the ledger and what it should do';
71check(S.score(SWORE, NORM) === 1, 'strong language is a signal, and on its own it is one',
72 String(S.score(SWORE, NORM)));
73check(S.missed(SWORE, NORM) === false,
74 'so it is not a miss by itself -- people swear at the weather',
75 String(S.score(SWORE, NORM)));
76check(S.score('fuck', NORM) === 2,
77 'while a four-letter reply is two signals: strong AND far shorter than this user writes',
78 String(S.score('fuck', NORM)));
79check(S.missed('no, i said use the other one', NORM) === false,
80 'nor is a correction alone', String(S.score('no, i said use the other one', NORM)));
81check(S.missed('wtf?? why would you do that', NORM) === true,
82 'two independent signals together are',
83 String(S.score('wtf?? why would you do that', NORM)));
84check(S.missed('THIS IS COMPLETELY WRONG AGAIN, as i already told you', NORM) === true,
85 'and shouting a correction certainly is',
86 String(S.score('THIS IS COMPLETELY WRONG AGAIN, as i already told you', NORM)));
87check(S.missed('Could you try the other approach please?', NORM) === false,
88 'an ordinary polite request is not a miss',
89 String(S.score('Could you try the other approach please?', NORM)));
90check(S.missed('', NORM) === false, 'and an empty message is nothing at all');
91
92// ══ 4. Terseness is relative ═══════════════════════════════════════
93check(S.score('no', 400) > S.score('no', 0),
94 '"no" from someone who writes long messages scores higher than from someone with no norm yet',
95 `${S.score('no', 400)} vs ${S.score('no', 0)}`);
96check(S.score('no', 30) === S.score('no', 0),
97 'and a user whose own norm is short gets no terseness signal at all',
98 `${S.score('no', 30)} vs ${S.score('no', 0)}`);
99
100// ══ 5. Repeats are counted by meaning ══════════════════════════════
101{
102 const a = S.intentHash('Can you please summarise the meeting notes for me');
103 const b = S.intentHash('summarise the meeting notes');
104 check(!!a && a === b,
105 'the same ask worded two ways lands on one hash', `${a} / ${b}`);
106 const c = S.intentHash('deploy the gateway to production');
107 check(a !== c, 'and a different ask does not', `${a} / ${c}`);
108 check(S.intentHash('ok') === '', 'something too short to mean anything twice is not counted');
109}
110
111// ══ 2. No message text is kept ═════════════════════════════════════
112{
113 S.reset();
114 const RARE = 'zarquon-fnord-quibble';
115 S.noteUserMessage({ diamondId: 'd1', text: `fuck this ${RARE} is wrong again!!`, prevModel: 'm1' });
116 const stored = JSON.stringify(S.snapshot());
117 check(stored.indexOf(RARE) < 0,
118 'the message does not appear in the stored index -- it cannot leak what it does not hold',
119 stored.length + ' bytes stored');
120 check(stored.indexOf('fuck') < 0, 'nor does anything else that was typed');
121 check(S.snapshot().diamonds.d1 && S.snapshot().diamonds.d1.missed === 1,
122 'but the counter moved', JSON.stringify(S.snapshot().diamonds.d1));
123}
124
125// ══ 1. THE PRESENTATION RULE ═══════════════════════════════════════
126// Fed the most provocative input this module can be given, the digest must
127// describe DEFECTS and never the person's mood.
128{
129 S.reset();
130 const diamonds = [{ id: 'd1', name: 'Ledger work' }];
131 for (let i = 0; i < 12; i++) {
132 S.noteTurn({ ts: Date.now(), diamondId: 'd1', model: 'cheap/model', usd: 0.01 });
133 S.noteTool('web_fetch', true);
134 S.noteUserMessage({ diamondId: 'd1', text: 'WHAT?! that is wrong AGAIN, as i already told you',
135 prevModel: 'cheap/model', prevTools: ['web_fetch'] });
136 }
137 const md = S.digest(diamonds);
138 const banned = /frustrat|angry|upset|annoy|mood|swear|swore|profan|temper|emotion|stressed|irritat/i;
139 check(!banned.test(md),
140 'the digest never names the reader’s state of mind, however it was provoked',
141 (md.match(banned) || [''])[0] || 'clean');
142 check(!/\bscore\b/i.test(md), 'and shows no score for it either');
143 // What it SHOULD say: the defect the signal pointed at.
144 check(/web_fetch/.test(md),
145 'it names the tool whose turns kept needing correcting', 'web_fetch present');
146 check(/cheap\/model/.test(md), 'and the model they ran on');
147 check(/correct/i.test(md),
148 'in the language of work that had to be redone, which is the observable fact');
149}
150
151// ══ 6. Findings carry a number and an action ═══════════════════════
152{
153 const f = S.findings([{ id: 'd1', name: 'Ledger work' }]);
154 check(f.length > 0, 'the run above produced findings', String(f.length));
155 check(f.every(x => /\d/.test(x.what)),
156 'each states a number, so it can be checked rather than believed',
157 f.map(x => x.what).join(' | '));
158 check(f.every(x => x.do_ && x.do_.length > 10),
159 'and each says what to do about it -- a finding with no action is a complaint');
160}
161
162// ══ 7. Silence is a valid answer ═══════════════════════════════════
163{
164 S.reset();
165 const md = S.digest([]);
166 check(/Nothing\./.test(md) || /Nothing recorded yet/.test(md),
167 'an index with nothing to report says so');
168 check(!/frustrat/i.test(md), 'and still says nothing about anybody’s mood');
169 check(/does not know/i.test(md),
170 'and states what it cannot see, so a reader does not take it for the whole picture');
171}
172
173// ══ The browser half ═══════════════════════════════════════════════
174if (!process.argv.includes('--pure')) {
175 const { open, connectMock, signInAs, scratch } = await import('./harness.mjs');
176 const s = await open({ name: 'signals', signIn: false, connect: false,
177 profile: scratch('pw', 'signals-' + process.pid) });
178 const { page: p } = s;
179 try {
180 await signInAs(s, 'signals');
181 await connectMock(s);
182 await p.evaluate(() => DaimondDiamond.seedDefaults());
183 await p.waitForFunction(() =>
184 [...document.querySelectorAll('#diamond-list .session-box-name')]
185 .some(n => /Daimond Optimiser/.test(n.textContent)), null, { timeout: 20000 });
186
187 const id = await p.evaluate(() => {
188 const b = [...document.querySelectorAll('#diamond-list .diamond-box')]
189 .find(x => /Daimond Optimiser/.test(x.textContent));
190 return b ? b.dataset.id : '';
191 });
192 check(!!id, 'the Optimiser is there', String(id));
193
194 // The digest is a real file, in the workspace, where a file tool reaches it.
195 const file = await p.evaluate(async () => {
196 const W = await import('/pkg/oxedyne_daimond.js');
197 try { return await W.store_read('system/usage/digest.md'); }
198 catch (e) { return ''; }
199 });
200 check(!!file, 'the digest is written to system/usage/digest.md',
201 file ? file.split('\n')[0] : '(absent)');
202 check(/does not know/i.test(file || ''),
203 'and it is the digest this module writes, not an empty file');
204
205 // And the Optimiser can actually see it. This is the whole point: the
206 // grant is the ordinary read-only attachment, so bounds() reports it.
207 const bounds = await p.evaluate(async (did) => {
208 const W = window;
209 // Files.bounds is the same call scopeAgentTo makes before dispatching
210 // an agent, so this asks exactly what the engine will be told.
211 return await W.DaimondDiamond.bounds(did);
212 }, id).catch(() => null);
213 if (bounds) {
214 check((bounds.read_only || []).indexOf('system/usage') >= 0,
215 'and the Optimiser holds it READ-ONLY -- it reports on the work, it does not rewrite it',
216 JSON.stringify(bounds.read_only));
217 check((bounds.attached || []).indexOf('system/usage') >= 0,
218 'and it is in reach at all', JSON.stringify(bounds.attached));
219 } else {
220 check(false, 'the Optimiser’s bounds could be read', 'DaimondDiamond.bounds not published');
221 }
222 // ── And Help gets the guide, the same way ──────────
223 // The guide ships inside the bundle as HTML, so Help -- the Diamond
224 // whose whole job is explaining the app -- could not read the manual it
225 // exists to explain. Mirrored from the search index that already ships,
226 // one file per page so a daimon reads the page it needs rather than
227 // eighty kilobytes to answer one question.
228 const helpId = await p.evaluate(() => {
229 const b = [...document.querySelectorAll('#diamond-list .diamond-box')]
230 .find(x => /Daimond Help/.test(x.textContent));
231 return b ? b.dataset.id : '';
232 });
233 check(!!helpId, 'Daimond Help is there', String(helpId));
234
235 const mirror = await p.evaluate(async () => {
236 const W = await import('/pkg/oxedyne_daimond.js');
237 const out = {};
238 try { out.readme = await W.store_read('system/guide/README.md'); } catch (e) { out.readme = ''; }
239 try { out.page = await W.store_read('system/guide/accounts.md'); } catch (e) { out.page = ''; }
240 return out;
241 });
242 check(!!mirror.readme, 'the guide is mirrored, with a contents page',
243 (mirror.readme || '').split('\n')[0]);
244 check(/\.md/.test(mirror.readme || ''),
245 'that names the pages, so a daimon knows what to search');
246 check((mirror.page || '').length > 200,
247 'and a page carries the guide’s actual words',
248 String((mirror.page || '').length) + ' chars');
249 check(!/<[a-z]+[ >]/i.test(mirror.page || ''),
250 'as text, not as the HTML it ships in',
251 ((mirror.page || '').match(/<[a-z]+[ >]/i) || [''])[0] || 'clean');
252
253 const hb = await p.evaluate((did) => window.DaimondDiamond.bounds(did), helpId).catch(() => null);
254 check(hb && (hb.read_only || []).indexOf('system/guide') >= 0,
255 'and Help holds it READ-ONLY -- it explains the guide, it does not edit it',
256 hb && JSON.stringify(hb.read_only));
257 } catch (e) {
258 failures++;
259 console.log(' FAIL browser half threw -- ' + (e && e.message ? e.message.split('\n')[0] : e));
260 } finally {
261 await s.close();
262 }
263}
264
265console.log('');
266console.log(failures ? `verify_signals: ${failures} FAILED` : 'verify_signals: all checks pass.');
267process.exit(failures ? 1 : 0);