Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/probe_register.mjs

32.4 KiB, 1 run

created by r2519314175:71, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// probe_register.mjs — is a standing prompt's REGISTER worth anything, once its length is held?
2//
3// The owner's hypothesis of 2026-08-25 is that the system prompting is not calibrated for
4// the more powerful models. `dev/PROMPT_NOTES.md` §6 already showed one half of that: two
5// notes turned out to bite on the weaker half of the panel and on nothing else. The sharper
6// form, which is what this file measures, is that the split runs along KIND rather than
7// along the note list -- tell a strong model what must be true and what would surprise it,
8// tell a weak model how to proceed.
9//
10// ── The confound this exists to control ─────────────────────────────
11//
12// A constraints register is naturally shorter than the procedure that implements it, so the
13// pair already on the shelf -- `dev/house/prose.md` at 884 tokens against
14// `dev/house/prose_short.md` at 511 -- varies KIND and LENGTH at once, and the flip measured
15// on it (sonnet-4.5 better on the short one, haiku-4.5 better on the long one) is explained
16// equally well by either. `dev/house/reg/` holds a 2x2 instead: constraints and procedure,
17// long and short, each pair matched to within 2% of the other's measured tokens and each
18// carrying the same fifteen pieces of information. **A run that cannot separate kind from
19// length has not tested the hypothesis, whichever way its numbers fall.**
20//
21// ── Why this is not `dev/reflux.mjs` ────────────────────────────────
22//
23// Reflux runs the same task through the real browser, the real extension and the real fence,
24// and `--house` was added to it for exactly this. It costs a browser and up to ten minutes a
25// turn, so a 2x2 across five models at n=3 is a day of wall clock and cannot be read as a
26// rate. This runs the same fixture, the same brief and the same check against the provider
27// directly, so an arm is a rate rather than an anecdote.
28//
29// **What it therefore cannot see**, and reflux is the instrument for each: the extension's
30// own refusals, the fence, the panel, and anything that depends on the app's streaming. The
31// seven tools implemented here are the seven the prose task can reach; the other twenty-two
32// schemas of the real request are not offered, so a model cannot spend a round on `web_search`
33// the way it could in the product.
34//
35// ── What makes it real ──────────────────────────────────────────────
36//
37// **Nothing here is transcribed.** The system prompt and every tool schema come out of
38// `<log>.request.json`, which reflux's relay writes from the body it is about to forward.
39// The fixture, the brief, the owner's draft and the rewrite it is scored against come out of
40// `dev/reflux.mjs` by name, so a reworded task is a hard stop rather than a silent drift, and
41// the three checkers are his own files copied read-only into the scratch tree.
42//
43// **The register is counted into the request.** It goes where `Instructions::compose` puts
44// `DAIMOND.md`, under the same heading `dev/reflux.mjs` uses, and a run whose register never
45// reached a request aborts rather than reporting a baseline as an arm.
46//
47// node dev/probe_register.mjs --selftest free; proves every check
48// node dev/probe_register.mjs --model panel --n 3
49// --reg none,c_short,c_long,p_short,p_long which registers (default: all five)
50// --task prose default and, today, the only one
51// --n 3 runs per cell
52// --model a/b,c/d | panel | baseline
53// --max-rounds 20 --max-secs 900 --max-bytes 70000
54// --keep <dir> every transcript written there
55// --dry build every request, send none
56//
57// The key is read as `dev/probe_notes.mjs` reads it and there is no default:
58// ~/.config/oxedyne/daimond/openrouter.key (0600), or DAIMOND_PROBE_KEY.
59import fs from 'node:fs';
60import os from 'node:os';
61import path from 'node:path';
62import { spawnSync } from 'node:child_process';
63import { fileURLToPath } from 'node:url';
64import { PANEL, BASELINE, OPEN, priceOf } from './models.mjs';
65
66const HERE = path.dirname(fileURLToPath(import.meta.url));
67const REFLUX = path.join(HERE, 'reflux.mjs');
68const argv = process.argv.slice(2);
69const flag = (n, d) => {
70 const i = argv.indexOf('--' + n);
71 return i >= 0 && argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[i + 1] : d;
72};
73const has = (n) => argv.includes('--' + n);
74
75const SELFTEST = has('selftest');
76const DRY = has('dry');
77const REQUEST = flag('request', path.join(os.homedir(), '.cache/daimond/lane-n/reflux/reflux.request.json'));
78const N = Number(flag('n', '3'));
79const MAXROUND = Number(flag('max-rounds', '12'));
80const MAXSECS = Number(flag('max-secs', '900'));
81const MAXBYTES = Number(flag('max-bytes', '70000'));
82const MAXTOK = Number(flag('max-tokens', '4000'));
83const KEEP = flag('keep', '');
84const SCRATCH = flag('scratch', path.join(os.homedir(), '.cache/daimond/lane-ab'));
85// ── The baseline gate ───────────────────────────────────────────────
86//
87// **A model that cannot do the task with NO standing prompt at all is unmeasurable here, and
88// its register arms are not evidence about a register.** Nothing separates "this prompt hurt
89// the model" from "this model cannot drive these tools" once both arms are floored, and the
90// difference is the whole finding. So `--gate` runs the bare arm first, and where the model
91// never writes the file in any baseline sample it is recorded UNMEASURABLE and the register
92// arms are not run -- which also stops the money going to a column nobody could read.
93//
94// It is not on by default, because a run that already holds its model's baseline should not
95// pay for another. Measured 2026-08-25, it would have caught two of the five models then on
96// the panel: qwen3.8-max never wrote the paragraph in three bare samples out of three, and
97// glm-5.2 wrote it in two of three and then in none of twelve with any register at all.
98const GATE = has('gate');
99const MODELS = (() => {
100 const raw = String(flag('model', 'baseline'));
101 if (raw === 'panel') return PANEL.filter((s) => !s.endsWith(':free'));
102 if (raw === 'baseline') return BASELINE;
103 if (raw === 'open') return OPEN;
104 return raw.split(',').map((s) => s.trim()).filter(Boolean);
105})();
106
107// ── The registers under test ────────────────────────────────────────
108//
109// `none` is the control that says how much of any arm's score is the register at all. The two
110// shipped files are carried as well, unmeasured by the 2x2, because the flip that started this
111// was measured on them and a run that cannot be compared with it has lost the thread.
112const REGISTERS = {
113 none: null,
114 c_short: 'reg/prose_constraints_short.md',
115 c_long: 'reg/prose_constraints_long.md',
116 p_short: 'reg/prose_procedure_short.md',
117 p_long: 'reg/prose_procedure_long.md',
118 shipped_short: 'prose_short.md',
119 shipped_long: 'prose.md',
120};
121const REGS = String(flag('reg', 'none,c_short,c_long,p_short,p_long'))
122 .split(',').map((s) => s.trim()).filter(Boolean);
123for (const r of REGS) {
124 if (!(r in REGISTERS)) {
125 throw new Error(`--reg ${r}: no such register. One of ${Object.keys(REGISTERS).join(', ')}.`);
126 }
127}
128const HOUSE_HEAD = '## Standing instructions from the user';
129const registerText = (name) => (REGISTERS[name]
130 ? fs.readFileSync(path.join(HERE, 'house', REGISTERS[name]), 'utf8').trim() : '');
131
132// ── The task, read out of `dev/reflux.mjs` rather than copied ───────
133//
134// A JS template literal, by name. The parser understands the one shape these four constants
135// use and refuses anything else by name, for `dev/promptparts.mjs`'s reason: an instrument
136// holding its own copy of a fixture reports on a task nothing runs.
137function templateConst(src, name) {
138 const decl = new RegExp(`\\bconst\\s+${name}\\s*=\\s*\``);
139 const m = decl.exec(src);
140 if (!m) throw new Error(`probe_register: no const ${name} = \`…\` in dev/reflux.mjs`);
141 let out = '';
142 for (let i = m.index + m[0].length; i < src.length; i++) {
143 const ch = src[i];
144 if (ch === '`') return out;
145 if (ch === '$' && src[i + 1] === '{') {
146 throw new Error(`probe_register: ${name} interpolates, which this parser will not `
147 + 'guess at. Teach it that shape rather than letting it drift.');
148 }
149 if (ch === '\\') { out += { n: '\n', t: '\t', '\\': '\\', '`': '`', '$': '$' }[src[i + 1]] ?? src[i + 1]; i++; continue; }
150 out += ch;
151 }
152 throw new Error(`probe_register: ${name} is unterminated`);
153}
154const RSRC = fs.readFileSync(REFLUX, 'utf8');
155const PROSE_DRAFT = templateConst(RSRC, 'PROSE_DRAFT');
156const PROSE_KEPT = templateConst(RSRC, 'PROSE_KEPT');
157
158// The brief, held here because reflux's is a function of its own context object. It is the
159// same sentence, and `--selftest` asserts that every clause of it is still in reflux's source:
160// a brief that has drifted from the one the browser harness sends is two experiments.
161const BRIEF = (mark) => `The "${mark}" folder of this workspace is on this computer, and `
162 + 'draft.typ in it holds one paragraph of a book the user is writing. It reads as an '
163 + 'AI draft rather than as his own prose. Rewrite it so that it reads as his, and '
164 + `write the result back to draft.typ. Use commands with the working directory set to "${mark}". `
165 + 'Keep every claim, every name and every date. Change no other file.';
166
167// ── The request the app really sends ────────────────────────────────
168
169const req = JSON.parse(fs.readFileSync(REQUEST, 'utf8'));
170const sysMsg = (req.messages || []).find((m) => m.role === 'system');
171if (!sysMsg) throw new Error(`${REQUEST} carries no system message.`);
172const SYSTEM = typeof sysMsg.content === 'string' ? sysMsg.content
173 : (sysMsg.content || []).map((p) => (p && p.text) || '').join('');
174// The seven the prose task can reach, and no others: a schema offered but not implemented is
175// a round the model spends on a tool that answers nothing.
176const OFFERED = ['file_read', 'file_write', 'file_edit', 'file_list', 'file_search', 'file_glob', 'run'];
177const TOOLS = (req.tools || []).filter((t) => OFFERED.includes(t.function && t.function.name));
178if (TOOLS.length !== OFFERED.length) {
179 throw new Error(`${REQUEST} carries ${TOOLS.length} of the ${OFFERED.length} tool schemas `
180 + 'this probe implements. Capture a request from a role that holds the file tools.');
181}
182
183function readKey() {
184 const env = (process.env.DAIMOND_PROBE_KEY || '').trim();
185 if (env) return env;
186 const file = process.env.DAIMOND_PROBE_KEY_FILE
187 || path.join(os.homedir(), '.config/oxedyne/daimond/openrouter.key');
188 let text;
189 try { text = fs.readFileSync(file, 'utf8'); }
190 catch (e) { throw new Error(`No provider key. Put one in ${file} (0600), or set DAIMOND_PROBE_KEY.`); }
191 const key = text.split('\n').map((l) => l.trim()).find((l) => l && !l.startsWith('#'));
192 if (!key) throw new Error(`${file} holds no key.`);
193 return key;
194}
195
196// ── The fixture ─────────────────────────────────────────────────────
197//
198// Built exactly as `dev/reflux.mjs`'s prose task builds it, and for the same reason: `~/usr` is
199// a Syncthing folder and a command could write in it, so the run is handed READ-ONLY COPIES and
200// the worst it can do is spoil its own scratch tree.
201const BOOKS = path.join(os.homedir(), 'usr/books');
202function buildFixture(dir) {
203 fs.rmSync(dir, { recursive: true, force: true });
204 fs.mkdirSync(path.join(dir, 'tools'), { recursive: true });
205 fs.mkdirSync(path.join(dir, 'style'), { recursive: true });
206 fs.mkdirSync(path.join(dir, 'elearnity'), { recursive: true });
207 // AT THE PATH EVERY REGISTER NAMES, which is where reflux's own fixture differs: it puts the
208 // spec at the root, so every register arm pays a wasted round on ENOENT that the `none`
209 // control does not. Measured on haiku-4.5, 2026-08-25, and it was the second call of the run.
210 fs.copyFileSync(path.join(BOOKS, 'elearnity/writing_spec.md'),
211 path.join(dir, 'elearnity/writing_spec.md'));
212 fs.copyFileSync(path.join(BOOKS, 'style/exemplars.md'), path.join(dir, 'style/exemplars.md'));
213 for (const t of ['cadence', 'spec_check', 'decode_load']) {
214 fs.copyFileSync(path.join(BOOKS, 'tools', t), path.join(dir, 'tools', t));
215 }
216 fs.writeFileSync(path.join(dir, 'draft.typ'), PROSE_DRAFT);
217 return path.join(dir, 'draft.typ');
218}
219
220/// The draft's own band distance, which is the bar the gate holds a bare arm to.
221///
222/// Floored the way every other distance is, so a model that hands the paragraph back scores
223/// exactly this and a model that moved it at all scores less.
224const DRAFT_DIST = 6;
225
226/// How far outside his band a scored run landed, in half-widths of the band itself.
227///
228/// The same arithmetic `dev/register_verdict.mjs` reports with, kept here because the gate has
229/// to make the same judgement before it will spend anything on a register.
230function bandDistance(r) {
231 if (typeof r.dl !== 'number' || typeof r.mean !== 'number') return DRAFT_DIST;
232 let d = r.dl;
233 if (r.mean < 19.5) d += (19.5 - r.mean) / 2.5;
234 if (r.mean > 24.5) d += (r.mean - 24.5) / 2.5;
235 if (typeof r.long === 'number' && r.long > 20) d += (r.long - 20) / 20;
236 if (r.spec === false) d += 1;
237 return Math.min(d, DRAFT_DIST);
238}
239
240// ── The check, which is reflux's, number for number ─────────────────
241//
242// It returns the SCORE as well as the verdict, so a failing arm still says how far out of his
243// band it landed. An arm scored only pass/fail prices nothing.
244function scoreProse(dir, file) {
245 const s = { ok: null, dl: null, mean: null, long: null, spec: null, changed: false };
246 let now;
247 try { now = fs.readFileSync(file, 'utf8'); }
248 catch (e) { s.ok = 'draft.typ is gone'; return s; }
249 s.changed = now.trim() !== PROSE_DRAFT.trim();
250 if (!s.changed) { s.ok = 'draft.typ is unchanged'; return s; }
251 for (const who of ['Clark', 'George', 'Cobb', 'Douglas', 'Solow', '1899', '1928', '1956']) {
252 if (!now.includes(who)) { s.ok = `the rewrite has lost ${who}`; return s; }
253 }
254 const tool = (name) => spawnSync('python3', [path.join(dir, 'tools', name), file],
255 { encoding: 'utf8', timeout: 60_000 });
256 const dl = tool('decode_load').stdout || '';
257 const m = dl.match(/(\d+) sentences? at or above decode-load 4/);
258 if (!m) { s.ok = 'decode_load said nothing this check can read: ' + dl.slice(0, 120); return s; }
259 s.dl = Number(m[1]);
260 const cad = tool('cadence').stdout || '';
261 const num = (k) => {
262 const g = cad.match(new RegExp(`${k}\\s+([0-9.]+)`));
263 return g ? Number(g[1]) : null;
264 };
265 s.mean = num('mean_sentence_words');
266 s.long = num('pct_long_gt35');
267 s.spec = /Clean|All clean/.test(tool('spec_check').stdout || '');
268 if (s.dl > 0) {
269 s.ok = `decode_load still ranks ${s.dl} sentence(s) at or above 4`;
270 return s;
271 }
272 if (s.mean === null || s.long === null) { s.ok = 'cadence said nothing this check can read'; return s; }
273 if (s.mean < 19.5 || s.mean > 24.5) {
274 s.ok = `cadence puts the mean sentence at ${s.mean} words, outside his band of 19.5-24.5`;
275 return s;
276 }
277 if (s.long > 20) {
278 s.ok = `cadence puts ${s.long}% of sentences over 35 words, above his band of 10-20`;
279 return s;
280 }
281 if (!s.spec) { s.ok = 'spec_check is not clean'; return s; }
282 return s;
283}
284
285// ── The seven tools, executed for real inside the scratch tree ──────
286//
287// `run` takes a WHITELIST and refuses anything else in words. A refusal is a measurement here
288// and not an accident: half of what a register is being asked to buy is a turn that meets a
289// refusal and gets past it, so the count of refusals and the count of rounds after the first
290// one are both reported.
291const ALLOWED = new Set(['python3', 'cat', 'ls', 'head', 'tail', 'wc', 'grep', 'sed', 'sort', 'uniq', 'diff']);
292const LINE = (t) => t.split('\n').map((l, i) => `${i + 1}\t${l}`).join('\n');
293
294function inside(root, rel) {
295 const p = path.resolve(root, rel || '.');
296 if (p !== root && !p.startsWith(root + path.sep)) return null;
297 return p;
298}
299
300function runTool(state, name, args) {
301 const root = state.root;
302 const a = (() => { try { return JSON.parse(args || '{}'); } catch (e) { return null; } })();
303 if (!a) return { text: `Error: ${name}: the arguments were not valid JSON.`, bad: true };
304 const rel = (v) => {
305 if (typeof v !== 'string' || !v) return null;
306 if (path.isAbsolute(v)) return null;
307 return inside(root, v);
308 };
309 switch (name) {
310 case 'file_read': {
311 const p = rel(a.path);
312 if (!p) return { text: `Error: file_read: '${a.path}' is not a workspace-relative path.`, bad: true };
313 let text;
314 try { text = fs.readFileSync(p, 'utf8'); }
315 catch (e) { return { text: `Error: file_read: ${a.path}: ${e.code || e.message}`, bad: true }; }
316 const lines = text.split('\n');
317 const off = Math.max(1, Number(a.offset || 1));
318 const lim = Math.min(Number(a.limit || 2000), 10000);
319 let out = lines.slice(off - 1, off - 1 + lim);
320 let note = '';
321 // The output budget the real tool has, so a model that asks for a 120 KB spec is told
322 // what it is told in the product rather than being handed the file.
323 let joined = out.map((l, i) => `${off + i}\t${l}`).join('\n');
324 if (joined.length > 16_000) {
325 let keep = 0, n = 0;
326 for (const l of out) { if (n + l.length > 16_000) break; n += l.length + 1; keep++; }
327 out = out.slice(0, keep);
328 joined = out.map((l, i) => `${off + i}\t${l}`).join('\n');
329 note = `\n\n[the output budget ran out after line ${off + keep - 1} of ${lines.length}; `
330 + `read on with offset ${off + keep}]`;
331 }
332 return { text: joined + note };
333 }
334 case 'file_write': {
335 const p = rel(a.path);
336 if (!p) return { text: `Error: file_write: '${a.path}' is not a workspace-relative path.`, bad: true };
337 if (typeof a.content !== 'string') return { text: 'Error: file_write: no content.', bad: true };
338 fs.mkdirSync(path.dirname(p), { recursive: true });
339 fs.writeFileSync(p, a.content);
340 return { text: `Wrote ${a.content.length} bytes to ${a.path}.` };
341 }
342 case 'file_edit': {
343 const p = rel(a.path);
344 if (!p) return { text: `Error: file_edit: '${a.path}' is not a workspace-relative path.`, bad: true };
345 let text;
346 try { text = fs.readFileSync(p, 'utf8'); }
347 catch (e) { return { text: `Error: file_edit: ${a.path}: ${e.code || e.message}`, bad: true }; }
348 const parts = text.split(a.old_string || '\u0000');
349 if (parts.length !== 2) {
350 return { text: `Error: file_edit: 'old_string' appears ${parts.length - 1} times in `
351 + `${a.path}; it must appear exactly once.`, bad: true };
352 }
353 fs.writeFileSync(p, parts.join(a.new_string || ''));
354 return { text: `Edited ${a.path}.` };
355 }
356 case 'file_list': {
357 const p = rel(a.path || '.');
358 if (!p) return { text: `Error: file_list: '${a.path}' is not a workspace-relative path.`, bad: true };
359 let ent;
360 try { ent = fs.readdirSync(p, { withFileTypes: true }); }
361 catch (e) { return { text: `Error: file_list: ${a.path}: ${e.code || e.message}`, bad: true }; }
362 return { text: ent.map((d) => (d.isDirectory() ? d.name + '/' : d.name)).join('\n') || '(empty)' };
363 }
364 case 'file_glob': {
365 const walk = (d, acc) => {
366 for (const e of fs.readdirSync(d, { withFileTypes: true })) {
367 const q = path.join(d, e.name);
368 if (e.isDirectory()) walk(q, acc); else acc.push(path.relative(root, q));
369 }
370 return acc;
371 };
372 const pat = String(a.pattern || '*');
373 const rx = new RegExp('^' + pat.replace(/[.+^${}()|[\]\\]/g, '\\$&')
374 .replace(/\*\*\//g, '\u0001').replace(/\*/g, '[^/]*').replace(/\u0001/g, '(?:.*/)?')
375 .replace(/\?/g, '.') + '$');
376 return { text: walk(root, []).filter((f) => rx.test(f)).slice(0, 500).join('\n') || '(no match)' };
377 }
378 case 'file_search': {
379 const flags = ['-rn'];
380 if (a.fixed) flags.push('-F');
381 else flags.push('-E');
382 if (a.ignore_case) flags.push('-i');
383 const where = rel(a.path || '.') || root;
384 const r = spawnSync('grep', [...flags, '-a', String(a.query || ''), '-r', where],
385 { encoding: 'utf8', timeout: 30_000, maxBuffer: 4 << 20 });
386 const out = (r.stdout || '').split('\n').slice(0, Number(a.limit || 200))
387 .map((l) => l.replace(root + '/', '')).join('\n');
388 return { text: out || '(no match)' };
389 }
390 case 'run': {
391 if (!Array.isArray(a.argv) || !a.argv.length) {
392 return { text: "Error: run: 'argv' must be an array, the program then each argument.", bad: true };
393 }
394 const prog = path.basename(String(a.argv[0]));
395 if (!ALLOWED.has(prog)) {
396 return { text: `Refused: this probe's fence runs only ${[...ALLOWED].join(', ')}, and `
397 + `'${prog}' is not among them. Nothing was run.`, refused: true };
398 }
399 const cwd = a.cwd ? rel(a.cwd) : root;
400 if (!cwd) return { text: `Error: run: cwd '${a.cwd}' is outside the workspace.`, bad: true };
401 const r = spawnSync(String(a.argv[0]), a.argv.slice(1).map(String),
402 { cwd, encoding: 'utf8', timeout: Math.min(Number(a.timeout_ms || 120_000), 300_000),
403 maxBuffer: 8 << 20, input: a.stdin ? String(a.stdin) : undefined });
404 const body = ((r.stdout || '') + (r.stderr || '')).slice(0, 24_000);
405 const code = r.status === null ? 'killed' : r.status;
406 return { text: `exit ${code}\n${body}`, bad: r.status !== 0 };
407 }
408 default:
409 return { text: `Error: ${name} is not a tool this probe implements.`, bad: true };
410 }
411}
412
413// ── One run ─────────────────────────────────────────────────────────
414
415async function post(key, body, secs) {
416 const ctl = new AbortController();
417 const t = setTimeout(() => ctl.abort(), secs * 1000);
418 try {
419 const r = await fetch('https://openrouter.ai/api/v1/chat/completions', {
420 method: 'POST', signal: ctl.signal,
421 headers: { 'Authorization': `Bearer ${key}`, 'Content-Type': 'application/json' },
422 body: JSON.stringify(body),
423 });
424 const text = await r.text();
425 let json;
426 try { json = JSON.parse(text); } catch (e) { return { err: `unparseable: ${text.slice(0, 200)}` }; }
427 if (json.error) return { err: `${json.error.code || ''} ${json.error.message || ''}`.trim() };
428 return { json };
429 } catch (e) {
430 return { err: e.name === 'AbortError' ? `no reply in ${secs}s` : String(e.message || e) };
431 } finally { clearTimeout(t); }
432}
433
434async function oneRun(key, model, reg, tag) {
435 const root = path.join(SCRATCH, 'reg', tag);
436 const file = buildFixture(root);
437 const house = registerText(reg);
438 let system = SYSTEM;
439 let placed = 0;
440 if (house) { system = `${SYSTEM}\n\n${HOUSE_HEAD}\n\n${house}`; placed = 1; }
441 if (house && !system.includes(house)) throw new Error(`${reg} never reached the request.`);
442 const state = { root };
443 const messages = [{ role: 'system', content: system },
444 { role: 'user', content: BRIEF('.') }];
445 const rec = { model, reg, rounds: 0, calls: 0, bytes: 0, bad: 0, refused: 0,
446 in_tok: 0, out_tok: 0, secs: 0, err: null, placed };
447 const t0 = Date.now();
448 for (rec.rounds = 1; rec.rounds <= MAXROUND; rec.rounds++) {
449 if ((Date.now() - t0) / 1000 > MAXSECS) { rec.err = 'wall clock'; break; }
450 const body = { model, messages, tools: TOOLS, max_tokens: MAXTOK, usage: { include: true } };
451 if (DRY) { rec.err = 'dry'; break; }
452 const left = Math.max(30, MAXSECS - (Date.now() - t0) / 1000);
453 const { json, err } = await post(key, body, Math.min(600, left));
454 if (err) { rec.err = err; break; }
455 const ch = ((json.choices || [])[0] || {}).message || {};
456 const u = json.usage || {};
457 rec.in_tok += Number(u.prompt_tokens || 0);
458 rec.out_tok += Number(u.completion_tokens || 0);
459 messages.push({ role: 'assistant', content: ch.content || '', tool_calls: ch.tool_calls || undefined });
460 const tc = ch.tool_calls || [];
461 if (!tc.length) break;
462 for (const c of tc) {
463 rec.calls++;
464 const name = (c.function || {}).name || '?';
465 const r = runTool(state, name, (c.function || {}).arguments);
466 rec.bytes += r.text.length;
467 if (r.bad) rec.bad++;
468 if (r.refused) rec.refused++;
469 messages.push({ role: 'tool', tool_call_id: c.id, name, content: r.text });
470 }
471 if (rec.bytes > MAXBYTES) { rec.err = `read ${rec.bytes} bytes, past the ${MAXBYTES} budget`; break; }
472 // A run that used every round is not the same outcome as one that stopped, and reading a
473 // truncated arm as a verdict on the register is how a cap becomes a finding.
474 if (rec.rounds === MAXROUND) rec.err = `used all ${MAXROUND} rounds`;
475 }
476 rec.secs = Math.round((Date.now() - t0) / 10) / 100;
477 rec.usd = priceOf(model, rec.in_tok, rec.out_tok);
478 Object.assign(rec, scoreProse(root, file));
479 if (KEEP) {
480 fs.mkdirSync(KEEP, { recursive: true });
481 fs.writeFileSync(path.join(KEEP, `${tag}.json`),
482 JSON.stringify({ rec, messages, system_chars: system.length }, null, '\t'));
483 }
484 fs.rmSync(root, { recursive: true, force: true });
485 return rec;
486}
487
488// ── The self-test: every check proved red before it is trusted ──────
489
490function selftest() {
491 let bad = 0;
492 const say = (ok, what) => { if (!ok) bad++; console.log(` ${ok ? 'ok ' : 'FAIL'} ${what}`); };
493 const dir = path.join(SCRATCH, 'reg', 'selftest');
494 const file = buildFixture(dir);
495
496 // RED: the owner's own draft, which is the text he threw away.
497 const red = scoreProse(dir, file);
498 say(red.ok === 'draft.typ is unchanged', `an untouched draft is red: ${red.ok}`);
499 fs.writeFileSync(file, PROSE_DRAFT + '\n% a comment, so the file differs\n');
500 const red2 = scoreProse(dir, file);
501 say(red2.ok !== null, `his 2026-08-06 draft is red: ${red2.ok}`);
502 say(red2.mean !== null && red2.mean > 24.5, `and red on cadence: mean ${red2.mean} words`);
503
504 // RED: a rewrite that scores well and drops the argument.
505 fs.writeFileSync(file, 'Land is not capital. The return on capital falls back when investors '
506 + 'build more of it, and the supply of land is fixed, so a high return to it persists in a '
507 + 'way that no amount of building can compete away from the owners who hold it.\n');
508 const red3 = scoreProse(dir, file);
509 say(/has lost Clark/.test(red3.ok || ''), `a rewrite that drops the names is red: ${red3.ok}`);
510
511 // GREEN: the paragraph he kept.
512 fs.writeFileSync(file, PROSE_KEPT);
513 const green = scoreProse(dir, file);
514 say(green.ok === null, `his own rewrite is green: ${green.ok || 'clean'}`);
515 say(green.dl === 0, `decode_load 0 on the kept paragraph, got ${green.dl}`);
516
517 // The fixture and the brief are reflux's, and a drift is a hard stop.
518 say(RSRC.includes('reads as an '), 'the brief is still reflux\'s wording');
519 for (const clause of ['It reads as an ', 'Keep every claim, every name and every date.',
520 'write the result back to draft.typ']) {
521 say(RSRC.includes(clause), `reflux still sends: ${JSON.stringify(clause.slice(0, 40))}`);
522 }
523
524 // Every register is matched to its opposite number within 2%, which is the whole control.
525 const chars = {};
526 for (const [k, v] of Object.entries(REGISTERS)) if (v) chars[k] = registerText(k).length;
527 const near = (a, b) => Math.abs(chars[a] - chars[b]) / Math.max(chars[a], chars[b]) < 0.02;
528 say(near('c_long', 'p_long'), `long pair matched: ${chars.c_long} vs ${chars.p_long} chars`);
529 say(near('c_short', 'p_short'), `short pair matched: ${chars.c_short} vs ${chars.p_short} chars`);
530 say(chars.c_long > chars.c_short * 1.4, 'long really is longer than short');
531
532 // Every tool answers, and `run` refuses what it must.
533 const st = { root: dir };
534 say(!runTool(st, 'file_read', '{"path":"draft.typ"}').bad, 'file_read reads the draft');
535 say(runTool(st, 'file_read', '{"path":"/etc/passwd"}').bad, 'file_read refuses an absolute path');
536 say(runTool(st, 'file_read', '{"path":"../../../etc/passwd"}').bad, 'file_read refuses an escape');
537 say(runTool(st, 'run', '{"argv":["curl","http://example.com"]}').refused, 'run refuses curl');
538 say(runTool(st, 'run', '{"argv":["python3","tools/cadence","draft.typ"]}').text.startsWith('exit 0'),
539 'run runs the owner\'s own cadence');
540 const bigRead = runTool(st, 'file_read', '{"path":"elearnity/writing_spec.md"}');
541 say(/output budget ran out/.test(bigRead.text), 'a 120 KB spec comes back paged, not whole');
542 // THE GATE'S OWN BAR, put to the run that made it necessary: gpt-oss-120b's three bare
543 // samples of 2026-08-25, which wrote a file and handed the draft back inside it.
544 say(bandDistance({ dl: 2, mean: 35.2, long: 67 }) >= DRAFT_DIST,
545 'a near-copy of the draft does not pass the gate');
546 say(bandDistance({ dl: 2, mean: 20.3, long: 10 }) < DRAFT_DIST,
547 'a paragraph that really moved does pass it');
548 say(bandDistance({ dl: 0, mean: 22.3, long: 11 }) < 0.5, 'his own register scores near zero');
549 say(TOOLS.length === OFFERED.length, `${TOOLS.length} schemas taken from the captured request`);
550 say(PROSE_DRAFT.includes('John Bates Clark') && PROSE_KEPT.includes('Progress and Poverty'),
551 'both paragraphs came out of dev/reflux.mjs by name');
552
553 fs.rmSync(dir, { recursive: true, force: true });
554 console.log(bad ? `\n${bad} FAILED` : '\nall green');
555 return bad;
556}
557
558// ── Report ──────────────────────────────────────────────────────────
559
560function report(rows) {
561 const models = [...new Set(rows.map((r) => r.model))];
562 console.log('\nEach cell: passes/n, mean calls, mean bytes read, mean cadence mean-sentence.\n');
563 const w = 22;
564 console.log('model'.padEnd(30) + REGS.map((r) => r.padEnd(w)).join(''));
565 for (const m of models) {
566 let line = m.padEnd(30);
567 for (const g of REGS) {
568 const cell = rows.filter((r) => r.model === m && r.reg === g);
569 if (!cell.length) { line += '-'.padEnd(w); continue; }
570 const pass = cell.filter((r) => r.ok === null).length;
571 const mean = (f) => {
572 const v = cell.map(f).filter((x) => typeof x === 'number' && isFinite(x));
573 return v.length ? Math.round(v.reduce((a, b) => a + b, 0) / v.length * 10) / 10 : null;
574 };
575 line += `${pass}/${cell.length} ${mean((r) => r.calls)}c ${Math.round((mean((r) => r.bytes) || 0) / 1000)}k ${mean((r) => r.mean) ?? '-'}`.padEnd(w);
576 }
577 console.log(line);
578 }
579 const usd = rows.reduce((a, r) => a + (r.usd || 0), 0);
580 console.log(`\n${rows.length} runs, $${usd.toFixed(4)} by the table in dev/models.mjs.`);
581 for (const r of rows.filter((x) => x.err)) {
582 console.log(` ${r.model} ${r.reg}: ${r.err}`);
583 }
584}
585
586// ── Main ────────────────────────────────────────────────────────────
587
588if (SELFTEST) {
589 process.exit(selftest() ? 1 : 0);
590}
591const key = DRY ? 'dry' : readKey();
592const rows = [];
593const out = KEEP ? path.join(KEEP, 'rows.json') : '';
594const say = (rec, i) => console.log(`${rec.model.padEnd(28)} ${rec.reg.padEnd(14)} #${i} `
595 + `${rec.ok === null ? 'PASS' : 'fail'} ${String(rec.calls).padStart(2)}c `
596 + `${String(Math.round((rec.bytes || 0) / 1000)).padStart(3)}k `
597 + `dl=${rec.dl ?? '-'} mean=${rec.mean ?? '-'} long=${rec.long ?? '-'} `
598 + `${rec.secs}s $${(rec.usd || 0).toFixed(4)}`
599 + (rec.ok ? ` — ${String(rec.ok).slice(0, 70)}` : '')
600 + (rec.err ? ` [${rec.err}]` : ''));
601const once = async (model, reg, i) => {
602 const tag = `${model.replace(/[^a-z0-9]+/gi, '_')}__${reg}__${i}`;
603 let rec;
604 try { rec = await oneRun(key, model, reg, tag); }
605 catch (e) { rec = { model, reg, err: String(e.message || e), ok: 'threw', calls: 0, bytes: 0 }; }
606 rows.push(rec);
607 say(rec, i);
608 if (out) fs.writeFileSync(out, JSON.stringify(rows, null, '\t'));
609 return rec;
610};
611
612for (const model of MODELS) {
613 // THE GATE, and it runs before a penny goes on any register.
614 if (GATE) {
615 const base = [];
616 for (let i = 0; i < Math.max(2, N); i++) base.push(await once(model, 'none', i));
617 // **The bar is MOVING the paragraph, not writing a file.** Written as "wrote something
618 // different" it passed `openai/gpt-oss-120b`, which reads the draft, writes a near-copy
619 // of it back -- mean sentence 35.2 words against the draft's 35, three bare samples out
620 // of three -- and declares the work done. A model that hands back what it was given has
621 // not shown it can do the task, and its register arms would measure nothing.
622 const moved = base.filter((r) => typeof r.mean === 'number' && bandDistance(r) < DRAFT_DIST).length;
623 if (!moved) {
624 console.log(`${model.padEnd(28)} UNMEASURABLE: in ${base.length} bare samples it never `
625 + `moved the paragraph past the draft's own score, so nothing it does with a `
626 + 'register is evidence about the register. Its arms are not run.');
627 continue;
628 }
629 console.log(`${model.padEnd(28)} gate passed: moved the paragraph in ${moved}/${base.length} bare samples.`);
630 }
631 for (const reg of REGS) {
632 if (GATE && reg === 'none') continue;
633 for (let i = 0; i < N; i++) await once(model, reg, i);
634 }
635}
636report(rows);