Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/probe_thinking_live.mjs

4.7 KiB, 1 run

created by r2519314175:87, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// The model's working, shown in the real app, by a real model — the external oracle.
2//
3// `dev/verify_thinking.mjs` proves the behaviour against the mock, which is the mock
4// doing what this repository told it to. That is worth having and it is not evidence
5// about GLM or DeepSeek. This drives the same page against OpenRouter, on the two
6// models the finding was measured on, and reports what a person would have seen: how
7// long after pressing Send the first of the model's working reached the screen, how
8// long until the first word of the ANSWER did, and how much working there was in
9// between. The gap between those two numbers is the wait this whole change is about.
10//
11// IT SPENDS REAL MONEY. A run is a few tenths of a cent per model; the figure is
12// printed at the end from OpenRouter's own `usage` block rather than estimated. It is
13// NOT part of `dev/run_all.sh` and must never be added to it.
14//
15// OPENROUTER_KEY=$(cat ~/.config/oxedyne/<...>) node dev/probe_thinking_live.mjs
16// ... node dev/probe_thinking_live.mjs --model z-ai/glm-5.2
17//
18// First run, 2026-08-28, world 12, this tree: see the table this file prints.
19
20import path from 'node:path';
21import { fileURLToPath } from 'node:url';
22
23const HERE = path.dirname(fileURLToPath(import.meta.url));
24const H = await import(path.join(HERE, 'harness.mjs'));
25const { open, newChat, connectMock, scratch, shot } = H;
26
27const KEY = process.env.OPENROUTER_KEY;
28if (!KEY) {
29 console.error('Set OPENROUTER_KEY before running this. It spends real money, so it '
30 + 'will not run without one, and no key is written into this file.');
31 process.exit(2);
32}
33const arg = (name, dflt) => {
34 const i = process.argv.indexOf('--' + name);
35 return i === -1 ? dflt : process.argv[i + 1];
36};
37const MODELS = arg('model', 'z-ai/glm-5.2,deepseek/deepseek-v4-pro').split(',');
38const ASK = arg('ask', 'A shop sells pens at 43p and pencils at 27p. Someone spends '
39 + 'exactly £10.00 on 30 items. How many of each? Show your working.');
40
41const s = await open({ name: 'thinklive', connect: false, profile: scratch('pw', 'thinklive') });
42
43const rows = [];
44for (const model of MODELS) {
45 const cfg = await connectMock(s,
46 { baseUrl: 'https://openrouter.ai/api/v1/chat/completions', model, apiKey: KEY });
47 console.log(`connected: ${JSON.stringify(cfg)}`);
48 await H.clearChats(s);
49 await newChat(s);
50
51 await s.page.fill('#chat-input', ASK);
52 const t0 = Date.now();
53 await s.page.click('#chat-send', { force: true });
54
55 let firstThink = null, firstAnswer = null, thinkChars = 0, tiles = 0;
56 while (Date.now() - t0 < 240000) {
57 const now = await s.page.evaluate(() => {
58 const t = Array.from(document.querySelectorAll('.chat-msg-thinking'));
59 const body = t.map(x => String(x.querySelector('.chat-thinking-body').textContent || '')).join('');
60 const a = document.querySelector('.chat-msg-assistant');
61 const b = document.getElementById('chat-send');
62 const busy = b ? (/stop/i.test((b.getAttribute('title') || '') + (b.className || '')) || b.disabled) : false;
63 return { tiles: t.length, think: body.length, open: t.length ? t[0].open : null,
64 answer: a ? String(a.textContent || '').trim().length : 0, busy };
65 });
66 if (firstThink === null && now.think > 0) {
67 firstThink = Date.now() - t0;
68 await shot(s, `thinklive-${model.replace(/[^a-z0-9]+/gi, '-')}-firstthink`);
69 }
70 if (firstAnswer === null && now.answer > 0) firstAnswer = Date.now() - t0;
71 thinkChars = Math.max(thinkChars, now.think);
72 tiles = Math.max(tiles, now.tiles);
73 // The turn has to have STARTED before its stopping means anything: the send
74 // button is idle for the first moment too.
75 if (!now.busy && (firstThink !== null || firstAnswer !== null)) break;
76 if (!now.busy && (Date.now() - t0) > 20000) { console.log(' (the turn never started)'); break; }
77 await s.page.waitForTimeout(120);
78 }
79 const total = Date.now() - t0;
80 await shot(s, `thinklive-${model.replace(/[^a-z0-9]+/gi, '-')}-done`);
81 const spent = await s.page.evaluate(() => {
82 try { return JSON.parse(localStorage.getItem('daimond-spend') || 'null'); } catch { return null; }
83 });
84 rows.push({ model, firstThink, firstAnswer, thinkChars, tiles, total, spent });
85 console.log(`\n${model}`);
86 console.log(` first WORKING on screen ${firstThink === null ? 'never' : firstThink + ' ms'}`);
87 console.log(` first ANSWER on screen ${firstAnswer === null ? 'never' : firstAnswer + ' ms'}`);
88 console.log(` working shown ${thinkChars} characters in ${tiles} tile(s)`);
89 console.log(` round total ${total} ms`);
90 console.log(` BLANK SPINNER BEFORE ${firstAnswer === null ? 'n/a' : firstAnswer + ' ms'}`);
91 console.log(` BLANK SPINNER NOW ${firstThink === null ? 'unchanged' : firstThink + ' ms'}`);
92}
93
94console.log('\n' + JSON.stringify(rows, null, 1));
95await s.close();