Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/probe_reasoning_ab.mjs

5.1 KiB, 1 run

created by r2519314175:69, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// What `reasoning: {exclude: true}` is actually worth — the A/B, 2026-08-28.
2//
3// A single sample of each had suggested that asking OpenRouter to omit the reasoning
4// made a round twice as fast. It also showed the reasoning-token count moving between
5// runs, which is enough to explain the whole difference on its own, so the wall clock
6// was confounded and could not be quoted. This runs the pair INTERLEAVED -- default and
7// then excluded, back to back, same task, same minute -- so provider load drifts across
8// both arms rather than into one, and it reports the reasoning-token count beside every
9// timing so the confound is visible rather than assumed.
10//
11// It reports, per arm: seconds, bytes on the wire, SSE chunks, prompt/completion/reasoning
12// tokens, and the cost OpenRouter itself charged. Bytes and cost are what the arms can be
13// compared on honestly. Seconds are printed and should be read with the token counts.
14//
15// IT SPENDS REAL MONEY. About a third of a cent a round. Not part of dev/run_all.sh.
16//
17// OPENROUTER_KEY=... node dev/probe_reasoning_ab.mjs [--n 5] [--models a,b]
18
19const KEY = process.env.OPENROUTER_KEY;
20if (!KEY) {
21 console.error('Set OPENROUTER_KEY. This spends real money and no key lives in this file.');
22 process.exit(2);
23}
24const arg = (n, d) => { const i = process.argv.indexOf('--' + n); return i === -1 ? d : process.argv[i + 1]; };
25const N = Number(arg('n', 5));
26const MODELS = String(arg('models', 'z-ai/glm-5.2,deepseek/deepseek-v4-pro')).split(',');
27
28// One task, fixed, and hard enough that the model actually reasons. Every arm of every
29// sample sends exactly this.
30const TASK = 'A shop sells pens at 43p and pencils at 27p. Someone spends exactly £10.00 '
31 + 'on 30 items. How many of each? Show your working.';
32
33async function once(model, exclude) {
34 const body = {
35 model,
36 messages: [{ role: 'user', content: TASK }],
37 stream: true,
38 max_tokens: 2000,
39 stream_options: { include_usage: true },
40 };
41 if (exclude) body.reasoning = { exclude: true };
42 const t0 = Date.now();
43 const res = await fetch('https://openrouter.ai/api/v1/chat/completions', {
44 method: 'POST',
45 headers: { authorization: 'Bearer ' + KEY, 'content-type': 'application/json' },
46 body: JSON.stringify(body),
47 });
48 if (!res.ok) throw new Error(`${model}: HTTP ${res.status} ${await res.text()}`);
49 const reader = res.body.getReader();
50 let bytes = 0, chunks = 0, buf = '', usage = null, firstByteMs = null;
51 const dec = new TextDecoder();
52 for (;;) {
53 const { done, value } = await reader.read();
54 if (done) break;
55 if (firstByteMs === null) firstByteMs = Date.now() - t0;
56 bytes += value.length;
57 buf += dec.decode(value, { stream: true });
58 let i;
59 while ((i = buf.indexOf('\n\n')) !== -1) {
60 const ev = buf.slice(0, i); buf = buf.slice(i + 2);
61 const line = ev.split('\n').find(l => l.startsWith('data: '));
62 if (!line) continue;
63 const data = line.slice(6);
64 if (data === '[DONE]') continue;
65 chunks++;
66 try { const j = JSON.parse(data); if (j.usage) usage = j.usage; } catch {}
67 }
68 }
69 const ms = Date.now() - t0;
70 const u = usage || {};
71 return {
72 ms, firstByteMs, bytes, chunks,
73 prompt: u.prompt_tokens || 0,
74 completion: u.completion_tokens || 0,
75 reasoning: (u.completion_tokens_details || {}).reasoning_tokens || 0,
76 cost: u.cost || 0,
77 };
78}
79
80const rows = [];
81for (const model of MODELS) {
82 for (let i = 0; i < N; i++) {
83 // INTERLEAVED, and in this order every time: whatever the provider is doing at
84 // this minute, both arms are inside it.
85 for (const exclude of [false, true]) {
86 let r;
87 try { r = await once(model, exclude); }
88 catch (e) { console.error(' ' + e.message); continue; }
89 rows.push({ model, arm: exclude ? 'exclude' : 'default', i, ...r });
90 console.log(`${model.padEnd(26)} ${(exclude ? 'exclude' : 'default').padEnd(8)}`
91 + ` #${i + 1} ${(r.ms / 1000).toFixed(1)}s ${(r.bytes / 1024).toFixed(0)}KB`
92 + ` ${String(r.chunks).padStart(5)} chunks`
93 + ` reason ${String(r.reasoning).padStart(5)} out ${String(r.completion).padStart(5)}`
94 + ` $${r.cost.toFixed(6)}`);
95 }
96 }
97}
98
99const med = (a) => { const s = [...a].sort((x, y) => x - y); return s.length ? s[Math.floor(s.length / 2)] : 0; };
100const sum = (a) => a.reduce((x, y) => x + y, 0);
101console.log('\n════ medians, and the total billed ════');
102console.log('model arm n median s median KB median chunks median reasoning tok billed $');
103for (const model of MODELS) {
104 for (const arm of ['default', 'exclude']) {
105 const r = rows.filter(x => x.model === model && x.arm === arm);
106 if (!r.length) continue;
107 console.log(`${model.padEnd(26)} ${arm.padEnd(8)} ${String(r.length).padStart(2)}`
108 + ` ${(med(r.map(x => x.ms)) / 1000).toFixed(1).padStart(8)}`
109 + ` ${(med(r.map(x => x.bytes)) / 1024).toFixed(0).padStart(9)}`
110 + ` ${String(med(r.map(x => x.chunks))).padStart(13)}`
111 + ` ${String(med(r.map(x => x.reasoning))).padStart(20)}`
112 + ` ${sum(r.map(x => x.cost)).toFixed(5).padStart(8)}`);
113 }
114}
115console.log(`\ntotal billed this run: $${sum(rows.map(r => r.cost)).toFixed(5)}`);
116console.log('\n' + JSON.stringify(rows));