Oregami
Repositories/oxedyne/daimond

oxedyne/daimond/dev/wf_codefix.mjs

6.0 KiB, 1 run

created by r2519314175:837, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1// Workflow 1 — real code task with ground-truth verification.
2// Seed the rustcalc project into the OPFS workspace, ask a real model to find and
3// fix the median bug and document the API, then export the edited file back to
4// disk and run `cargo test` to prove the fix is real — not just plausible text.
5import fs from 'node:fs';
6import { execSync } from 'node:child_process';
7import { open, chat, errors } from './harness.mjs';
8
9const cfg = JSON.parse(fs.readFileSync('dev/.secrets/testcfg.json', 'utf8'));
10const MODEL = cfg.models.value; // gpt-oss-120b (value)
11// The rustcalc fixture is not in this repository — it is a scratch project one
12// developer keeps outside it, so the path is a setting and not a constant.
13// `dev/` is carved into a public mirror, and a home directory baked in here is
14// both a leak and a script nobody else can run.
15const FIXTURE = process.env.DAIMOND_RUSTCALC
16 || `${process.env.HOME}/usr/scratch/daimond-test/rustcalc`;
17if (!fs.existsSync(FIXTURE)) {
18 console.log(`SKIP wf_codefix — no rustcalc fixture at ${FIXTURE}; set DAIMOND_RUSTCALC.`);
19 process.exit(0);
20}
21const SEED = fs.readFileSync(`${FIXTURE}/src/lib.rs`, 'utf8');
22
23function connectReal(page) {
24 return page.evaluate(async (c) => {
25 document.getElementById('settings-btn')?.click();
26 await new Promise(r => setTimeout(r, 250));
27 const prov = document.getElementById('cfg-provider'); prov.value='custom'; prov.dispatchEvent(new Event('change',{bubbles:true}));
28 await new Promise(r=>setTimeout(r,200));
29 const url=document.getElementById('cfg-base-url'); url.value=c.baseUrl; url.dispatchEvent(new Event('input',{bubbles:true})); url.dispatchEvent(new Event('change',{bubbles:true}));
30 const key=document.getElementById('cfg-api-key'); key.value=c.apiKey; key.dispatchEvent(new Event('input',{bubbles:true})); key.dispatchEvent(new Event('change',{bubbles:true}));
31 await new Promise(r=>setTimeout(r,1500));
32 const cus=document.getElementById('cfg-model-custom'); if(cus){cus.style.display='';cus.value=c.model;cus.dispatchEvent(new Event('input',{bubbles:true}));}
33 document.getElementById('byok-save')?.click();
34 }, c => c, cfg);
35}
36const spendOf = (page) => page.evaluate(() => { try { return JSON.parse(localStorage.getItem('daimond-ledger')||'[]').reduce((a,e)=>a+(e.u||0),0); } catch { return 0; } });
37
38const s = await open({ name: 'wf-codefix', connect: false });
39await s.page.evaluate(async (c)=>{ // inline connectReal (evaluate can't take a fn ref easily)
40 document.getElementById('settings-btn')?.click(); await new Promise(r=>setTimeout(r,250));
41 const prov=document.getElementById('cfg-provider'); prov.value='custom'; prov.dispatchEvent(new Event('change',{bubbles:true})); await new Promise(r=>setTimeout(r,200));
42 const url=document.getElementById('cfg-base-url'); url.value=c.baseUrl; url.dispatchEvent(new Event('input',{bubbles:true})); url.dispatchEvent(new Event('change',{bubbles:true}));
43 const key=document.getElementById('cfg-api-key'); key.value=c.apiKey; key.dispatchEvent(new Event('input',{bubbles:true})); key.dispatchEvent(new Event('change',{bubbles:true}));
44 await new Promise(r=>setTimeout(r,1500));
45 const cus=document.getElementById('cfg-model-custom'); if(cus){cus.style.display='';cus.value=c.model;cus.dispatchEvent(new Event('input',{bubbles:true}));}
46 document.getElementById('byok-save')?.click();
47}, { baseUrl: cfg.baseUrl, apiKey: cfg.apiKey, model: MODEL });
48await s.page.waitForTimeout(1500);
49
50// Seed the file into OPFS so the agent's file tools can read/edit it.
51await s.page.evaluate(async (src) => {
52 const root = await navigator.storage.getDirectory();
53 const dir = await root.getDirectoryHandle('rustcalc', { create: true });
54 const sub = await dir.getDirectoryHandle('src', { create: true });
55 const fh = await sub.getFileHandle('lib.rs', { create: true });
56 const w = await fh.createWritable(); await w.write(src); await w.close();
57}, SEED);
58
59console.log('model:', MODEL);
60const t0 = Date.now();
61const task = 'The file rustcalc/src/lib.rs has a bug in the `median` function: for an '
62 + 'even-length slice it returns a single middle element instead of the average of '
63 + 'the two middle elements, so the test `median_even` fails. Read the file, fix the '
64 + 'median function so median_even passes, and add a short /// doc comment to each of '
65 + 'the three public functions (mean, median, stddev). Use the file tools. When done, say DONE.';
66const out = await chat(s, task, { timeout: 180000 });
67console.log('elapsed:', ((Date.now()-t0)/1000).toFixed(1)+'s');
68console.log('--- transcript tail ---\n' + out.slice(-500));
69
70// Pull the edited file back out of OPFS and verify with cargo test.
71const edited = await s.page.evaluate(async () => {
72 try { const root=await navigator.storage.getDirectory(); const d=await root.getDirectoryHandle('rustcalc'); const sub=await d.getDirectoryHandle('src'); const fh=await sub.getFileHandle('lib.rs'); return await (await fh.getFile()).text(); }
73 catch(e){ return '(missing: '+e+')'; }
74});
75const verifyDir = `${FIXTURE}-verify`;
76fs.rmSync(verifyDir, { recursive: true, force: true });
77fs.mkdirSync(verifyDir+'/src', { recursive: true });
78fs.copyFileSync(`${FIXTURE}/Cargo.toml`, verifyDir+'/Cargo.toml');
79fs.writeFileSync(verifyDir+'/src/lib.rs', edited);
80let testOut = '', pass = false;
81try { testOut = execSync('cargo test 2>&1', { cwd: verifyDir, encoding: 'utf8' }); pass = /test result: ok/.test(testOut) && !/FAILED/.test(testOut); }
82catch (e) { testOut = (e.stdout||'') + (e.stderr||''); pass = false; }
83const docs = (edited.match(/^\/\/\/ /gm) || []).length;
84
85const spend = await spendOf(s.page);
86const result = {
87 workflow: 'codefix', model: MODEL, elapsedS: ((Date.now()-t0)/1000).toFixed(1),
88 cargoTest: /test result: ([^\n]+)/.exec(testOut)?.[1] || 'no result line',
89 testsPass: pass, docCommentsAdded: docs, spendUsd: +spend.toFixed(4),
90 consoleErrors: errors(s),
91};
92fs.writeFileSync('dev/results/codefix.json', JSON.stringify(result, null, 2));
93console.log('\n=== RESULT ===\n' + JSON.stringify(result, null, 2));
94await s.close();