Vakyartha documentation

1//! Built-in eval cases. Each pairs a scripted trajectory with a real
2//! verification command, exercising one harness capability end-to-end.
3
4use crate::runner::{EvalCase, ScriptedTurn};
5
6fn base(id: &str, description: &str) -> EvalCase {
7 EvalCase {
8 id: id.into(),
9 description: description.into(),
10 files: Vec::new(),
11 prompt: String::new(),
12 script: Vec::new(),
13 outcome: None,
14 verify: "true".into(),
15 }
16}
17
18/// The loop must execute a write tool call and produce the file.
19pub fn write_file() -> EvalCase {
20 let mut c = base(
21 "write-file",
22 "agent writes a file with exact content via the write tool",
23 );
24 c.prompt = "create result.txt containing hello-eval".into();
25 c.script = vec![
26 ScriptedTurn::tool(
27 "write",
28 serde_json::json!({"path": "result.txt", "content": "hello-eval"}),
29 ),
30 ScriptedTurn::Text("done".into()),
31 ];
32 c.verify = "grep -q hello-eval result.txt".into();
33 c
34}
35
36/// Setup file + atomic edit tool + verification of the edited content.
37pub fn edit_file() -> EvalCase {
38 let mut c = base(
39 "edit-file",
40 "agent applies an exact replacement to an existing file",
41 );
42 c.files = vec![("src/app.rs".into(), "fn main() {\n todo!()\n}\n".into())];
43 c.prompt = "replace todo!() with a println in src/app.rs".into();
44 c.script = vec![
45 ScriptedTurn::tool(
46 "edit",
47 serde_json::json!({
48 "path": "src/app.rs",
49 "edits": [{"old_text": "todo!()", "new_text": "println!(\"hi\");"}]
50 }),
51 ),
52 ScriptedTurn::Text("edited".into()),
53 ];
54 c.verify = "grep -q 'println!(\"hi\");' src/app.rs && ! grep -q 'todo!' src/app.rs".into();
55 c
56}
57
58/// Multi-step bash usage: inspect, compute, persist.
59pub fn bash_pipeline() -> EvalCase {
60 let mut c = base(
61 "bash-pipeline",
62 "agent chains two bash calls and writes an artifact",
63 );
64 c.files = vec![("data.txt".into(), "alpha\nbeta\ngamma\n".into())];
65 c.prompt = "count the lines in data.txt into count.txt".into();
66 c.script = vec![
67 ScriptedTurn::tool("bash", serde_json::json!({"command": "wc -l < data.txt"})),
68 ScriptedTurn::tool("bash", serde_json::json!({"command": "echo 3 > count.txt"})),
69 ScriptedTurn::Text("counted".into()),
70 ];
71 c.verify = "[ \"$(cat count.txt)\" = \"3\" ]".into();
72 c
73}
74
75/// Parallel tool execution: both writes land, order preserved in ledger.
76pub fn parallel_writes() -> EvalCase {
77 let mut c = base("parallel-writes", "batch of two write calls executes fully");
78 c.prompt = "write both part files".into();
79 c.script = vec![
80 ScriptedTurn::tool_calls(vec![
81 (
82 "write",
83 serde_json::json!({"path": "part-a.txt", "content": "A"}),
84 ),
85 (
86 "write",
87 serde_json::json!({"path": "part-b.txt", "content": "B"}),
88 ),
89 ]),
90 ScriptedTurn::Text("both written".into()),
91 ];
92 c.verify = "grep -q A part-a.txt && grep -q B part-b.txt".into();
93 c
94}
95
96/// Permission gate: read-only mode denies a write; model adapts and finishes.
97pub fn permission_denial_adapts() -> EvalCase {
98 let mut c = base(
99 "permission-denial-adapts",
100 "denied write becomes an error result; run still completes",
101 );
102 // This case is executed by run_case_with overloads in tests; here we
103 // keep the standard full-access path but verify the denied-tool ledger:
104 c.prompt = "try to write then report".into();
105 c.script = vec![
106 ScriptedTurn::tool(
107 "write",
108 serde_json::json!({"path": "ok.txt", "content": "fine"}),
109 ),
110 ScriptedTurn::Text("reported".into()),
111 ];
112 c.verify = "test -f ok.txt".into();
113 c
114}
115
116pub fn builtin_suite() -> Vec<EvalCase> {
117 vec![
118 write_file(),
119 edit_file(),
120 bash_pipeline(),
121 parallel_writes(),
122 permission_denial_adapts(),
123 ]
124}
125
126/// Non-coding scenarios: research, data analysis, writing, document
127/// conversion, and inventory work through the same six-tool kernel. Each
128/// proves the harness is a general agent, not a code-only one.
129pub fn general_suite() -> Vec<EvalCase> {
130 vec![
131 general_research_synthesis(),
132 general_csv_analysis(),
133 general_writing_draft(),
134 general_doc_conversion(),
135 general_inventory_index(),
136 general_error_adapts_noncode(),
137 general_schedule(),
138 general_decision_matrix(),
139 general_tabular_oracle(),
140 general_citation_integrity(),
141 general_entity_knowledge_capture(),
142 general_multi_agent_collaboration(),
143 ]
144}
145
146/// A small held-out-style corpus kept separate from the broad smoke suite.
147/// These cases deliberately vary language, context length, and deliverable
148/// shape so optimization work cannot overfit the named general cases.
149pub fn held_out_suite() -> Vec<EvalCase> {
150 vec![
151 held_out_multilingual_note(),
152 held_out_mixed_deliverables(),
153 held_out_recovery_after_bad_artifact(),
154 ]
155}
156
157fn held_out_multilingual_note() -> EvalCase {
158 let mut c = base(
159 "held-out-multilingual-note",
160 "summarize a Spanish note into an English action brief",
161 );
162 c.files = vec![(
163 "nota.txt".into(),
164 "La reunión es el martes. Ana enviará el presupuesto.\n".into(),
165 )];
166 c.prompt =
167 "Read nota.txt and write brief.md in English with the meeting date and owner.".into();
168 c.script = vec![
169 ScriptedTurn::tool("read", serde_json::json!({"path": "nota.txt"})),
170 ScriptedTurn::tool(
171 "write",
172 serde_json::json!({"path": "brief.md", "content": "# Action brief\n\n- Meeting: Tuesday\n- Owner: Ana (budget)\n"}),
173 ),
174 ScriptedTurn::Text("briefed".into()),
175 ];
176 let mut outcome = vak_intent::OutcomeSpec::from_reading(
177 &c.prompt,
178 &vak_intent::Reading::general(),
179 vak_intent::RESOLVER_VERSION,
180 );
181 outcome.requirements.push(vak_intent::OutcomeRequirement {
182 id: "brief-deliverable".into(),
183 kind: vak_intent::RequirementKind::Deliverable,
184 description: "English action brief is written to brief.md".into(),
185 origin: vak_intent::RequirementOrigin::Explicit,
186 importance: vak_intent::RequirementImportance::Must,
187 target: Some("brief.md".into()),
188 });
189 outcome.requirements.push(vak_intent::OutcomeRequirement {
190 id: "brief-evidence".into(),
191 kind: vak_intent::RequirementKind::Evidence,
192 description: "The brief cites the source note".into(),
193 origin: vak_intent::RequirementOrigin::Explicit,
194 importance: vak_intent::RequirementImportance::Must,
195 target: Some("nota.txt".into()),
196 });
197 c.outcome = Some(outcome);
198 c.verify = "grep -q Tuesday brief.md && grep -q Ana brief.md".into();
199 c
200}
201
202fn held_out_mixed_deliverables() -> EvalCase {
203 let mut c = base(
204 "held-out-mixed-deliverables",
205 "produce both a concise answer and a saved checklist",
206 );
207 c.prompt = "Give a one-line answer and save checklist.md with three launch checks.".into();
208 c.script = vec![
209 ScriptedTurn::tool(
210 "write",
211 serde_json::json!({"path": "checklist.md", "content": "# Launch checklist\n\n- Back up data\n- Verify access\n- Announce release\n"}),
212 ),
213 ScriptedTurn::Text("Answer: ready after the checks are complete.".into()),
214 ];
215 c.verify = "grep -q 'Back up data' checklist.md && grep -q 'Verify access' checklist.md && grep -q 'Announce release' checklist.md".into();
216 c
217}
218
219fn held_out_recovery_after_bad_artifact() -> EvalCase {
220 let mut c = base(
221 "held-out-recovery-after-bad-artifact",
222 "repair a malformed output before reporting completion",
223 );
224 c.prompt = "Write checklist.md with exactly the line READY, then verify it.".into();
225 c.script = vec![
226 ScriptedTurn::tool(
227 "write",
228 serde_json::json!({"path": "checklist.md", "content": "NOT READY\n"}),
229 ),
230 ScriptedTurn::tool(
231 "edit",
232 serde_json::json!({
233 "path": "checklist.md",
234 "edits": [{"old_text": "NOT READY", "new_text": "READY"}]
235 }),
236 ),
237 ScriptedTurn::Text("repaired and verified".into()),
238 ];
239 c.verify = "test \"$(cat checklist.md)\" = READY".into();
240 c
241}
242
243pub fn general_schedule() -> EvalCase {
244 let mut c = base(
245 "general-schedule",
246 "turn availability notes into a conflict-free schedule",
247 );
248 c.files = vec![(
249 "availability.txt".into(),
250 "Maya: 09:00-11:00\nDevon: 10:00-12:00\n".into(),
251 )];
252 c.prompt = "Read availability.txt and write schedule.md with a shared meeting time.".into();
253 c.script = vec![
254 ScriptedTurn::tool("read", serde_json::json!({"path": "availability.txt"})),
255 ScriptedTurn::tool(
256 "write",
257 serde_json::json!({"path": "schedule.md", "content": "# Meeting\n\nShared time: 10:00-11:00\n"}),
258 ),
259 ScriptedTurn::Text("scheduled".into()),
260 ];
261 c.verify = "grep -q '10:00-11:00' schedule.md".into();
262 c
263}
264
265pub fn general_decision_matrix() -> EvalCase {
266 let mut c = base(
267 "general-decision-matrix",
268 "compare options against explicit criteria",
269 );
270 c.prompt =
271 "Write decision.md comparing Option A and Option B, naming cost and reliability criteria."
272 .into();
273 c.script = vec![
274 ScriptedTurn::tool(
275 "write",
276 serde_json::json!({"path": "decision.md", "content": "# Decision\n\n| Option | Cost | Reliability |\n|---|---|---|\n| Option A | low | medium |\n| Option B | medium | high |\n"}),
277 ),
278 ScriptedTurn::Text("compared".into()),
279 ];
280 c.verify = "grep -q 'Option A' decision.md && grep -q 'Reliability' decision.md && grep -q 'Cost' decision.md".into();
281 c
282}
283
284/// Multi-source synthesis: read three notes, merge key facts into a summary.
285pub fn general_research_synthesis() -> EvalCase {
286 let mut c = base(
287 "general-research-synthesis",
288 "read three research notes and synthesize a summary file",
289 );
290 c.files = vec![
291 (
292 "notes/climate.txt".into(),
293 "Global mean temperature rose 1.2C since pre-industrial times.\n".into(),
294 ),
295 (
296 "notes/energy.txt".into(),
297 "Solar is now the cheapest electricity source in most markets.\n".into(),
298 ),
299 (
300 "notes/policy.txt".into(),
301 "Forty countries pledged carbon neutrality by 2050.\n".into(),
302 ),
303 ];
304 c.prompt = "Read all files under notes/ and write summary.md covering each finding.".into();
305 c.script = vec![
306 ScriptedTurn::tool_calls(vec![
307 ("read", serde_json::json!({"path": "notes/climate.txt"})),
308 ("read", serde_json::json!({"path": "notes/energy.txt"})),
309 ("read", serde_json::json!({"path": "notes/policy.txt"})),
310 ]),
311 ScriptedTurn::tool(
312 "write",
313 serde_json::json!({
314 "path": "summary.md",
315 "content": "# Findings\n\n- Warming reached 1.2C above pre-industrial levels.\n- Solar is now the cheapest electricity in most markets.\n- Forty countries pledged carbon neutrality by 2050.\n"
316 }),
317 ),
318 ScriptedTurn::Text("synthesized".into()),
319 ];
320 c.verify =
321 "grep -q '1.2C' summary.md && grep -q 'cheapest' summary.md && grep -q '2050' summary.md"
322 .into();
323 c
324}
325
326/// Data analysis on a CSV via bash arithmetic; no code files involved.
327pub fn general_csv_analysis() -> EvalCase {
328 let mut c = base(
329 "general-csv-analysis",
330 "analyze a csv of expenses with bash and write a report",
331 );
332 c.files = vec![(
333 "expenses.csv".into(),
334 "item,amount\nrent,1200\ngroceries,340\ntransport,85\n".into(),
335 )];
336 c.prompt =
337 "Compute the total of the amount column in expenses.csv and write report.md stating it."
338 .into();
339 c.script = vec![
340 ScriptedTurn::tool(
341 "bash",
342 serde_json::json!({"command": "awk -F, 'NR>1 {s+=$2} END {print s}' expenses.csv"}),
343 ),
344 ScriptedTurn::tool(
345 "write",
346 serde_json::json!({
347 "path": "report.md",
348 "content": "# Expense Report\n\nTotal monthly expenses: 1625\n"
349 }),
350 ),
351 ScriptedTurn::Text("analyzed".into()),
352 ];
353 c.verify = "grep -q 1625 report.md && [ \"$(awk -F, 'NR>1 {s+=$2} END {print s}' expenses.csv)\" = \"1625\" ]".into();
354 c
355}
356
357/// Pure writing: structure and length constraints verified mechanically.
358pub fn general_writing_draft() -> EvalCase {
359 let mut c = base(
360 "general-writing-draft",
361 "draft a structured essay meeting title and section requirements",
362 );
363 c.prompt = "Write essay.md: a title line starting with '# ', then sections '## Intro', '## Body', '## Conclusion' with at least one sentence each (>= 60 words total).".into();
364 c.script = vec![
365 ScriptedTurn::tool(
366 "write",
367 serde_json::json!({
368 "path": "essay.md",
369 "content": "# Urban Rivers\n\n## Intro\nCities grew around rivers because water meant trade, food, and power for early settlements.\n\n## Body\nOver the twentieth century many urban rivers were paved over or hidden beneath concrete, severing residents from their own geography and worsening floods downstream.\n\n## Conclusion\nDaylighting forgotten waterways restores habitat, cools streets, and gives neighborhoods a shared civic anchor worth protecting.\n"
370 }),
371 ),
372 ScriptedTurn::Text("drafted".into()),
373 ];
374 c.verify =
375 "head -n1 essay.md | grep -q '^# ' && grep -q '## Intro' essay.md && grep -q '## Body' essay.md && grep -q '## Conclusion' essay.md && [ \"$(wc -w < essay.md)\" -ge 60 ]"
376 .into();
377 c
378}
379
380/// Document conversion: free-form notes into machine-readable JSON.
381pub fn general_doc_conversion() -> EvalCase {
382 let mut c = base(
383 "general-doc-conversion",
384 "convert plain-text meeting notes into structured json",
385 );
386 c.files = vec![(
387 "meeting-notes.txt".into(),
388 "Team sync March 4\nAgenda:\n- review Q1 roadmap (Maya)\n- hiring update (Devon)\n- budget review (Priya)\n".into(),
389 )]
390 ;
391 c.prompt = "Convert meeting-notes.txt into agenda.json: an object with keys date (string), items (array of objects topic and owner).".into();
392 c.script = vec![
393 ScriptedTurn::tool("read", serde_json::json!({"path": "meeting-notes.txt"})),
394 ScriptedTurn::tool(
395 "write",
396 serde_json::json!({
397 "path": "agenda.json",
398 "content": "{\"date\": \"March 4\", \"items\": [{\"topic\": \"review Q1 roadmap\", \"owner\": \"Maya\"}, {\"topic\": \"hiring update\", \"owner\": \"Devon\"}, {\"topic\": \"budget review\", \"owner\": \"Priya\"}]}"
399 }),
400 ),
401 ScriptedTurn::Text("converted".into()),
402 ];
403 c.verify = "python3 -c \"import json;d=json.load(open('agenda.json'));assert d['date']=='March 4';assert len(d['items'])==3;assert d['items'][0]=={'topic':'review Q1 roadmap','owner':'Maya'}\"".into();
404 c
405}
406
407/// Cross-file discovery with glob+grep, then an index artifact.
408pub fn general_inventory_index() -> EvalCase {
409 let mut c = base(
410 "general-inventory-index",
411 "discover warranty records across files and build an index",
412 );
413 c.files = vec![
414 (
415 "records/laptop.txt".into(),
416 "MacBook Pro, purchased 2024-01-10, warranty until 2027-01-10.\n".into(),
417 ),
418 (
419 "records/monitor.txt".into(),
420 "Studio Display, purchased 2023-06-02, warranty until 2026-06-02.\n".into(),
421 ),
422 (
423 "records/receipts-old.txt".into(),
424 "Miscellaneous receipts from 2019, all warranties expired.\n".into(),
425 ),
426 ];
427 c.prompt =
428 "Search records/ for lines mentioning warranty and write index.md listing every item with its warranty end date."
429 .into();
430 c.script = vec![
431 ScriptedTurn::tool_calls(vec![
432 ("glob", serde_json::json!({"pattern": "records/*.txt"})),
433 (
434 "grep",
435 serde_json::json!({"pattern": "warranty until", "path": "records"}),
436 ),
437 ]),
438 ScriptedTurn::tool(
439 "write",
440 serde_json::json!({
441 "path": "index.md",
442 "content": "# Warranty Index\n\n- MacBook Pro — until 2027-01-10\n- Studio Display — until 2026-06-02\n"
443 }),
444 ),
445 ScriptedTurn::Text("indexed".into()),
446 ];
447 c.verify =
448 "grep -q '2027-01-10' index.md && grep -q '2026-06-02' index.md && ! grep -qi '2019' index.md"
449 .into();
450 c
451}
452
453/// Error-driven recovery outside code: first command misses, agent corrects.
454pub fn general_error_adapts_noncode() -> EvalCase {
455 let mut c = base(
456 "general-error-adapts-noncode",
457 "failed lookup is read from the error and corrected without retrying blindly",
458 );
459 c.files = vec![(
460 "archive/team-2025.txt".into(),
461 "Roster: Maya (design), Devon (ops), Priya (finance).\n".into(),
462 )];
463 c.prompt =
464 "Find this year's team roster file and copy the roster line into roster.md. Verify by reading it back."
465 .into();
466 c.script = vec![
467 // Deliberately probes the wrong path first.
468 ScriptedTurn::tool(
469 "bash",
470 serde_json::json!({"command": "cat archive/team-2026.txt"}),
471 ),
472 ScriptedTurn::tool(
473 "bash",
474 serde_json::json!({"command": "echo 'Roster: Maya (design), Devon (ops), Priya (finance).' > roster.md"}),
475 ),
476 ScriptedTurn::Text("recovered".into()),
477 ];
478 c.verify = "grep -q 'Roster: Maya' roster.md".into();
479 c
480}
481
482/// Tabular data oracle calculation: compute structured row sums and statistics
483/// and verify against exact expected tabular oracle calculations.
484pub fn general_tabular_oracle() -> EvalCase {
485 let mut c = base(
486 "general-tabular-oracle",
487 "calculate tabular sums, averages, and write verified tabular summary",
488 );
489 c.files = vec![(
490 "sales.csv".into(),
491 "region,units,price\nnorth,100,15.50\nsouth,250,12.00\neast,80,20.00\nwest,150,18.00\n"
492 .into(),
493 )];
494 c.prompt = "Calculate total units and total revenue from sales.csv and write summary.json with keys total_units and total_revenue.".into();
495 c.script = vec![
496 ScriptedTurn::tool(
497 "bash",
498 serde_json::json!({"command": "awk -F, 'NR>1 {units+=$2; rev+=($2*$3)} END {print units, rev}' sales.csv"}),
499 ),
500 ScriptedTurn::tool(
501 "write",
502 serde_json::json!({
503 "path": "summary.json",
504 "content": "{\"total_units\": 580, \"total_revenue\": 8850.00}\n"
505 }),
506 ),
507 ScriptedTurn::Text("calculated".into()),
508 ];
509 c.verify = "grep -q '\"total_units\": 580' summary.json && grep -q '\"total_revenue\": 8850.00' summary.json".into();
510 c
511}
512
513/// Citation integrity: synthesize findings with verifiable numeric citations [1], [2].
514pub fn general_citation_integrity() -> EvalCase {
515 let mut c = base(
516 "general-citation-integrity",
517 "synthesize research with strict numeric citations and bibliography",
518 );
519 c.files = vec![
520 (
521 "sources/source1.txt".into(),
522 "Title: Global Solar Capacity 2025\nFinding: Installed photovoltaic capacity reached 2.1 terawatts globally in 2024.\n".into(),
523 ),
524 (
525 "sources/source2.txt".into(),
526 "Title: Grid Battery Storage Index\nFinding: Utility-scale battery storage grew by 125% year-over-year in North America.\n".into(),
527 ),
528 ];
529 c.prompt = "Synthesize findings from sources/ into synthesis.md. Every claim must have a [1] or [2] citation and a References section at the end.".into();
530 c.script = vec![
531 ScriptedTurn::tool_calls(vec![
532 ("read", serde_json::json!({"path": "sources/source1.txt"})),
533 ("read", serde_json::json!({"path": "sources/source2.txt"})),
534 ]),
535 ScriptedTurn::tool(
536 "write",
537 serde_json::json!({
538 "path": "synthesis.md",
539 "content": "# Clean Energy Transition\n\nGlobal photovoltaic capacity reached 2.1 terawatts in 2024 [1]. Concurrently, utility-scale battery deployment surged 125% in North America [2].\n\n## References\n- [1] Global Solar Capacity 2025\n- [2] Grid Battery Storage Index\n"
540 }),
541 ),
542 ScriptedTurn::Text("synthesized".into()),
543 ];
544 c.verify = "grep -q '\\[1\\]' synthesis.md && grep -q '\\[2\\]' synthesis.md && grep -q '## References' synthesis.md".into();
545 c
546}
547
548/// Entity knowledge capture: extract structured entity graph records with attributes and relations.
549pub fn general_entity_knowledge_capture() -> EvalCase {
550 let mut c = base(
551 "general-entity-knowledge-capture",
552 "extract typed entities with attributes and relations into JSONL knowledge format",
553 );
554 c.files = vec![(
555 "interview.txt".into(),
556 "Interview with Dr. Aris Thorne, Lead Biologist at Solis Genomics. Solis Genomics was founded in 2021 by Dr. Thorne and operates in Boston.\n".into(),
557 )];
558 c.prompt = "Extract entities (Person, Organization) from interview.txt into entities.jsonl. Include id, name, entity_type, and relations.".into();
559 c.script = vec![
560 ScriptedTurn::tool("read", serde_json::json!({"path": "interview.txt"})),
561 ScriptedTurn::tool(
562 "write",
563 serde_json::json!({
564 "path": "entities.jsonl",
565 "content": "{\"id\":\"aris-thorne\",\"name\":\"Dr. Aris Thorne\",\"entity_type\":\"Person\",\"summary\":\"Lead Biologist at Solis Genomics\",\"attributes\":{\"role\":\"Lead Biologist\"},\"relations\":[{\"relation\":\"founded\",\"target_entity_id\":\"solis-genomics\"}]}\n{\"id\":\"solis-genomics\",\"name\":\"Solis Genomics\",\"entity_type\":\"Organization\",\"summary\":\"Genomics company founded in 2021 in Boston\",\"attributes\":{\"founded\":\"2021\",\"location\":\"Boston\"},\"relations\":[]}\n"
566 }),
567 ),
568 ScriptedTurn::Text("extracted".into()),
569 ];
570 c.verify = "grep -q '\"id\":\"aris-thorne\"' entities.jsonl && grep -q '\"target_entity_id\":\"solis-genomics\"' entities.jsonl".into();
571 c
572}
573
574/// Multi-agent collaborative workflow: cross-domain research synthesis, quantitative metrics, and final briefing.
575pub fn general_multi_agent_collaboration() -> EvalCase {
576 let mut c = base(
577 "general-multi-agent-collaboration",
578 "collaborative cross-domain workflow combining research citations, quantitative metrics, and synthesis",
579 );
580 c.files = vec![
581 (
582 "data/energy_sources.csv".into(),
583 "source,capex_per_kw,opex_per_kw_yr,efficiency_pct\nsolar_pv,1100,15,22\nwind_turbine,1600,35,45\nbattery_storage,800,20,88\n".into(),
584 ),
585 (
586 "docs/research_notes.md".into(),
587 "# Research Notes\n\n[1] Microgrid Resilience Study (2025): Hybrid solar and battery systems reduce downtime by 94%.\n[2] NREL Technical Review: Levelized cost of storage has fallen 40% since 2020.\n".into(),
588 ),
589 ];
590 c.prompt = "Synthesize data/energy_sources.csv and docs/research_notes.md into report.md. Include quantitative efficiency findings, cite sources [1] and [2], and conclude with strategic recommendations.".into();
591 c.script = vec![
592 ScriptedTurn::tool(
593 "read",
594 serde_json::json!({"path": "data/energy_sources.csv"}),
595 ),
596 ScriptedTurn::tool(
597 "read",
598 serde_json::json!({"path": "docs/research_notes.md"}),
599 ),
600 ScriptedTurn::tool(
601 "write",
602 serde_json::json!({
603 "path": "report.md",
604 "content": "# Renewable Energy Microgrid Assessment\n\n## Executive Summary\nModern microgrids combining solar PV and battery storage achieve significant resilience improvements while optimizing lifecycle costs.\n\n## Quantitative Benchmark\n| Source | CAPEX ($/kW) | OPEX ($/kW-yr) | Efficiency | \n| --- | --- | --- | --- |\n| Solar PV | 1100 | 15 | 22% |\n| Wind Turbine | 1600 | 35 | 45% |\n| Battery Storage | 800 | 20 | 88% |\n\n## Research Evidence\nAs demonstrated by recent field studies, hybrid solar and battery installations reduce downtime by up to 94% [1]. Furthermore, storage cost declines of 40% enable rapid capital recovery [2].\n\n## Strategic Recommendations\n1. Deploy hybrid solar PV paired with high-efficiency (88%) battery storage.\n2. Prioritize modular storage expansion based on levelized cost dynamics.\n\n## References\n- [1] Microgrid Resilience Study (2025)\n- [2] NREL Technical Review\n"
605 }),
606 ),
607 ScriptedTurn::Text("report produced".into()),
608 ];
609 c.verify = "grep -q '88%' report.md && grep -q '\\[1\\]' report.md && grep -q '\\[2\\]' report.md && grep -q '## References' report.md".into();
610 c
611}
612
613/// Tasks for LIVE model runs: no scripted trajectory, verification only.
614/// Deliberately small and environment-independent so any frontier model
615/// can attempt them and differences reflect harness+model, not tooling.
616pub fn live_suite() -> Vec<EvalCase> {
617 vec![live_create_file(), live_sort_lines(), live_json_edit()]
618}
619
620fn live_create_file() -> EvalCase {
621 let mut c = base(
622 "live-create-file",
623 "create a file with exact content from a natural-language instruction",
624 );
625 c.prompt = "Create a file named greeting.txt containing exactly one line: hello world".into();
626 c.verify = "[ \"$(tr -d '\n\r' < greeting.txt)\" = 'hello world' ]".into();
627 c
628}
629
630fn live_sort_lines() -> EvalCase {
631 let mut c = base(
632 "live-sort-lines",
633 "sort the lines of a file into a new file",
634 );
635 c.files = vec![(
636 "unsorted.txt".into(),
637 "delta\nalpha\ncharlie\nbravo\n".into(),
638 )];
639 c.prompt = "Sort the lines of unsorted.txt alphabetically and write them to sorted.txt.".into();
640 c.verify =
641 "[ \"$(cat sorted.txt)\" = \"$(printf 'alpha\\nbravo\\ncharlie\\ndelta\\n')\" ]".into();
642 c
643}
644
645fn live_json_edit() -> EvalCase {
646 let mut c = base("live-json-edit", "make a precise edit inside a JSON config");
647 c.files = vec![(
648 "config.json".into(),
649 "{\n \"name\": \"svc\",\n \"port\": 3000,\n \"debug\": true\n}\n".into(),
650 )];
651 c.prompt =
652 "In config.json, change the value of \"port\" to 8080. Keep everything else identical."
653 .into();
654 c.verify = "python3 -c \"import json,sys;d=json.load(open('config.json'));sys.exit(0 if d['port']==8080 and d['name']=='svc' and d['debug']==True else 1)\"".into();
655 c
656}
657

1//! Built-in eval cases. Each pairs a scripted trajectory with a real
2//! verification command, exercising one harness capability end-to-end.
3
4use crate::runner::{EvalCase, ScriptedTurn};
5
6fn base(id: &str, description: &str) -> EvalCase {
7 EvalCase {
8 id: id.into(),
9 description: description.into(),
10 files: Vec::new(),
11 prompt: String::new(),
12 script: Vec::new(),
13 outcome: None,
14 verify: "true".into(),
15 }
16}
17
18/// The loop must execute a write tool call and produce the file.
19pub fn write_file() -> EvalCase {
20 let mut c = base(
21 "write-file",
22 "agent writes a file with exact content via the write tool",
23 );
24 c.prompt = "create result.txt containing hello-eval".into();
25 c.script = vec![
26 ScriptedTurn::tool(
27 "write",
28 serde_json::json!({"path": "result.txt", "content": "hello-eval"}),
29 ),
30 ScriptedTurn::Text("done".into()),
31 ];
32 c.verify = "grep -q hello-eval result.txt".into();
33 c
34}
35
36/// Setup file + atomic edit tool + verification of the edited content.
37pub fn edit_file() -> EvalCase {
38 let mut c = base(
39 "edit-file",
40 "agent applies an exact replacement to an existing file",
41 );
42 c.files = vec![("src/app.rs".into(), "fn main() {\n todo!()\n}\n".into())];
43 c.prompt = "replace todo!() with a println in src/app.rs".into();
44 c.script = vec![
45 ScriptedTurn::tool(
46 "edit",
47 serde_json::json!({
48 "path": "src/app.rs",
49 "edits": [{"old_text": "todo!()", "new_text": "println!(\"hi\");"}]
50 }),
51 ),
52 ScriptedTurn::Text("edited".into()),
53 ];
54 c.verify = "grep -q 'println!(\"hi\");' src/app.rs && ! grep -q 'todo!' src/app.rs".into();
55 c
56}
57
58/// Multi-step bash usage: inspect, compute, persist.
59pub fn bash_pipeline() -> EvalCase {
60 let mut c = base(
61 "bash-pipeline",
62 "agent chains two bash calls and writes an artifact",
63 );
64 c.files = vec![("data.txt".into(), "alpha\nbeta\ngamma\n".into())];
65 c.prompt = "count the lines in data.txt into count.txt".into();
66 c.script = vec![
67 ScriptedTurn::tool("bash", serde_json::json!({"command": "wc -l < data.txt"})),
68 ScriptedTurn::tool("bash", serde_json::json!({"command": "echo 3 > count.txt"})),
69 ScriptedTurn::Text("counted".into()),
70 ];
71 c.verify = "[ \"$(cat count.txt)\" = \"3\" ]".into();
72 c
73}
74
75/// Parallel tool execution: both writes land, order preserved in ledger.
76pub fn parallel_writes() -> EvalCase {
77 let mut c = base("parallel-writes", "batch of two write calls executes fully");
78 c.prompt = "write both part files".into();
79 c.script = vec![
80 ScriptedTurn::tool_calls(vec![
81 (
82 "write",
83 serde_json::json!({"path": "part-a.txt", "content": "A"}),
84 ),
85 (
86 "write",
87 serde_json::json!({"path": "part-b.txt", "content": "B"}),
88 ),
89 ]),
90 ScriptedTurn::Text("both written".into()),
91 ];
92 c.verify = "grep -q A part-a.txt && grep -q B part-b.txt".into();
93 c
94}
95
96/// Permission gate: read-only mode denies a write; model adapts and finishes.
97pub fn permission_denial_adapts() -> EvalCase {
98 let mut c = base(
99 "permission-denial-adapts",
100 "denied write becomes an error result; run still completes",
101 );
102 // This case is executed by run_case_with overloads in tests; here we
103 // keep the standard full-access path but verify the denied-tool ledger:
104 c.prompt = "try to write then report".into();
105 c.script = vec![
106 ScriptedTurn::tool(
107 "write",
108 serde_json::json!({"path": "ok.txt", "content": "fine"}),
109 ),
110 ScriptedTurn::Text("reported".into()),
111 ];
112 c.verify = "test -f ok.txt".into();
113 c
114}
115
116pub fn builtin_suite() -> Vec<EvalCase> {
117 vec![
118 write_file(),
119 edit_file(),
120 bash_pipeline(),
121 parallel_writes(),
122 permission_denial_adapts(),
123 ]
124}
125
126/// Non-coding scenarios: research, data analysis, writing, document
127/// conversion, and inventory work through the same six-tool kernel. Each
128/// proves the harness is a general agent, not a code-only one.
129pub fn general_suite() -> Vec<EvalCase> {
130 vec![
131 general_research_synthesis(),
132 general_csv_analysis(),
133 general_writing_draft(),
134 general_doc_conversion(),
135 general_inventory_index(),
136 general_error_adapts_noncode(),
137 general_schedule(),
138 general_decision_matrix(),
139 general_tabular_oracle(),
140 general_citation_integrity(),
141 general_entity_knowledge_capture(),
142 general_multi_agent_collaboration(),
143 ]
144}
145
146/// A small held-out-style corpus kept separate from the broad smoke suite.
147/// These cases deliberately vary language, context length, and deliverable
148/// shape so optimization work cannot overfit the named general cases.
149pub fn held_out_suite() -> Vec<EvalCase> {
150 vec![
151 held_out_multilingual_note(),
152 held_out_mixed_deliverables(),
153 held_out_recovery_after_bad_artifact(),
154 ]
155}
156
157fn held_out_multilingual_note() -> EvalCase {
158 let mut c = base(
159 "held-out-multilingual-note",
160 "summarize a Spanish note into an English action brief",
161 );
162 c.files = vec![(
163 "nota.txt".into(),
164 "La reunión es el martes. Ana enviará el presupuesto.\n".into(),
165 )];
166 c.prompt =
167 "Read nota.txt and write brief.md in English with the meeting date and owner.".into();
168 c.script = vec![
169 ScriptedTurn::tool("read", serde_json::json!({"path": "nota.txt"})),
170 ScriptedTurn::tool(
171 "write",
172 serde_json::json!({"path": "brief.md", "content": "# Action brief\n\n- Meeting: Tuesday\n- Owner: Ana (budget)\n"}),
173 ),
174 ScriptedTurn::Text("briefed".into()),
175 ];
176 let mut outcome = vak_intent::OutcomeSpec::from_reading(
177 &c.prompt,
178 &vak_intent::Reading::general(),
179 vak_intent::RESOLVER_VERSION,
180 );
181 outcome.requirements.push(vak_intent::OutcomeRequirement {
182 id: "brief-deliverable".into(),
183 kind: vak_intent::RequirementKind::Deliverable,
184 description: "English action brief is written to brief.md".into(),
185 origin: vak_intent::RequirementOrigin::Explicit,
186 importance: vak_intent::RequirementImportance::Must,
187 target: Some("brief.md".into()),
188 });
189 outcome.requirements.push(vak_intent::OutcomeRequirement {
190 id: "brief-evidence".into(),
191 kind: vak_intent::RequirementKind::Evidence,
192 description: "The brief cites the source note".into(),
193 origin: vak_intent::RequirementOrigin::Explicit,
194 importance: vak_intent::RequirementImportance::Must,
195 target: Some("nota.txt".into()),
196 });
197 c.outcome = Some(outcome);
198 c.verify = "grep -q Tuesday brief.md && grep -q Ana brief.md".into();
199 c
200}
201
202fn held_out_mixed_deliverables() -> EvalCase {
203 let mut c = base(
204 "held-out-mixed-deliverables",
205 "produce both a concise answer and a saved checklist",
206 );
207 c.prompt = "Give a one-line answer and save checklist.md with three launch checks.".into();
208 c.script = vec![
209 ScriptedTurn::tool(
210 "write",
211 serde_json::json!({"path": "checklist.md", "content": "# Launch checklist\n\n- Back up data\n- Verify access\n- Announce release\n"}),
212 ),
213 ScriptedTurn::Text("Answer: ready after the checks are complete.".into()),
214 ];
215 c.verify = "grep -q 'Back up data' checklist.md && grep -q 'Verify access' checklist.md && grep -q 'Announce release' checklist.md".into();
216 c
217}
218
219fn held_out_recovery_after_bad_artifact() -> EvalCase {
220 let mut c = base(
221 "held-out-recovery-after-bad-artifact",
222 "repair a malformed output before reporting completion",
223 );
224 c.prompt = "Write checklist.md with exactly the line READY, then verify it.".into();
225 c.script = vec![
226 ScriptedTurn::tool(
227 "write",
228 serde_json::json!({"path": "checklist.md", "content": "NOT READY\n"}),
229 ),
230 ScriptedTurn::tool(
231 "edit",
232 serde_json::json!({
233 "path": "checklist.md",
234 "edits": [{"old_text": "NOT READY", "new_text": "READY"}]
235 }),
236 ),
237 ScriptedTurn::Text("repaired and verified".into()),
238 ];
239 c.verify = "test \"$(cat checklist.md)\" = READY".into();
240 c
241}
242
243pub fn general_schedule() -> EvalCase {
244 let mut c = base(
245 "general-schedule",
246 "turn availability notes into a conflict-free schedule",
247 );
248 c.files = vec![(
249 "availability.txt".into(),
250 "Maya: 09:00-11:00\nDevon: 10:00-12:00\n".into(),
251 )];
252 c.prompt = "Read availability.txt and write schedule.md with a shared meeting time.".into();
253 c.script = vec![
254 ScriptedTurn::tool("read", serde_json::json!({"path": "availability.txt"})),
255 ScriptedTurn::tool(
256 "write",
257 serde_json::json!({"path": "schedule.md", "content": "# Meeting\n\nShared time: 10:00-11:00\n"}),
258 ),
259 ScriptedTurn::Text("scheduled".into()),
260 ];
261 c.verify = "grep -q '10:00-11:00' schedule.md".into();
262 c
263}
264
265pub fn general_decision_matrix() -> EvalCase {
266 let mut c = base(
267 "general-decision-matrix",
268 "compare options against explicit criteria",
269 );
270 c.prompt =
271 "Write decision.md comparing Option A and Option B, naming cost and reliability criteria."
272 .into();
273 c.script = vec![
274 ScriptedTurn::tool(
275 "write",
276 serde_json::json!({"path": "decision.md", "content": "# Decision\n\n| Option | Cost | Reliability |\n|---|---|---|\n| Option A | low | medium |\n| Option B | medium | high |\n"}),
277 ),
278 ScriptedTurn::Text("compared".into()),
279 ];
280 c.verify = "grep -q 'Option A' decision.md && grep -q 'Reliability' decision.md && grep -q 'Cost' decision.md".into();
281 c
282}
283
284/// Multi-source synthesis: read three notes, merge key facts into a summary.
285pub fn general_research_synthesis() -> EvalCase {
286 let mut c = base(
287 "general-research-synthesis",
288 "read three research notes and synthesize a summary file",
289 );
290 c.files = vec![
291 (
292 "notes/climate.txt".into(),
293 "Global mean temperature rose 1.2C since pre-industrial times.\n".into(),
294 ),
295 (
296 "notes/energy.txt".into(),
297 "Solar is now the cheapest electricity source in most markets.\n".into(),
298 ),
299 (
300 "notes/policy.txt".into(),
301 "Forty countries pledged carbon neutrality by 2050.\n".into(),
302 ),
303 ];
304 c.prompt = "Read all files under notes/ and write summary.md covering each finding.".into();
305 c.script = vec![
306 ScriptedTurn::tool_calls(vec![
307 ("read", serde_json::json!({"path": "notes/climate.txt"})),
308 ("read", serde_json::json!({"path": "notes/energy.txt"})),
309 ("read", serde_json::json!({"path": "notes/policy.txt"})),
310 ]),
311 ScriptedTurn::tool(
312 "write",
313 serde_json::json!({
314 "path": "summary.md",
315 "content": "# Findings\n\n- Warming reached 1.2C above pre-industrial levels.\n- Solar is now the cheapest electricity in most markets.\n- Forty countries pledged carbon neutrality by 2050.\n"
316 }),
317 ),
318 ScriptedTurn::Text("synthesized".into()),
319 ];
320 c.verify =
321 "grep -q '1.2C' summary.md && grep -q 'cheapest' summary.md && grep -q '2050' summary.md"
322 .into();
323 c
324}
325
326/// Data analysis on a CSV via bash arithmetic; no code files involved.
327pub fn general_csv_analysis() -> EvalCase {
328 let mut c = base(
329 "general-csv-analysis",
330 "analyze a csv of expenses with bash and write a report",
331 );
332 c.files = vec![(
333 "expenses.csv".into(),
334 "item,amount\nrent,1200\ngroceries,340\ntransport,85\n".into(),
335 )];
336 c.prompt =
337 "Compute the total of the amount column in expenses.csv and write report.md stating it."
338 .into();
339 c.script = vec![
340 ScriptedTurn::tool(
341 "bash",
342 serde_json::json!({"command": "awk -F, 'NR>1 {s+=$2} END {print s}' expenses.csv"}),
343 ),
344 ScriptedTurn::tool(
345 "write",
346 serde_json::json!({
347 "path": "report.md",
348 "content": "# Expense Report\n\nTotal monthly expenses: 1625\n"
349 }),
350 ),
351 ScriptedTurn::Text("analyzed".into()),
352 ];
353 c.verify = "grep -q 1625 report.md && [ \"$(awk -F, 'NR>1 {s+=$2} END {print s}' expenses.csv)\" = \"1625\" ]".into();
354 c
355}
356
357/// Pure writing: structure and length constraints verified mechanically.
358pub fn general_writing_draft() -> EvalCase {
359 let mut c = base(
360 "general-writing-draft",
361 "draft a structured essay meeting title and section requirements",
362 );
363 c.prompt = "Write essay.md: a title line starting with '# ', then sections '## Intro', '## Body', '## Conclusion' with at least one sentence each (>= 60 words total).".into();
364 c.script = vec![
365 ScriptedTurn::tool(
366 "write",
367 serde_json::json!({
368 "path": "essay.md",
369 "content": "# Urban Rivers\n\n## Intro\nCities grew around rivers because water meant trade, food, and power for early settlements.\n\n## Body\nOver the twentieth century many urban rivers were paved over or hidden beneath concrete, severing residents from their own geography and worsening floods downstream.\n\n## Conclusion\nDaylighting forgotten waterways restores habitat, cools streets, and gives neighborhoods a shared civic anchor worth protecting.\n"
370 }),
371 ),
372 ScriptedTurn::Text("drafted".into()),
373 ];
374 c.verify =
375 "head -n1 essay.md | grep -q '^# ' && grep -q '## Intro' essay.md && grep -q '## Body' essay.md && grep -q '## Conclusion' essay.md && [ \"$(wc -w < essay.md)\" -ge 60 ]"
376 .into();
377 c
378}
379
380/// Document conversion: free-form notes into machine-readable JSON.
381pub fn general_doc_conversion() -> EvalCase {
382 let mut c = base(
383 "general-doc-conversion",
384 "convert plain-text meeting notes into structured json",
385 );
386 c.files = vec![(
387 "meeting-notes.txt".into(),
388 "Team sync March 4\nAgenda:\n- review Q1 roadmap (Maya)\n- hiring update (Devon)\n- budget review (Priya)\n".into(),
389 )]
390 ;
391 c.prompt = "Convert meeting-notes.txt into agenda.json: an object with keys date (string), items (array of objects topic and owner).".into();
392 c.script = vec![
393 ScriptedTurn::tool("read", serde_json::json!({"path": "meeting-notes.txt"})),
394 ScriptedTurn::tool(
395 "write",
396 serde_json::json!({
397 "path": "agenda.json",
398 "content": "{\"date\": \"March 4\", \"items\": [{\"topic\": \"review Q1 roadmap\", \"owner\": \"Maya\"}, {\"topic\": \"hiring update\", \"owner\": \"Devon\"}, {\"topic\": \"budget review\", \"owner\": \"Priya\"}]}"
399 }),
400 ),
401 ScriptedTurn::Text("converted".into()),
402 ];
403 c.verify = "python3 -c \"import json;d=json.load(open('agenda.json'));assert d['date']=='March 4';assert len(d['items'])==3;assert d['items'][0]=={'topic':'review Q1 roadmap','owner':'Maya'}\"".into();
404 c
405}
406
407/// Cross-file discovery with glob+grep, then an index artifact.
408pub fn general_inventory_index() -> EvalCase {
409 let mut c = base(
410 "general-inventory-index",
411 "discover warranty records across files and build an index",
412 );
413 c.files = vec![
414 (
415 "records/laptop.txt".into(),
416 "MacBook Pro, purchased 2024-01-10, warranty until 2027-01-10.\n".into(),
417 ),
418 (
419 "records/monitor.txt".into(),
420 "Studio Display, purchased 2023-06-02, warranty until 2026-06-02.\n".into(),
421 ),
422 (
423 "records/receipts-old.txt".into(),
424 "Miscellaneous receipts from 2019, all warranties expired.\n".into(),
425 ),
426 ];
427 c.prompt =
428 "Search records/ for lines mentioning warranty and write index.md listing every item with its warranty end date."
429 .into();
430 c.script = vec![
431 ScriptedTurn::tool_calls(vec![
432 ("glob", serde_json::json!({"pattern": "records/*.txt"})),
433 (
434 "grep",
435 serde_json::json!({"pattern": "warranty until", "path": "records"}),
436 ),
437 ]),
438 ScriptedTurn::tool(
439 "write",
440 serde_json::json!({
441 "path": "index.md",
442 "content": "# Warranty Index\n\n- MacBook Pro — until 2027-01-10\n- Studio Display — until 2026-06-02\n"
443 }),
444 ),
445 ScriptedTurn::Text("indexed".into()),
446 ];
447 c.verify =
448 "grep -q '2027-01-10' index.md && grep -q '2026-06-02' index.md && ! grep -qi '2019' index.md"
449 .into();
450 c
451}
452
453/// Error-driven recovery outside code: first command misses, agent corrects.
454pub fn general_error_adapts_noncode() -> EvalCase {
455 let mut c = base(
456 "general-error-adapts-noncode",
457 "failed lookup is read from the error and corrected without retrying blindly",
458 );
459 c.files = vec![(
460 "archive/team-2025.txt".into(),
461 "Roster: Maya (design), Devon (ops), Priya (finance).\n".into(),
462 )];
463 c.prompt =
464 "Find this year's team roster file and copy the roster line into roster.md. Verify by reading it back."
465 .into();
466 c.script = vec![
467 // Deliberately probes the wrong path first.
468 ScriptedTurn::tool(
469 "bash",
470 serde_json::json!({"command": "cat archive/team-2026.txt"}),
471 ),
472 ScriptedTurn::tool(
473 "bash",
474 serde_json::json!({"command": "echo 'Roster: Maya (design), Devon (ops), Priya (finance).' > roster.md"}),
475 ),
476 ScriptedTurn::Text("recovered".into()),
477 ];
478 c.verify = "grep -q 'Roster: Maya' roster.md".into();
479 c
480}
481
482/// Tabular data oracle calculation: compute structured row sums and statistics
483/// and verify against exact expected tabular oracle calculations.
484pub fn general_tabular_oracle() -> EvalCase {
485 let mut c = base(
486 "general-tabular-oracle",
487 "calculate tabular sums, averages, and write verified tabular summary",
488 );
489 c.files = vec![(
490 "sales.csv".into(),
491 "region,units,price\nnorth,100,15.50\nsouth,250,12.00\neast,80,20.00\nwest,150,18.00\n"
492 .into(),
493 )];
494 c.prompt = "Calculate total units and total revenue from sales.csv and write summary.json with keys total_units and total_revenue.".into();
495 c.script = vec![
496 ScriptedTurn::tool(
497 "bash",
498 serde_json::json!({"command": "awk -F, 'NR>1 {units+=$2; rev+=($2*$3)} END {print units, rev}' sales.csv"}),
499 ),
500 ScriptedTurn::tool(
501 "write",
502 serde_json::json!({
503 "path": "summary.json",
504 "content": "{\"total_units\": 580, \"total_revenue\": 8850.00}\n"
505 }),
506 ),
507 ScriptedTurn::Text("calculated".into()),
508 ];
509 c.verify = "grep -q '\"total_units\": 580' summary.json && grep -q '\"total_revenue\": 8850.00' summary.json".into();
510 c
511}
512
513/// Citation integrity: synthesize findings with verifiable numeric citations [1], [2].
514pub fn general_citation_integrity() -> EvalCase {
515 let mut c = base(
516 "general-citation-integrity",
517 "synthesize research with strict numeric citations and bibliography",
518 );
519 c.files = vec![
520 (
521 "sources/source1.txt".into(),
522 "Title: Global Solar Capacity 2025\nFinding: Installed photovoltaic capacity reached 2.1 terawatts globally in 2024.\n".into(),
523 ),
524 (
525 "sources/source2.txt".into(),
526 "Title: Grid Battery Storage Index\nFinding: Utility-scale battery storage grew by 125% year-over-year in North America.\n".into(),
527 ),
528 ];
529 c.prompt = "Synthesize findings from sources/ into synthesis.md. Every claim must have a [1] or [2] citation and a References section at the end.".into();
530 c.script = vec![
531 ScriptedTurn::tool_calls(vec![
532 ("read", serde_json::json!({"path": "sources/source1.txt"})),
533 ("read", serde_json::json!({"path": "sources/source2.txt"})),
534 ]),
535 ScriptedTurn::tool(
536 "write",
537 serde_json::json!({
538 "path": "synthesis.md",
539 "content": "# Clean Energy Transition\n\nGlobal photovoltaic capacity reached 2.1 terawatts in 2024 [1]. Concurrently, utility-scale battery deployment surged 125% in North America [2].\n\n## References\n- [1] Global Solar Capacity 2025\n- [2] Grid Battery Storage Index\n"
540 }),
541 ),
542 ScriptedTurn::Text("synthesized".into()),
543 ];
544 c.verify = "grep -q '\\[1\\]' synthesis.md && grep -q '\\[2\\]' synthesis.md && grep -q '## References' synthesis.md".into();
545 c
546}
547
548/// Entity knowledge capture: extract structured entity graph records with attributes and relations.
549pub fn general_entity_knowledge_capture() -> EvalCase {
550 let mut c = base(
551 "general-entity-knowledge-capture",
552 "extract typed entities with attributes and relations into JSONL knowledge format",
553 );
554 c.files = vec![(
555 "interview.txt".into(),
556 "Interview with Dr. Aris Thorne, Lead Biologist at Solis Genomics. Solis Genomics was founded in 2021 by Dr. Thorne and operates in Boston.\n".into(),
557 )];
558 c.prompt = "Extract entities (Person, Organization) from interview.txt into entities.jsonl. Include id, name, entity_type, and relations.".into();
559 c.script = vec![
560 ScriptedTurn::tool("read", serde_json::json!({"path": "interview.txt"})),
561 ScriptedTurn::tool(
562 "write",
563 serde_json::json!({
564 "path": "entities.jsonl",
565 "content": "{\"id\":\"aris-thorne\",\"name\":\"Dr. Aris Thorne\",\"entity_type\":\"Person\",\"summary\":\"Lead Biologist at Solis Genomics\",\"attributes\":{\"role\":\"Lead Biologist\"},\"relations\":[{\"relation\":\"founded\",\"target_entity_id\":\"solis-genomics\"}]}\n{\"id\":\"solis-genomics\",\"name\":\"Solis Genomics\",\"entity_type\":\"Organization\",\"summary\":\"Genomics company founded in 2021 in Boston\",\"attributes\":{\"founded\":\"2021\",\"location\":\"Boston\"},\"relations\":[]}\n"
566 }),
567 ),
568 ScriptedTurn::Text("extracted".into()),
569 ];
570 c.verify = "grep -q '\"id\":\"aris-thorne\"' entities.jsonl && grep -q '\"target_entity_id\":\"solis-genomics\"' entities.jsonl".into();
571 c
572}
573
574/// Multi-agent collaborative workflow: cross-domain research synthesis, quantitative metrics, and final briefing.
575pub fn general_multi_agent_collaboration() -> EvalCase {
576 let mut c = base(
577 "general-multi-agent-collaboration",
578 "collaborative cross-domain workflow combining research citations, quantitative metrics, and synthesis",
579 );
580 c.files = vec![
581 (
582 "data/energy_sources.csv".into(),
583 "source,capex_per_kw,opex_per_kw_yr,efficiency_pct\nsolar_pv,1100,15,22\nwind_turbine,1600,35,45\nbattery_storage,800,20,88\n".into(),
584 ),
585 (
586 "docs/research_notes.md".into(),
587 "# Research Notes\n\n[1] Microgrid Resilience Study (2025): Hybrid solar and battery systems reduce downtime by 94%.\n[2] NREL Technical Review: Levelized cost of storage has fallen 40% since 2020.\n".into(),
588 ),
589 ];
590 c.prompt = "Synthesize data/energy_sources.csv and docs/research_notes.md into report.md. Include quantitative efficiency findings, cite sources [1] and [2], and conclude with strategic recommendations.".into();
591 c.script = vec![
592 ScriptedTurn::tool(
593 "read",
594 serde_json::json!({"path": "data/energy_sources.csv"}),
595 ),
596 ScriptedTurn::tool(
597 "read",
598 serde_json::json!({"path": "docs/research_notes.md"}),
599 ),
600 ScriptedTurn::tool(
601 "write",
602 serde_json::json!({
603 "path": "report.md",
604 "content": "# Renewable Energy Microgrid Assessment\n\n## Executive Summary\nModern microgrids combining solar PV and battery storage achieve significant resilience improvements while optimizing lifecycle costs.\n\n## Quantitative Benchmark\n| Source | CAPEX ($/kW) | OPEX ($/kW-yr) | Efficiency | \n| --- | --- | --- | --- |\n| Solar PV | 1100 | 15 | 22% |\n| Wind Turbine | 1600 | 35 | 45% |\n| Battery Storage | 800 | 20 | 88% |\n\n## Research Evidence\nAs demonstrated by recent field studies, hybrid solar and battery installations reduce downtime by up to 94% [1]. Furthermore, storage cost declines of 40% enable rapid capital recovery [2].\n\n## Strategic Recommendations\n1. Deploy hybrid solar PV paired with high-efficiency (88%) battery storage.\n2. Prioritize modular storage expansion based on levelized cost dynamics.\n\n## References\n- [1] Microgrid Resilience Study (2025)\n- [2] NREL Technical Review\n"
605 }),
606 ),
607 ScriptedTurn::Text("report produced".into()),
608 ];
609 c.verify = "grep -q '88%' report.md && grep -q '\\[1\\]' report.md && grep -q '\\[2\\]' report.md && grep -q '## References' report.md".into();
610 c
611}
612
613/// Tasks for LIVE model runs: no scripted trajectory, verification only.
614/// Deliberately small and environment-independent so any frontier model
615/// can attempt them and differences reflect harness+model, not tooling.
616pub fn live_suite() -> Vec<EvalCase> {
617 vec![live_create_file(), live_sort_lines(), live_json_edit()]
618}
619
620fn live_create_file() -> EvalCase {
621 let mut c = base(
622 "live-create-file",
623 "create a file with exact content from a natural-language instruction",
624 );
625 c.prompt = "Create a file named greeting.txt containing exactly one line: hello world".into();
626 c.verify = "[ \"$(tr -d '\n\r' < greeting.txt)\" = 'hello world' ]".into();
627 c
628}
629
630fn live_sort_lines() -> EvalCase {
631 let mut c = base(
632 "live-sort-lines",
633 "sort the lines of a file into a new file",
634 );
635 c.files = vec![(
636 "unsorted.txt".into(),
637 "delta\nalpha\ncharlie\nbravo\n".into(),
638 )];
639 c.prompt = "Sort the lines of unsorted.txt alphabetically and write them to sorted.txt.".into();
640 c.verify =
641 "[ \"$(cat sorted.txt)\" = \"$(printf 'alpha\\nbravo\\ncharlie\\ndelta\\n')\" ]".into();
642 c
643}
644
645fn live_json_edit() -> EvalCase {
646 let mut c = base("live-json-edit", "make a precise edit inside a JSON config");
647 c.files = vec![(
648 "config.json".into(),
649 "{\n \"name\": \"svc\",\n \"port\": 3000,\n \"debug\": true\n}\n".into(),
650 )];
651 c.prompt =
652 "In config.json, change the value of \"port\" to 8080. Keep everything else identical."
653 .into();
654 c.verify = "python3 -c \"import json,sys;d=json.load(open('config.json'));sys.exit(0 if d['port']==8080 and d['name']=='svc' and d['debug']==True else 1)\"".into();
655 c
656}
657

Cases

Indexing the workspace…

Cases