Vakyartha documentation

1use std::collections::VecDeque;
2use std::path::Path;
3use std::sync::{Arc, Mutex};
4use std::time::Instant;
5
6use async_trait::async_trait;
7use serde::Serialize;
8use tokio_util::sync::CancellationToken;
9
10use vak_agent::{Agent, AgentConfig, TurnOutcome};
11use vak_llm::stream;
12use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
13use vak_llm::{EventStream, LlmError, Provider};
14use vak_permission::{Mode, PermissionEngine};
15use vak_session::SessionLog;
16use vak_session::types::{FrozenContract, SessionHeader};
17use vak_tools::bash::BashTool;
18use vak_tools::{Tool, ToolContext};
19
20/// One scripted assistant turn: a plain text reply or a batch of tool calls
21/// (the loop executes them and requests the next turn).
22#[derive(Debug, Clone)]
23pub enum ScriptedTurn {
24 Text(String),
25 ToolCalls(Vec<(String, String, serde_json::Value)>),
26}
27
28impl ScriptedTurn {
29 pub fn tool(name: &str, args: serde_json::Value) -> Self {
30 ScriptedTurn::ToolCalls(vec![(format!("t-{}", uuid_like()), name.to_string(), args)])
31 }
32
33 pub fn tool_calls(calls: Vec<(&str, serde_json::Value)>) -> Self {
34 ScriptedTurn::ToolCalls(
35 calls
36 .into_iter()
37 .map(|(name, args)| (format!("t-{}", uuid_like()), name.to_string(), args))
38 .collect(),
39 )
40 }
41
42 fn to_message(&self) -> AssistantMessage {
43 let usage = Usage {
44 input_tokens: 10,
45 output_tokens: 5,
46 ..Default::default()
47 };
48 match self {
49 ScriptedTurn::Text(t) => AssistantMessage {
50 content: vec![ContentBlock::text(t.clone())],
51 stop_reason: StopReason::EndTurn,
52 usage,
53 model: "eval-model".into(),
54 response_id: None,
55 },
56 ScriptedTurn::ToolCalls(calls) => AssistantMessage {
57 content: calls
58 .iter()
59 .map(|(id, name, input)| ContentBlock::ToolUse {
60 id: id.clone(),
61 name: name.clone(),
62 input: input.clone(),
63 })
64 .collect(),
65 stop_reason: StopReason::ToolUse,
66 usage,
67 model: "eval-model".into(),
68 response_id: None,
69 },
70 }
71 }
72}
73
74fn uuid_like() -> String {
75 use std::sync::atomic::{AtomicU32, Ordering};
76 static C: AtomicU32 = AtomicU32::new(0);
77 format!("u{}", C.fetch_add(1, Ordering::Relaxed))
78}
79
80/// Serves the case's turns in order; records every request for audit.
81pub struct EvalProvider {
82 turns: Mutex<VecDeque<AssistantMessage>>,
83 pub requests: Arc<Mutex<Vec<ChatRequest>>>,
84}
85
86impl EvalProvider {
87 pub fn new(turns: Vec<ScriptedTurn>) -> Self {
88 EvalProvider {
89 turns: Mutex::new(turns.into_iter().map(|t| t.to_message()).collect()),
90 requests: Arc::new(Mutex::new(Vec::new())),
91 }
92 }
93}
94
95#[async_trait]
96impl Provider for EvalProvider {
97 fn name(&self) -> &str {
98 "eval-scripted"
99 }
100
101 async fn stream(
102 &self,
103 request: ChatRequest,
104 _cancel: CancellationToken,
105 ) -> Result<EventStream, LlmError> {
106 self.requests
107 .lock()
108 .unwrap_or_else(std::sync::PoisonError::into_inner)
109 .push(request);
110 let next = self
111 .turns
112 .lock()
113 .unwrap_or_else(std::sync::PoisonError::into_inner)
114 .pop_front();
115 let (mut sink, rx) = stream::channel(64);
116 match next {
117 Some(m) => {
118 sink.push(stream::StreamEvent::Start { partial: m.clone() });
119 sink.close_message(m).await;
120 }
121 None => {
122 sink.close_error(LlmError::Parse("eval script exhausted".into()))
123 .await;
124 }
125 }
126 Ok(rx)
127 }
128}
129
130#[derive(Debug, Clone)]
131pub struct EvalCase {
132 pub id: String,
133 pub description: String,
134 /// Files written into the sandbox workspace before the run.
135 pub files: Vec<(String, String)>,
136 pub prompt: String,
137 pub script: Vec<ScriptedTurn>,
138 /// Optional admitted contract for this case. When absent, the evaluator
139 /// uses the general-purpose baseline for compatibility with simple cases.
140 pub outcome: Option<vak_intent::OutcomeSpec>,
141 /// Bash command run in the workspace after the agent finishes; exit 0 = pass.
142 pub verify: String,
143}
144
145#[derive(Debug, Clone, Serialize)]
146pub struct EvalReport {
147 pub task_id: String,
148 pub passed: bool,
149 pub verify_exit: Option<i32>,
150 pub tokens_in: u64,
151 pub tokens_out: u64,
152 pub duration_ms: u128,
153 pub outcome_status: String,
154 pub completion_verdict: String,
155 pub human_review: String,
156 pub verification_evidence: String,
157 #[serde(skip_serializing_if = "Option::is_none")]
158 pub error: Option<String>,
159}
160
161/// Aggregate measurement for a corpus. This is deliberately computed from
162/// per-case production-loop reports so a green corpus cannot hide a failed
163/// case behind an average.
164#[derive(Debug, Clone, Serialize)]
165pub struct EvalSuiteReport {
166 pub cases: Vec<EvalReport>,
167 pub total: usize,
168 pub passed: usize,
169 pub pass_rate: f64,
170 pub mean_duration_ms: f64,
171 pub total_tokens_in: u64,
172 pub total_tokens_out: u64,
173 pub partial_or_unknown: usize,
174 pub human_review_recommended: usize,
175}
176
177#[derive(Debug, Clone, Serialize)]
178pub struct EvalComparisonReport {
179 pub task_id: String,
180 pub baseline: EvalReport,
181 pub outcome_directed: EvalReport,
182 pub latency_delta_ms: i128,
183 pub token_delta: i128,
184}
185
186#[derive(Debug, Clone, Serialize)]
187pub struct EvalComparisonSuiteReport {
188 pub baseline: EvalSuiteReport,
189 pub outcome_directed: EvalSuiteReport,
190 pub pass_rate_delta: f64,
191 pub mean_duration_delta_ms: f64,
192 pub token_delta: i128,
193}
194
195pub async fn compare_case(case: &EvalCase) -> EvalComparisonReport {
196 let mut baseline_case = case.clone();
197 baseline_case.outcome = None;
198 let baseline = run_case(&baseline_case).await;
199 let outcome_directed = run_case(case).await;
200 EvalComparisonReport {
201 task_id: case.id.clone(),
202 latency_delta_ms: outcome_directed.duration_ms as i128 - baseline.duration_ms as i128,
203 token_delta: (outcome_directed.tokens_in + outcome_directed.tokens_out) as i128
204 - (baseline.tokens_in + baseline.tokens_out) as i128,
205 baseline,
206 outcome_directed,
207 }
208}
209
210pub async fn compare_suite(cases: &[EvalCase]) -> EvalComparisonSuiteReport {
211 let mut baseline_cases = Vec::with_capacity(cases.len());
212 for case in cases {
213 let mut baseline = case.clone();
214 baseline.outcome = None;
215 baseline_cases.push(baseline);
216 }
217 let baseline = run_suite(&baseline_cases).await;
218 let outcome_directed = run_suite(cases).await;
219 EvalComparisonSuiteReport {
220 pass_rate_delta: outcome_directed.pass_rate - baseline.pass_rate,
221 mean_duration_delta_ms: outcome_directed.mean_duration_ms - baseline.mean_duration_ms,
222 token_delta: (outcome_directed.total_tokens_in + outcome_directed.total_tokens_out) as i128
223 - (baseline.total_tokens_in + baseline.total_tokens_out) as i128,
224 baseline,
225 outcome_directed,
226 }
227}
228
229impl EvalSuiteReport {
230 fn from_cases(cases: Vec<EvalReport>) -> Self {
231 let total = cases.len();
232 let passed = cases.iter().filter(|case| case.passed).count();
233 let mean_duration_ms = if total == 0 {
234 0.0
235 } else {
236 cases
237 .iter()
238 .map(|case| case.duration_ms as f64)
239 .sum::<f64>()
240 / total as f64
241 };
242 EvalSuiteReport {
243 total,
244 passed,
245 pass_rate: if total == 0 {
246 0.0
247 } else {
248 passed as f64 / total as f64
249 },
250 mean_duration_ms,
251 total_tokens_in: cases.iter().map(|case| case.tokens_in).sum(),
252 total_tokens_out: cases.iter().map(|case| case.tokens_out).sum(),
253 partial_or_unknown: cases
254 .iter()
255 .filter(|case| matches!(case.completion_verdict.as_str(), "partial" | "unknown"))
256 .count(),
257 human_review_recommended: cases
258 .iter()
259 .filter(|case| case.human_review == "recommended")
260 .count(),
261 cases,
262 }
263 }
264}
265
266/// Run a corpus and retain every case report for inspection and comparison.
267pub async fn run_suite(cases: &[EvalCase]) -> EvalSuiteReport {
268 let mut reports = Vec::with_capacity(cases.len());
269 for case in cases {
270 reports.push(run_case(case).await);
271 }
272 EvalSuiteReport::from_cases(reports)
273}
274
275/// Runs one case with the built-in scripted provider (deterministic).
276pub async fn run_case(case: &EvalCase) -> EvalReport {
277 let provider = Arc::new(EvalProvider::new(case.script.clone()));
278 run_case_with_provider(case, provider, "eval-model").await
279}
280
281pub async fn run_case_brokered(case: &EvalCase, worker_exe: std::path::PathBuf) -> EvalReport {
282 let provider = Arc::new(EvalProvider::new(case.script.clone()));
283 run_case_with_provider_brokered(case, provider, "eval-model", worker_exe).await
284}
285
286/// Runs one case against ANY provider — the live-model path. The `script`
287/// field is ignored; the model must genuinely solve the task.
288pub async fn run_case_with_provider(
289 case: &EvalCase,
290 provider: Arc<dyn Provider>,
291 model: &str,
292) -> EvalReport {
293 run_case_with_tools(case, provider, model, vak_tools::default_tools()).await
294}
295
296pub async fn run_case_with_provider_brokered(
297 case: &EvalCase,
298 provider: Arc<dyn Provider>,
299 model: &str,
300 worker_exe: std::path::PathBuf,
301) -> EvalReport {
302 run_case_with_tools(
303 case,
304 provider,
305 model,
306 vak_tools::brokered_default_tools(worker_exe),
307 )
308 .await
309}
310
311async fn run_case_with_tools(
312 case: &EvalCase,
313 provider: Arc<dyn Provider>,
314 model: &str,
315 tools: Vec<Arc<dyn Tool>>,
316) -> EvalReport {
317 let deterministic = provider.name() == "eval-scripted";
318 let start = Instant::now();
319 let dir = match tempfile::tempdir() {
320 Ok(d) => d,
321 Err(e) => {
322 return EvalReport {
323 task_id: case.id.clone(),
324 passed: false,
325 verify_exit: None,
326 tokens_in: 0,
327 tokens_out: 0,
328 duration_ms: start.elapsed().as_millis(),
329 outcome_status: "unknown".into(),
330 completion_verdict: "failed".into(),
331 human_review: "required_for_recovery".into(),
332 verification_evidence: "none".into(),
333 error: Some(format!("workspace setup failed: {e}")),
334 };
335 }
336 };
337 let cwd = dir.path().to_path_buf();
338 for (rel, content) in &case.files {
339 let p = cwd.join(rel);
340 if let Some(parent) = p.parent() {
341 let _ = std::fs::create_dir_all(parent);
342 }
343 if std::fs::write(&p, content).is_err() {
344 return fail(&case.id, "setup file write failed", &start, None);
345 }
346 }
347
348 let header = SessionHeader {
349 agent: Some(vak_session::types::AgentIdentity {
350 id: "vak".into(),
351 revision: 1,
352 name: "Vakyartha".into(),
353 character: "vak".into(),
354 personality: String::new(),
355 animation: "subtle".into(),
356 voice: "default".into(),
357 behaviour: String::new(),
358 responsibilities: String::new(),
359 instructions: String::new(),
360 }),
361 session_id: format!("eval-{}", case.id),
362 created_at: chrono::Utc::now(),
363 cwd: cwd.clone(),
364 parent_session_id: None,
365 contract_id: None,
366 work_item_id: None,
367 conversation: Some(vak_session::ConversationContext::local(
368 format!("eval-{}", case.id),
369 "eval",
370 )),
371 contract: FrozenContract {
372 app_version: env!("CARGO_PKG_VERSION").into(),
373 provider: "eval-scripted".into(),
374 model: model.to_string(),
375 route_ladder: Vec::new(),
376 route_objective: String::new(),
377 route_annotations: Vec::new(),
378 system_prompt: "eval".into(),
379 permission_mode: "full-access".into(),
380 capabilities: Vec::new(),
381 prompt_layers: Vec::new(),
382 },
383 };
384 let sessions_home = cwd.join(".vak-home");
385 let log = match SessionLog::create(
386 vak_session::SessionPath::new_session_file(&sessions_home, &cwd, &header.session_id),
387 header,
388 ) {
389 Ok(l) => l,
390 Err(e) => {
391 return fail(
392 &case.id,
393 &format!("session setup failed: {e}"),
394 &start,
395 None,
396 );
397 }
398 };
399
400 let prepared = vak_core::PreparedTurn::from_parts("eval", tools, Vec::new());
401 let mut cfg = AgentConfig::new(prepared.system_prompt);
402 cfg.model = model.to_string();
403 cfg.tools = prepared.tools;
404 cfg.outcome = case.outcome.clone();
405 cfg.max_turns = 12;
406 if deterministic {
407 cfg.max_retries = 0;
408 cfg.run_retry_attempts = 0;
409 }
410 cfg.permission = Some(Arc::new(PermissionEngine::default()));
411 cfg.mode = Mode::FullAccess;
412 cfg.approver = Some(Arc::new(vak_agent::AutoApprove));
413 let mut agent = Agent::new(provider, log, cfg);
414
415 let (events, mut events_rx) = tokio::sync::mpsc::channel(256);
416 let drain = tokio::spawn(async move { while events_rx.recv().await.is_some() {} });
417 let outcome = agent
418 .run(
419 &case.prompt,
420 &Default::default(),
421 CancellationToken::new(),
422 events,
423 )
424 .await;
425 let _ = drain.await;
426
427 let (tokens_in, tokens_out, loop_error) = {
428 let session = agent.session.lock().await;
429 let u = session.total_usage();
430 let err = match outcome {
431 TurnOutcome::Completed { .. } => None,
432 TurnOutcome::Aborted { .. } => Some("aborted".to_string()),
433 TurnOutcome::Failed { ref error } => Some(error.to_string()),
434 TurnOutcome::MaxTurnsReached => Some("max turns reached".to_string()),
435 };
436 (u.input_tokens, u.output_tokens, err)
437 };
438
439 // Verification runs in the same workspace with the production bash tool.
440 let verify_ctx = ToolContext {
441 cwd: cwd.clone(),
442 cancel: CancellationToken::new(),
443 sandbox: None,
444 sandbox_sink: None,
445 agent_id: None,
446 new_documents: Vec::new(),
447 };
448 let verify_out = BashTool
449 .execute(&serde_json::json!({"command": case.verify}), &verify_ctx)
450 .await;
451
452 let report = EvalReport {
453 task_id: case.id.clone(),
454 passed: !verify_out.is_error,
455 verify_exit: Some(if verify_out.is_error { 1 } else { 0 }),
456 tokens_in,
457 tokens_out,
458 duration_ms: start.elapsed().as_millis(),
459 outcome_status: format!(
460 "{:?}",
461 vak_intent::evaluate_response(
462 match &outcome {
463 TurnOutcome::Completed { response } => Some(response.text_content()),
464 TurnOutcome::Aborted { partial } => {
465 partial.as_ref().map(|message| message.text_content())
466 }
467 TurnOutcome::Failed { .. } | TurnOutcome::MaxTurnsReached => None,
468 }
469 .as_deref(),
470 loop_error.is_some(),
471 matches!(outcome, TurnOutcome::Aborted { .. }),
472 )
473 )
474 .to_ascii_lowercase(),
475 completion_verdict: {
476 let response = match &outcome {
477 TurnOutcome::Completed { response } => Some(response.text_content()),
478 TurnOutcome::Aborted { partial } => {
479 partial.as_ref().map(|message| message.text_content())
480 }
481 TurnOutcome::Failed { .. } | TurnOutcome::MaxTurnsReached => None,
482 };
483 let spec = case.outcome.clone().unwrap_or_else(|| {
484 vak_intent::OutcomeSpec::from_reading(
485 &case.prompt,
486 &vak_intent::Reading::general(),
487 vak_intent::RESOLVER_VERSION,
488 )
489 });
490 let status = vak_intent::evaluate_response(
491 response.as_deref(),
492 loop_error.is_some(),
493 matches!(outcome, TurnOutcome::Aborted { .. }),
494 );
495 let evaluations = vak_intent::evaluate_requirements(&spec, response.as_deref());
496 let verdict = vak_intent::evaluate_completion(status, &evaluations, &spec);
497 if verify_out.is_error && matches!(verdict, vak_intent::CompletionVerdict::Complete) {
498 "partial".into()
499 } else {
500 format!("{verdict:?}").to_ascii_lowercase()
501 }
502 },
503 human_review: if verify_out.is_error
504 || case.outcome.as_ref().is_some_and(|outcome| {
505 outcome.requirements.iter().any(|requirement| {
506 matches!(requirement.kind, vak_intent::RequirementKind::Evidence)
507 })
508 }) {
509 "recommended".into()
510 } else {
511 "not_required".into()
512 },
513 verification_evidence: if verify_out.is_error {
514 "observed_failure".into()
515 } else {
516 "observed_success".into()
517 },
518 error: loop_error.or_else(|| {
519 if verify_out.is_error {
520 Some(format!("verify failed: {}", verify_out.content))
521 } else {
522 None
523 }
524 }),
525 };
526 keep_workspace(&cwd, &report);
527 report
528}
529
530fn fail(id: &str, msg: &str, start: &Instant, verify_exit: Option<i32>) -> EvalReport {
531 EvalReport {
532 task_id: id.to_string(),
533 passed: false,
534 verify_exit,
535 tokens_in: 0,
536 tokens_out: 0,
537 duration_ms: start.elapsed().as_millis(),
538 outcome_status: "unknown".into(),
539 completion_verdict: "failed".into(),
540 human_review: "required_for_recovery".into(),
541 verification_evidence: "none".into(),
542 error: Some(msg.to_string()),
543 }
544}
545
546/// Keep failing workspaces under target/ for post-mortem; drop passing ones.
547fn keep_workspace(_cwd: &Path, report: &EvalReport) {
548 let _ = report;
549}
550

1use std::collections::VecDeque;
2use std::path::Path;
3use std::sync::{Arc, Mutex};
4use std::time::Instant;
5
6use async_trait::async_trait;
7use serde::Serialize;
8use tokio_util::sync::CancellationToken;
9
10use vak_agent::{Agent, AgentConfig, TurnOutcome};
11use vak_llm::stream;
12use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
13use vak_llm::{EventStream, LlmError, Provider};
14use vak_permission::{Mode, PermissionEngine};
15use vak_session::SessionLog;
16use vak_session::types::{FrozenContract, SessionHeader};
17use vak_tools::bash::BashTool;
18use vak_tools::{Tool, ToolContext};
19
20/// One scripted assistant turn: a plain text reply or a batch of tool calls
21/// (the loop executes them and requests the next turn).
22#[derive(Debug, Clone)]
23pub enum ScriptedTurn {
24 Text(String),
25 ToolCalls(Vec<(String, String, serde_json::Value)>),
26}
27
28impl ScriptedTurn {
29 pub fn tool(name: &str, args: serde_json::Value) -> Self {
30 ScriptedTurn::ToolCalls(vec![(format!("t-{}", uuid_like()), name.to_string(), args)])
31 }
32
33 pub fn tool_calls(calls: Vec<(&str, serde_json::Value)>) -> Self {
34 ScriptedTurn::ToolCalls(
35 calls
36 .into_iter()
37 .map(|(name, args)| (format!("t-{}", uuid_like()), name.to_string(), args))
38 .collect(),
39 )
40 }
41
42 fn to_message(&self) -> AssistantMessage {
43 let usage = Usage {
44 input_tokens: 10,
45 output_tokens: 5,
46 ..Default::default()
47 };
48 match self {
49 ScriptedTurn::Text(t) => AssistantMessage {
50 content: vec![ContentBlock::text(t.clone())],
51 stop_reason: StopReason::EndTurn,
52 usage,
53 model: "eval-model".into(),
54 response_id: None,
55 },
56 ScriptedTurn::ToolCalls(calls) => AssistantMessage {
57 content: calls
58 .iter()
59 .map(|(id, name, input)| ContentBlock::ToolUse {
60 id: id.clone(),
61 name: name.clone(),
62 input: input.clone(),
63 })
64 .collect(),
65 stop_reason: StopReason::ToolUse,
66 usage,
67 model: "eval-model".into(),
68 response_id: None,
69 },
70 }
71 }
72}
73
74fn uuid_like() -> String {
75 use std::sync::atomic::{AtomicU32, Ordering};
76 static C: AtomicU32 = AtomicU32::new(0);
77 format!("u{}", C.fetch_add(1, Ordering::Relaxed))
78}
79
80/// Serves the case's turns in order; records every request for audit.
81pub struct EvalProvider {
82 turns: Mutex<VecDeque<AssistantMessage>>,
83 pub requests: Arc<Mutex<Vec<ChatRequest>>>,
84}
85
86impl EvalProvider {
87 pub fn new(turns: Vec<ScriptedTurn>) -> Self {
88 EvalProvider {
89 turns: Mutex::new(turns.into_iter().map(|t| t.to_message()).collect()),
90 requests: Arc::new(Mutex::new(Vec::new())),
91 }
92 }
93}
94
95#[async_trait]
96impl Provider for EvalProvider {
97 fn name(&self) -> &str {
98 "eval-scripted"
99 }
100
101 async fn stream(
102 &self,
103 request: ChatRequest,
104 _cancel: CancellationToken,
105 ) -> Result<EventStream, LlmError> {
106 self.requests
107 .lock()
108 .unwrap_or_else(std::sync::PoisonError::into_inner)
109 .push(request);
110 let next = self
111 .turns
112 .lock()
113 .unwrap_or_else(std::sync::PoisonError::into_inner)
114 .pop_front();
115 let (mut sink, rx) = stream::channel(64);
116 match next {
117 Some(m) => {
118 sink.push(stream::StreamEvent::Start { partial: m.clone() });
119 sink.close_message(m).await;
120 }
121 None => {
122 sink.close_error(LlmError::Parse("eval script exhausted".into()))
123 .await;
124 }
125 }
126 Ok(rx)
127 }
128}
129
130#[derive(Debug, Clone)]
131pub struct EvalCase {
132 pub id: String,
133 pub description: String,
134 /// Files written into the sandbox workspace before the run.
135 pub files: Vec<(String, String)>,
136 pub prompt: String,
137 pub script: Vec<ScriptedTurn>,
138 /// Optional admitted contract for this case. When absent, the evaluator
139 /// uses the general-purpose baseline for compatibility with simple cases.
140 pub outcome: Option<vak_intent::OutcomeSpec>,
141 /// Bash command run in the workspace after the agent finishes; exit 0 = pass.
142 pub verify: String,
143}
144
145#[derive(Debug, Clone, Serialize)]
146pub struct EvalReport {
147 pub task_id: String,
148 pub passed: bool,
149 pub verify_exit: Option<i32>,
150 pub tokens_in: u64,
151 pub tokens_out: u64,
152 pub duration_ms: u128,
153 pub outcome_status: String,
154 pub completion_verdict: String,
155 pub human_review: String,
156 pub verification_evidence: String,
157 #[serde(skip_serializing_if = "Option::is_none")]
158 pub error: Option<String>,
159}
160
161/// Aggregate measurement for a corpus. This is deliberately computed from
162/// per-case production-loop reports so a green corpus cannot hide a failed
163/// case behind an average.
164#[derive(Debug, Clone, Serialize)]
165pub struct EvalSuiteReport {
166 pub cases: Vec<EvalReport>,
167 pub total: usize,
168 pub passed: usize,
169 pub pass_rate: f64,
170 pub mean_duration_ms: f64,
171 pub total_tokens_in: u64,
172 pub total_tokens_out: u64,
173 pub partial_or_unknown: usize,
174 pub human_review_recommended: usize,
175}
176
177#[derive(Debug, Clone, Serialize)]
178pub struct EvalComparisonReport {
179 pub task_id: String,
180 pub baseline: EvalReport,
181 pub outcome_directed: EvalReport,
182 pub latency_delta_ms: i128,
183 pub token_delta: i128,
184}
185
186#[derive(Debug, Clone, Serialize)]
187pub struct EvalComparisonSuiteReport {
188 pub baseline: EvalSuiteReport,
189 pub outcome_directed: EvalSuiteReport,
190 pub pass_rate_delta: f64,
191 pub mean_duration_delta_ms: f64,
192 pub token_delta: i128,
193}
194
195pub async fn compare_case(case: &EvalCase) -> EvalComparisonReport {
196 let mut baseline_case = case.clone();
197 baseline_case.outcome = None;
198 let baseline = run_case(&baseline_case).await;
199 let outcome_directed = run_case(case).await;
200 EvalComparisonReport {
201 task_id: case.id.clone(),
202 latency_delta_ms: outcome_directed.duration_ms as i128 - baseline.duration_ms as i128,
203 token_delta: (outcome_directed.tokens_in + outcome_directed.tokens_out) as i128
204 - (baseline.tokens_in + baseline.tokens_out) as i128,
205 baseline,
206 outcome_directed,
207 }
208}
209
210pub async fn compare_suite(cases: &[EvalCase]) -> EvalComparisonSuiteReport {
211 let mut baseline_cases = Vec::with_capacity(cases.len());
212 for case in cases {
213 let mut baseline = case.clone();
214 baseline.outcome = None;
215 baseline_cases.push(baseline);
216 }
217 let baseline = run_suite(&baseline_cases).await;
218 let outcome_directed = run_suite(cases).await;
219 EvalComparisonSuiteReport {
220 pass_rate_delta: outcome_directed.pass_rate - baseline.pass_rate,
221 mean_duration_delta_ms: outcome_directed.mean_duration_ms - baseline.mean_duration_ms,
222 token_delta: (outcome_directed.total_tokens_in + outcome_directed.total_tokens_out) as i128
223 - (baseline.total_tokens_in + baseline.total_tokens_out) as i128,
224 baseline,
225 outcome_directed,
226 }
227}
228
229impl EvalSuiteReport {
230 fn from_cases(cases: Vec<EvalReport>) -> Self {
231 let total = cases.len();
232 let passed = cases.iter().filter(|case| case.passed).count();
233 let mean_duration_ms = if total == 0 {
234 0.0
235 } else {
236 cases
237 .iter()
238 .map(|case| case.duration_ms as f64)
239 .sum::<f64>()
240 / total as f64
241 };
242 EvalSuiteReport {
243 total,
244 passed,
245 pass_rate: if total == 0 {
246 0.0
247 } else {
248 passed as f64 / total as f64
249 },
250 mean_duration_ms,
251 total_tokens_in: cases.iter().map(|case| case.tokens_in).sum(),
252 total_tokens_out: cases.iter().map(|case| case.tokens_out).sum(),
253 partial_or_unknown: cases
254 .iter()
255 .filter(|case| matches!(case.completion_verdict.as_str(), "partial" | "unknown"))
256 .count(),
257 human_review_recommended: cases
258 .iter()
259 .filter(|case| case.human_review == "recommended")
260 .count(),
261 cases,
262 }
263 }
264}
265
266/// Run a corpus and retain every case report for inspection and comparison.
267pub async fn run_suite(cases: &[EvalCase]) -> EvalSuiteReport {
268 let mut reports = Vec::with_capacity(cases.len());
269 for case in cases {
270 reports.push(run_case(case).await);
271 }
272 EvalSuiteReport::from_cases(reports)
273}
274
275/// Runs one case with the built-in scripted provider (deterministic).
276pub async fn run_case(case: &EvalCase) -> EvalReport {
277 let provider = Arc::new(EvalProvider::new(case.script.clone()));
278 run_case_with_provider(case, provider, "eval-model").await
279}
280
281pub async fn run_case_brokered(case: &EvalCase, worker_exe: std::path::PathBuf) -> EvalReport {
282 let provider = Arc::new(EvalProvider::new(case.script.clone()));
283 run_case_with_provider_brokered(case, provider, "eval-model", worker_exe).await
284}
285
286/// Runs one case against ANY provider — the live-model path. The `script`
287/// field is ignored; the model must genuinely solve the task.
288pub async fn run_case_with_provider(
289 case: &EvalCase,
290 provider: Arc<dyn Provider>,
291 model: &str,
292) -> EvalReport {
293 run_case_with_tools(case, provider, model, vak_tools::default_tools()).await
294}
295
296pub async fn run_case_with_provider_brokered(
297 case: &EvalCase,
298 provider: Arc<dyn Provider>,
299 model: &str,
300 worker_exe: std::path::PathBuf,
301) -> EvalReport {
302 run_case_with_tools(
303 case,
304 provider,
305 model,
306 vak_tools::brokered_default_tools(worker_exe),
307 )
308 .await
309}
310
311async fn run_case_with_tools(
312 case: &EvalCase,
313 provider: Arc<dyn Provider>,
314 model: &str,
315 tools: Vec<Arc<dyn Tool>>,
316) -> EvalReport {
317 let deterministic = provider.name() == "eval-scripted";
318 let start = Instant::now();
319 let dir = match tempfile::tempdir() {
320 Ok(d) => d,
321 Err(e) => {
322 return EvalReport {
323 task_id: case.id.clone(),
324 passed: false,
325 verify_exit: None,
326 tokens_in: 0,
327 tokens_out: 0,
328 duration_ms: start.elapsed().as_millis(),
329 outcome_status: "unknown".into(),
330 completion_verdict: "failed".into(),
331 human_review: "required_for_recovery".into(),
332 verification_evidence: "none".into(),
333 error: Some(format!("workspace setup failed: {e}")),
334 };
335 }
336 };
337 let cwd = dir.path().to_path_buf();
338 for (rel, content) in &case.files {
339 let p = cwd.join(rel);
340 if let Some(parent) = p.parent() {
341 let _ = std::fs::create_dir_all(parent);
342 }
343 if std::fs::write(&p, content).is_err() {
344 return fail(&case.id, "setup file write failed", &start, None);
345 }
346 }
347
348 let header = SessionHeader {
349 agent: Some(vak_session::types::AgentIdentity {
350 id: "vak".into(),
351 revision: 1,
352 name: "Vakyartha".into(),
353 character: "vak".into(),
354 personality: String::new(),
355 animation: "subtle".into(),
356 voice: "default".into(),
357 behaviour: String::new(),
358 responsibilities: String::new(),
359 instructions: String::new(),
360 }),
361 session_id: format!("eval-{}", case.id),
362 created_at: chrono::Utc::now(),
363 cwd: cwd.clone(),
364 parent_session_id: None,
365 contract_id: None,
366 work_item_id: None,
367 conversation: Some(vak_session::ConversationContext::local(
368 format!("eval-{}", case.id),
369 "eval",
370 )),
371 contract: FrozenContract {
372 app_version: env!("CARGO_PKG_VERSION").into(),
373 provider: "eval-scripted".into(),
374 model: model.to_string(),
375 route_ladder: Vec::new(),
376 route_objective: String::new(),
377 route_annotations: Vec::new(),
378 system_prompt: "eval".into(),
379 permission_mode: "full-access".into(),
380 capabilities: Vec::new(),
381 prompt_layers: Vec::new(),
382 },
383 };
384 let sessions_home = cwd.join(".vak-home");
385 let log = match SessionLog::create(
386 vak_session::SessionPath::new_session_file(&sessions_home, &cwd, &header.session_id),
387 header,
388 ) {
389 Ok(l) => l,
390 Err(e) => {
391 return fail(
392 &case.id,
393 &format!("session setup failed: {e}"),
394 &start,
395 None,
396 );
397 }
398 };
399
400 let prepared = vak_core::PreparedTurn::from_parts("eval", tools, Vec::new());
401 let mut cfg = AgentConfig::new(prepared.system_prompt);
402 cfg.model = model.to_string();
403 cfg.tools = prepared.tools;
404 cfg.outcome = case.outcome.clone();
405 cfg.max_turns = 12;
406 if deterministic {
407 cfg.max_retries = 0;
408 cfg.run_retry_attempts = 0;
409 }
410 cfg.permission = Some(Arc::new(PermissionEngine::default()));
411 cfg.mode = Mode::FullAccess;
412 cfg.approver = Some(Arc::new(vak_agent::AutoApprove));
413 let mut agent = Agent::new(provider, log, cfg);
414
415 let (events, mut events_rx) = tokio::sync::mpsc::channel(256);
416 let drain = tokio::spawn(async move { while events_rx.recv().await.is_some() {} });
417 let outcome = agent
418 .run(
419 &case.prompt,
420 &Default::default(),
421 CancellationToken::new(),
422 events,
423 )
424 .await;
425 let _ = drain.await;
426
427 let (tokens_in, tokens_out, loop_error) = {
428 let session = agent.session.lock().await;
429 let u = session.total_usage();
430 let err = match outcome {
431 TurnOutcome::Completed { .. } => None,
432 TurnOutcome::Aborted { .. } => Some("aborted".to_string()),
433 TurnOutcome::Failed { ref error } => Some(error.to_string()),
434 TurnOutcome::MaxTurnsReached => Some("max turns reached".to_string()),
435 };
436 (u.input_tokens, u.output_tokens, err)
437 };
438
439 // Verification runs in the same workspace with the production bash tool.
440 let verify_ctx = ToolContext {
441 cwd: cwd.clone(),
442 cancel: CancellationToken::new(),
443 sandbox: None,
444 sandbox_sink: None,
445 agent_id: None,
446 new_documents: Vec::new(),
447 };
448 let verify_out = BashTool
449 .execute(&serde_json::json!({"command": case.verify}), &verify_ctx)
450 .await;
451
452 let report = EvalReport {
453 task_id: case.id.clone(),
454 passed: !verify_out.is_error,
455 verify_exit: Some(if verify_out.is_error { 1 } else { 0 }),
456 tokens_in,
457 tokens_out,
458 duration_ms: start.elapsed().as_millis(),
459 outcome_status: format!(
460 "{:?}",
461 vak_intent::evaluate_response(
462 match &outcome {
463 TurnOutcome::Completed { response } => Some(response.text_content()),
464 TurnOutcome::Aborted { partial } => {
465 partial.as_ref().map(|message| message.text_content())
466 }
467 TurnOutcome::Failed { .. } | TurnOutcome::MaxTurnsReached => None,
468 }
469 .as_deref(),
470 loop_error.is_some(),
471 matches!(outcome, TurnOutcome::Aborted { .. }),
472 )
473 )
474 .to_ascii_lowercase(),
475 completion_verdict: {
476 let response = match &outcome {
477 TurnOutcome::Completed { response } => Some(response.text_content()),
478 TurnOutcome::Aborted { partial } => {
479 partial.as_ref().map(|message| message.text_content())
480 }
481 TurnOutcome::Failed { .. } | TurnOutcome::MaxTurnsReached => None,
482 };
483 let spec = case.outcome.clone().unwrap_or_else(|| {
484 vak_intent::OutcomeSpec::from_reading(
485 &case.prompt,
486 &vak_intent::Reading::general(),
487 vak_intent::RESOLVER_VERSION,
488 )
489 });
490 let status = vak_intent::evaluate_response(
491 response.as_deref(),
492 loop_error.is_some(),
493 matches!(outcome, TurnOutcome::Aborted { .. }),
494 );
495 let evaluations = vak_intent::evaluate_requirements(&spec, response.as_deref());
496 let verdict = vak_intent::evaluate_completion(status, &evaluations, &spec);
497 if verify_out.is_error && matches!(verdict, vak_intent::CompletionVerdict::Complete) {
498 "partial".into()
499 } else {
500 format!("{verdict:?}").to_ascii_lowercase()
501 }
502 },
503 human_review: if verify_out.is_error
504 || case.outcome.as_ref().is_some_and(|outcome| {
505 outcome.requirements.iter().any(|requirement| {
506 matches!(requirement.kind, vak_intent::RequirementKind::Evidence)
507 })
508 }) {
509 "recommended".into()
510 } else {
511 "not_required".into()
512 },
513 verification_evidence: if verify_out.is_error {
514 "observed_failure".into()
515 } else {
516 "observed_success".into()
517 },
518 error: loop_error.or_else(|| {
519 if verify_out.is_error {
520 Some(format!("verify failed: {}", verify_out.content))
521 } else {
522 None
523 }
524 }),
525 };
526 keep_workspace(&cwd, &report);
527 report
528}
529
530fn fail(id: &str, msg: &str, start: &Instant, verify_exit: Option<i32>) -> EvalReport {
531 EvalReport {
532 task_id: id.to_string(),
533 passed: false,
534 verify_exit,
535 tokens_in: 0,
536 tokens_out: 0,
537 duration_ms: start.elapsed().as_millis(),
538 outcome_status: "unknown".into(),
539 completion_verdict: "failed".into(),
540 human_review: "required_for_recovery".into(),
541 verification_evidence: "none".into(),
542 error: Some(msg.to_string()),
543 }
544}
545
546/// Keep failing workspaces under target/ for post-mortem; drop passing ones.
547fn keep_workspace(_cwd: &Path, report: &EvalReport) {
548 let _ = report;
549}
550

Runner

Indexing the workspace…

Runner