1. 1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2. 2
3. 3//! The built-in suite must always pass: these are regression gates for the
4. 4//! harness itself, not model benchmarks. CI fails if any case regresses.
5. 5
6. 6#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7. 7async fn builtin_suite_fully_green() {
8. 8 let mut failures = Vec::new();
9. 9 for case in vak_eval::builtin_suite() {
10. 10 let report = vak_eval::run_case(&case).await;
11. 11 if !report.passed {
12. 12 failures.push(format!(
13. 13 "{}: {:?}",
14. 14 report.task_id,
15. 15 report.error.as_deref().unwrap_or("verify failed")
16. 16 ));
17. 17 }
18. 18 }
19. 19 assert!(
20. 20 failures.is_empty(),
21. 21 "eval regressions:\n{}",
22. 22 failures.join("\n")
23. 23 );
24. 24}
25. 25
26. 26#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
27. 27async fn exhausted_script_is_reported_not_hung() {
28. 28 let mut case = vak_eval::EvalCase {
29. 29 id: "exhausted".into(),
30. 30 description: "script ends before the loop stops requesting".into(),
31. 31 files: vec![],
32. 32 prompt: "loop forever".into(),
33. 33 script: vec![],
34. 34 outcome: None,
35. 35 verify: "true".into(),
36. 36 };
37. 37 let _ = &mut case;
38. 38 let report = vak_eval::run_case(&case).await;
39. 39 // The scripted provider errors; the loop surfaces Failed; verify still
40. 40 // passes (`true`), but the error is recorded on the report.
41. 41 assert!(report.error.is_some(), "exhaustion must be visible");
42. 42}
43. 43