1. 1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2. 2
3. 3//! The general-purpose (non-coding) suite must always pass: research,
4. 4//! data analysis, writing, conversion, inventory, and error-recovery
5. 5//! through the production loop with only the provider scripted.
6. 6
7. 7#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8. 8async fn general_suite_fully_green() {
9. 9 let mut failures = Vec::new();
10. 10 for case in vak_eval::general_suite() {
11. 11 let report = vak_eval::run_case(&case).await;
12. 12 assert_eq!(
13. 13 report.completion_verdict, "complete",
14. 14 "{} must satisfy its runtime outcome",
15. 15 report.task_id
16. 16 );
17. 17 assert_eq!(report.human_review, "not_required");
18. 18 assert_eq!(report.verification_evidence, "observed_success");
19. 19 if !report.passed {
20. 20 failures.push(format!(
21. 21 "{}: {:?}",
22. 22 report.task_id,
23. 23 report.error.as_deref().unwrap_or("verify failed")
24. 24 ));
25. 25 }
26. 26 }
27. 27 assert!(
28. 28 failures.is_empty(),
29. 29 "general eval regressions:\n{}",
30. 30 failures.join("\n")
31. 31 );
32. 32}
33. 33
34. 34#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
35. 35async fn general_suite_records_token_metrics() {
36. 36 let mut saw_usage = false;
37. 37 for case in vak_eval::general_suite() {
38. 38 let report = vak_eval::run_case(&case).await;
39. 39 assert!(report.passed, "case {} must pass", report.task_id);
40. 40 if report.tokens_in > 0 && report.tokens_out > 0 {
41. 41 saw_usage = true;
42. 42 }
43. 43 }
44. 44 assert!(saw_usage, "token accounting must be recorded per case");
45. 45}
46. 46
47. 47#[test]
48. 48fn general_suite_is_not_coding_only() {
49. 49 let cases = vak_eval::general_suite();
50. 50 assert!(
51. 51 cases.len() >= 5,
52. 52 "general-purpose corpus needs multiple domains"
53. 53 );
54. 54 let ids = cases
55. 55 .iter()
56. 56 .map(|case| case.id.to_ascii_lowercase())
57. 57 .collect::<Vec<_>>();
58. 58 for domain in [
59. 59 "research",
60. 60 "csv",
61. 61 "writing",
62. 62 "conversion",
63. 63 "inventory",
64. 64 "schedule",
65. 65 "decision",
66. 66 ] {
67. 67 assert!(
68. 68 ids.iter().any(|id| id.contains(domain)),
69. 69 "missing {domain} case"
70. 70 );
71. 71 }
72. 72}
73. 73
74. 74#[test]
75. 75fn held_out_corpus_has_context_and_deliverable_variation() {
76. 76 let cases = vak_eval::held_out_suite();
77. 77 assert!(cases.len() >= 2);
78. 78 assert!(cases.iter().any(|case| case.prompt.contains("English")));
79. 79 assert!(cases.iter().any(|case| case.prompt.contains("one-line")));
80. 80 assert!(cases.iter().all(|case| !case.verify.is_empty()));
81. 81 assert!(cases.iter().any(|case| {
82. 82 case.outcome
83. 83 .as_ref()
84. 84 .is_some_and(|outcome| outcome.requirements.len() >= 2)
85. 85 }));
86. 86}
87. 87
88. 88#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
89. 89async fn held_out_corpus_runs_through_the_production_loop() {
90. 90 for case in vak_eval::held_out_suite() {
91. 91 let report = vak_eval::run_case(&case).await;
92. 92 assert!(report.passed, "{} must pass", report.task_id);
93. 93 assert_eq!(report.verification_evidence, "observed_success");
94. 94 }
95. 95}
96. 96
97. 97#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
98. 98async fn explicit_held_out_evidence_requirement_downgrades_verdict() {
99. 99 let case = vak_eval::held_out_suite()
100. 100 .into_iter()
101. 101 .find(|case| case.id == "held-out-multilingual-note")
102. 102 .expect("multilingual held-out case");
103. 103 let report = vak_eval::run_case(&case).await;
104. 104 assert!(report.passed);
105. 105 assert_eq!(report.completion_verdict, "partial");
106. 106 assert_eq!(report.human_review, "recommended");
107. 107}
108. 108
109. 109#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
110. 110async fn comparison_proves_outcome_directed_evidence_changes_verdict() {
111. 111 let case = vak_eval::held_out_suite()
112. 112 .into_iter()
113. 113 .find(|case| case.id == "held-out-multilingual-note")
114. 114 .expect("multilingual held-out case");
115. 115 let comparison = vak_eval::compare_case(&case).await;
116. 116 assert!(comparison.baseline.passed);
117. 117 assert!(comparison.outcome_directed.passed);
118. 118 assert_eq!(comparison.baseline.completion_verdict, "complete");
119. 119 assert_eq!(comparison.outcome_directed.completion_verdict, "partial");
120. 120 assert_eq!(comparison.outcome_directed.human_review, "recommended");
121. 121}
122. 122
123. 123#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
124. 124async fn held_out_comparison_reports_corpus_level_deltas() {
125. 125 let comparison = vak_eval::compare_suite(&vak_eval::held_out_suite()).await;
126. 126 assert_eq!(comparison.baseline.total, 3);
127. 127 assert_eq!(comparison.outcome_directed.total, 3);
128. 128 assert_eq!(comparison.baseline.passed, 3);
129. 129 assert_eq!(comparison.outcome_directed.passed, 3);
130. 130 assert_eq!(comparison.outcome_directed.human_review_recommended, 1);
131. 131 assert!(comparison.token_delta >= 0);
132. 132}
133. 133
134. 134#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
135. 135async fn held_out_corpus_reports_aggregate_measurement() {
136. 136 let report = vak_eval::run_suite(&vak_eval::held_out_suite()).await;
137. 137 assert_eq!(report.total, 3);
138. 138 assert_eq!(report.passed, report.total);
139. 139 assert_eq!(report.pass_rate, 1.0);
140. 140 assert!(report.total_tokens_in > 0);
141. 141 assert!(report.total_tokens_out > 0);
142. 142 assert_eq!(report.cases.len(), report.total);
143. 143 assert_eq!(report.partial_or_unknown, 1);
144. 144 assert_eq!(report.human_review_recommended, 1);
145. 145}
146. 146
147. 147#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
148. 148async fn failed_external_verification_cannot_claim_completion() {
149. 149 let case = vak_eval::EvalCase {
150. 150 id: "negative-verification".into(),
151. 151 description: "a plausible response with a failed external check".into(),
152. 152 files: vec![],
153. 153 prompt: "produce the requested result".into(),
154. 154 script: vec![vak_eval::ScriptedTurn::Text("The result is ready.".into())],
155. 155 outcome: None,
156. 156 verify: "false".into(),
157. 157 };
158. 158 let report = vak_eval::run_case(&case).await;
159. 159 assert!(!report.passed);
160. 160 assert_eq!(report.outcome_status, "produced");
161. 161 assert_eq!(report.completion_verdict, "partial");
162. 162 assert_eq!(report.human_review, "recommended");
163. 163 assert_eq!(report.verification_evidence, "observed_failure");
164. 164}
165. 165