1. 1//! Built-in stop gate: blocks premature completions.
2. 2//!
3. 3//! Small models frequently end their turn mid-plan ("Fixing both:" /
4. 4//! "Now I'll write the tests") or finish without running verification they
5. 5//! explicitly promised. The dogfood campaign measured this at 5/7 runs on a
6. 6//! free tier. This policy reuses the existing stop-hook continuation
7. 7//! machinery (append "[stop-guard]: reason / Please continue.", emit
8. 8//! StopHookContinuation) at most `max_blocks` times per run, so it can
9. 9//! never trap a model in a loop.
10. 10
11. 11/// What triggered the block — also the operator-visible reason.
12. 12#[derive(Debug, Clone, PartialEq, Eq)]
13. 13pub enum BlockReason {
14. 14 /// Final text looks truncated: ends with ':', a bare plan marker, or an
15. 15 /// unclosed fenced code block.
16. 16 TruncatedPlan(String),
17. 17 /// The prompt demanded running/testing something and the run never
18. 18 /// executed a single bash command.
19. 19 VerificationMissing,
20. 20 /// A file-changing tool ran after the last verification command.
21. 21 VerificationStale,
22. 22 /// The user explicitly asked the agent to continue until a later user
23. 23 /// message authorizes completion.
24. 24 UserCompletionRequired,
25. 25 /// The request requires execution or tool receipts, but none were produced.
26. 26 ExecutionReceiptMissing { act: String, hint: String },
27. 27 /// A tool failed with an error and the model neither repaired it nor reported the blocker.
28. 28 UnresolvedToolFailure { tool: String, error: String },
29. 29}
30. 30
31. 31impl BlockReason {
32. 32 pub fn message(&self) -> String {
33. 33 match self {
34. 34 BlockReason::TruncatedPlan(tail) => format!(
35. 35 "your last message appears cut off mid-plan (ends with {tail:?}). \
36. 36 Finish the work now; if you are actually done, say so plainly."
37. 37 ),
38. 38 BlockReason::VerificationMissing => String::from(
39. 39 "the task asked for verification or sandbox execution, but no substantive commands were \
40. 40 executed this run. Call the `bash` tool to actually execute, build, or verify the work now; do not print commands or dummy echo statements.",
41. 41 ),
42. 42 BlockReason::VerificationStale => String::from(
43. 43 "the task changed files after its last verification command. \
44. 44 Call the `bash` tool to run the verification again before finishing; do not describe it in text.",
45. 45 ),
46. 46 BlockReason::UserCompletionRequired => String::from(
47. 47 "the user asked you to keep working until they say done. Continue making \
48. 48 useful progress; do not declare completion yet.",
49. 49 ),
50. 50 BlockReason::ExecutionReceiptMissing { act, hint } => format!(
51. 51 "the request requires {act} ({hint}), but no execution or modification receipts were produced. \
52. 52 Execute the necessary commands or file edits now using the available tools; do not just describe the work in prose.",
53. 53 ),
54. 54 BlockReason::UnresolvedToolFailure { tool, error } => format!(
55. 55 "the `{tool}` tool failed with an error: {error}. \
56. 56 Repair the failure using the appropriate tools, or clearly report the concrete blocker to the user.",
57. 57 ),
58. 58 }
59. 59 }
60. 60}
61. 61
62. 62/// Summary of tool execution receipts produced during a run.
63. 63#[derive(Debug, Clone, Default, PartialEq, Eq)]
64. 64pub struct ReceiptSummary {
65. 65 /// Proven saved file from the same intent thread before a step-limit
66. 66 /// continuation. Only counts with a successful inspection in this turn.
67. 67 pub continued_saved_file: bool,
68. 68 /// Total tool invocations attempted.
69. 69 pub total_tool_calls: u32,
70. 70 /// Number of tool calls that completed with ToolRunOutput::Ok.
71. 71 pub successful_tool_calls: u32,
72. 72 /// Number of tool calls that completed with ToolRunOutput::Err.
73. 73 pub failed_tool_calls: u32,
74. 74 /// Number of substantive bash invocations.
75. 75 pub substantive_bash_calls: u32,
76. 76 /// Number of files modified/written (via edit, write, etc.).
77. 77 pub files_modified: u32,
78. 78 /// Number of code files modified/written.
79. 79 pub code_files_modified: u32,
80. 80 /// Number of documentation/content/non-code files modified/written.
81. 81 pub doc_files_modified: u32,
82. 82 /// Number of inspection/read tool calls (read_file, glob, grep, etc.).
83. 83 pub read_or_inspected: u32,
84. 84 /// Inspections that actually returned successfully. A prior saved file
85. 85 /// only counts after one of these in the continuation turn.
86. 86 pub successful_inspections: u32,
87. 87 /// Work that succeeded outside the built-in tools: an integration's tool
88. 88 /// invoked through `mcp` (`action = "call"`) or a delegated `task`,
89. 89 /// whose worker keeps its own receipts. Counted on success only.
90. 90 pub external_effects: u32,
91. 91 /// Most recent unresolved tool failure, if any.
92. 92 pub unresolved_error: Option<(String, String)>,
93. 93}
94. 94
95. 95impl ReceiptSummary {
96. 96 pub fn has_execution_receipt(&self) -> bool {
97. 97 self.substantive_bash_calls > 0
98. 98 || self.files_modified > 0
99. 99 || self.external_effects > 0
100. 100 || (self.continued_saved_file && self.successful_inspections > 0)
101. 101 }
102. 102
103. 103 pub fn has_inspection_receipt(&self) -> bool {
104. 104 self.read_or_inspected > 0 || self.has_execution_receipt()
105. 105 }
106. 106
107. 107 pub fn has_any_receipt(&self) -> bool {
108. 108 self.successful_tool_calls > 0
109. 109 }
110. 110}
111. 111
112. 112fn reports_blocker(text: &str, tool: &str, error: &str) -> bool {
113. 113 let lower = text.to_ascii_lowercase();
114. 114 let err_first_line = error
115. 115 .lines()
116. 116 .next()
117. 117 .unwrap_or("")
118. 118 .trim()
119. 119 .to_ascii_lowercase();
120. 120 let keywords = [
121. 121 "error",
122. 122 "failed",
123. 123 "failure",
124. 124 "failing",
125. 125 "blocked",
126. 126 "blocker",
127. 127 "could not",
128. 128 "cannot",
129. 129 "can't",
130. 130 "unable to",
131. 131 "unavailable",
132. 132 "issue",
133. 133 "problem",
134. 134 "exit code",
135. 135 "exception",
136. 136 "recover",
137. 137 "repaired",
138. 138 "unsupported",
139. 139 "guard",
140. 140 "denied",
141. 141 "denial",
142. 142 "rejected",
143. 143 "rejection",
144. 144 "repeated",
145. 145 "skip",
146. 146 "skipping",
147. 147 ];
148. 148 let mentions_keyword = keywords.iter().any(|k| lower.contains(k));
149. 149 let mentions_tool = lower.contains(&tool.to_ascii_lowercase());
150. 150 let mentions_snippet = !err_first_line.is_empty() && lower.contains(&err_first_line);
151. 151 mentions_keyword || mentions_tool || mentions_snippet
152. 152}
153. 153
154. 154#[derive(Debug, Clone)]
155. 155pub struct StopPolicy {
156. 156 /// Gate on truncated-looking final messages.
157. 157 pub marker_gate: bool,
158. 158 /// Gate on promised-but-never-run verification.
159. 159 pub verify_gate: bool,
160. 160 /// Hard cap of guard continuations per run.
161. 161 pub max_blocks: u32,
162. 162}
163. 163
164. 164impl Default for StopPolicy {
165. 165 fn default() -> Self {
166. 166 StopPolicy {
167. 167 marker_gate: true,
168. 168 verify_gate: true,
169. 169 max_blocks: 2,
170. 170 }
171. 171 }
172. 172}
173. 173
174. 174impl StopPolicy {
175. 175 pub fn requires_user_completion(prompt: &str) -> bool {
176. 176 let p = prompt.to_ascii_lowercase();
177. 177 [
178. 178 "until i say done",
179. 179 "until i tell you to stop",
180. 180 "until i tell you you're done",
181. 181 "keep working until",
182. 182 "keep improving until",
183. 183 "don't stop until",
184. 184 "do not stop until",
185. 185 ]
186. 186 .iter()
187. 187 .any(|marker| p.contains(marker))
188. 188 }
189. 189
190. 190 pub fn is_done_message(message: &str) -> bool {
191. 191 let normalized = message.trim().to_ascii_lowercase();
192. 192 [
193. 193 "done",
194. 194 "stop",
195. 195 "you can stop",
196. 196 "that's enough",
197. 197 "that’s enough",
198. 198 ]
199. 199 .iter()
200. 200 .any(|marker| normalized == *marker)
201. 201 }
202. 202
203. 203 /// Conservative trailing-intent patterns: only fire on line-final
204. 204 /// markers so normal prose summaries never match.
205. 205 fn truncated_plan(final_text: &str) -> Option<BlockReason> {
206. 206 let trimmed = final_text.trim_end();
207. 207 if trimmed.is_empty() {
208. 208 return None;
209. 209 }
210. 210 // Unclosed fenced code block: strong truncation signal.
211. 211 if trimmed.matches("```").count() % 2 == 1 {
212. 212 return Some(BlockReason::TruncatedPlan("unclosed code fence".into()));
213. 213 }
214. 214 let last_line = trimmed.lines().next_back()?.trim_end();
215. 215 if last_line.ends_with(':') && !last_line.starts_with('#') && last_line.len() < 200 {
216. 216 return Some(BlockReason::TruncatedPlan(format!(
217. 217 "'{}'",
218. 218 last_line
219. 219 .chars()
220. 220 .rev()
221. 221 .take(40)
222. 222 .collect::<Vec<_>>()
223. 223 .into_iter()
224. 224 .rev()
225. 225 .collect::<String>()
226. 226 )));
227. 227 }
228. 228 const MARKERS: [&str; 8] = [
229. 229 "now i'll",
230. 230 "now let me",
231. 231 "let me ",
232. 232 "i will ",
233. 233 "i'll ",
234. 234 "going to ",
235. 235 "next,",
236. 236 "then,",
237. 237 ];
238. 238 let lower = last_line.to_ascii_lowercase();
239. 239 if lower.len() < 300
240. 240 && MARKERS
241. 241 .iter()
242. 242 .any(|m| lower.starts_with(m) || lower.contains(m))
243. 243 && !lower.ends_with('.')
244. 244 && !lower.ends_with('!')
245. 245 && !lower.ends_with('?')
246. 246 {
247. 247 return Some(BlockReason::TruncatedPlan("a plan marker".into()));
248. 248 }
249. 249 None
250. 250 }
251. 251
252. 252 /// True when the prompt itself explicitly asks for executed verification, testing, or sandbox commands.
253. 253 pub fn demands_code_execution(prompt: &str) -> bool {
254. 254 let stripped = if let Some(idx) = prompt.find("[Scheduled-run context:") {
255. 255 &prompt[..idx]
256. 256 } else {
257. 257 prompt
258. 258 };
259. 259 const DEMANDS: [&str; 11] = [
260. 260 "must pass",
261. 261 "tests pass",
262. 262 "test pass",
263. 263 "run it",
264. 264 "run them",
265. 265 "run the test",
266. 266 "run tests",
267. 267 "verify by running",
268. 268 "prove by running",
269. 269 "run in sandbox",
270. 270 "execute in sandbox",
271. 271 ];
272. 272 let p = stripped.to_ascii_lowercase();
273. 273 if DEMANDS.iter().any(|d| p.contains(d)) {
274. 274 return true;
275. 275 }
276. 276 if p.contains("sandbox")
277. 277 && ["run", "show", "test", "build", "execute", "serve", "start"]
278. 278 .iter()
279. 279 .any(|action| p.contains(action))
280. 280 {
281. 281 return true;
282. 282 }
283. 283 if p.contains("cargo test")
284. 284 || p.contains("pytest")
285. 285 || p.contains("npm test")
286. 286 || p.contains("go test")
287. 287 || p.contains("python -m unittest")
288. 288 {
289. 289 return true;
290. 290 }
291. 291 p.contains("run ")
292. 292 && [
293. 293 "test", "tests", "command", "script", "check", "app", "code", "python", "cargo",
294. 294 "binary",
295. 295 ]
296. 296 .iter()
297. 297 .any(|word| p.contains(word))
298. 298 }
299. 299
300. 300 /// True when the prompt asks for any verification (code or universal/content).
301. 301 pub fn demands_verification(prompt: &str) -> bool {
302. 302 if Self::demands_code_execution(prompt) {
303. 303 return true;
304. 304 }
305. 305 let stripped = if let Some(idx) = prompt.find("[Scheduled-run context:") {
306. 306 &prompt[..idx]
307. 307 } else {
308. 308 prompt
309. 309 };
310. 310 let p = stripped.to_ascii_lowercase();
311. 311 const VERIFY_MARKERS: [&str; 7] = [
312. 312 "verify that",
313. 313 "verify the",
314. 314 "double check",
315. 315 "double-check",
316. 316 "make sure that",
317. 317 "check that",
318. 318 "verify whether",
319. 319 ];
320. 320 VERIFY_MARKERS.iter().any(|m| p.contains(m))
321. 321 }
322. 322
323. 323 /// True when the assistant response claims execution or emits shell scripts without tool calls having run.
324. 324 pub fn claims_execution_unexecuted(final_text: &str) -> bool {
325. 325 let lower = final_text.to_ascii_lowercase();
326. 326 let markers = [
327. 327 "use the bash tool",
328. 328 "use the `bash` tool",
329. 329 "using the bash tool",
330. 330 "using the `bash` tool",
331. 331 "run the python script",
332. 332 "running the python script",
333. 333 "execute the python script",
334. 334 "executing the python script",
335. 335 "execute in the sandbox",
336. 336 "running in the sandbox",
337. 337 "run in the sandbox",
338. 338 "execute in sandbox",
339. 339 "running in sandbox",
340. 340 ];
341. 341 if markers.iter().any(|m| lower.contains(m)) {
342. 342 return true;
343. 343 }
344. 344 if (lower.contains("```bash") || lower.contains("```sh"))
345. 345 && (lower.contains(".vak/scratch")
346. 346 || lower.contains("python3 ")
347. 347 || lower.contains("node ")
348. 348 || lower.contains("cargo "))
349. 349 {
350. 350 return true;
351. 351 }
352. 352 false
353. 353 }
354. 354
355. 355 /// Intent- and receipt-driven evaluation of completion validity.
356. 356 pub fn evaluate_receipts(
357. 357 &self,
358. 358 prompt: &str,
359. 359 final_text: &str,
360. 360 outcome: Option<&vak_intent::OutcomeSpec>,
361. 361 receipts: &ReceiptSummary,
362. 362 verification_stale: bool,
363. 363 ) -> Option<BlockReason> {
364. 364 if final_text.trim().is_empty() {
365. 365 return None;
366. 366 }
367. 367 if Self::requires_user_completion(prompt) {
368. 368 return Some(BlockReason::UserCompletionRequired);
369. 369 }
370. 370 if self.marker_gate
371. 371 && let Some(r) = Self::truncated_plan(final_text)
372. 372 {
373. 373 return Some(r);
374. 374 }
375. 375
376. 376 // If a tool failed and hasn't been repaired or reported in text, block.
377. 377 if let Some((tool, err)) = &receipts.unresolved_error
378. 378 && !reports_blocker(final_text, tool, err)
379. 379 {
380. 380 return Some(BlockReason::UnresolvedToolFailure {
381. 381 tool: tool.clone(),
382. 382 error: err.clone(),
383. 383 });
384. 384 }
385. 385
386. 386 // Intent-driven gate: when outcome specification is available. The
387. 387 // engagement's stop profile (docs/design/47-commitment-kernel.md)
388. 388 // decides first; the typed acts refine it.
389. 389 if let Some(spec) = outcome {
390. 390 use vak_intent::StopProfile;
391. 391 // `Verification`: a checkable result was demanded. An execution
392. 392 // receipt is required, and it must not be stale.
393. 393 if spec.stop == StopProfile::Verification {
394. 394 if !receipts.has_execution_receipt() {
395. 395 return Some(BlockReason::ExecutionReceiptMissing {
396. 396 act: spec.deliverable_act().unwrap_or("verification").to_string(),
397. 397 hint: "run the check that proves this is done".into(),
398. 398 });
399. 399 }
400. 400 if verification_stale && receipts.code_files_modified > 0 {
401. 401 return Some(BlockReason::VerificationStale);
402. 402 }
403. 403 }
404. 404 if spec.stop == StopProfile::Effect || spec.requires_execution() {
405. 405 if !receipts.has_execution_receipt() {
406. 406 let act = spec.deliverable_act().unwrap_or("execution").to_string();
407. 407 return Some(BlockReason::ExecutionReceiptMissing {
408. 408 act,
409. 409 hint: "run code, build, test, or modify files".into(),
410. 410 });
411. 411 }
412. 412 if self.verify_gate
413. 413 && verification_stale
414. 414 && (spec.requires_execution() || Self::demands_verification(prompt))
415. 415 && (receipts.code_files_modified > 0 || Self::demands_code_execution(prompt))
416. 416 {
417. 417 return Some(BlockReason::VerificationStale);
418. 418 }
419. 419 // `Inspection` on its own gates nothing: "something was looked
420. 420 // at" includes the material the request carried (an attachment,
421. 421 // pasted text), which leaves no receipt. Only a `locate` act
422. 422 // demands an inspection receipt.
423. 423 } else if spec.requires_inspection() {
424. 424 let direct_substantive = final_text.trim().len() >= 80
425. 425 && !Self::demands_code_execution(prompt)
426. 426 && !Self::claims_execution_unexecuted(final_text);
427. 427 if !receipts.has_inspection_receipt() && !direct_substantive {
428. 428 let act = spec.deliverable_act().unwrap_or("inspection").to_string();
429. 429 return Some(BlockReason::ExecutionReceiptMissing {
430. 430 act,
431. 431 hint: "read, search, inspect files or data".into(),
432. 432 });
433. 433 }
434. 434 } else if spec.requires_tool() && !receipts.has_any_receipt() {
435. 435 let direct_substantive = final_text.trim().len() >= 80
436. 436 && !Self::demands_code_execution(prompt)
437. 437 && !Self::claims_execution_unexecuted(final_text);
438. 438 if !direct_substantive {
439. 439 return Some(BlockReason::ExecutionReceiptMissing {
440. 440 act: "tool execution".into(),
441. 441 hint: "execute relevant tools".into(),
442. 442 });
443. 443 }
444. 444 }
445. 445 }
446. 446
447. 447 // Verification and execution gate: runs whenever verify_gate is enabled.
448. 448 if self.verify_gate {
449. 449 let demands_code = Self::demands_code_execution(prompt);
450. 450 let claims_exec = Self::claims_execution_unexecuted(final_text);
451. 451
452. 452 if claims_exec {
453. 453 return Some(BlockReason::VerificationMissing);
454. 454 }
455. 455
456. 456 if demands_code && receipts.substantive_bash_calls == 0 {
457. 457 return Some(BlockReason::VerificationMissing);
458. 458 }
459. 459
460. 460 if Self::demands_verification(prompt) && receipts.substantive_bash_calls == 0 {
461. 461 // If code files were touched, verification commands are required.
462. 462 if receipts.code_files_modified > 0 {
463. 463 return Some(BlockReason::VerificationMissing);
464. 464 }
465. 465 // If no tools were called and the text is not a substantive direct answer:
466. 466 let direct_substantive = final_text.trim().len() >= 80;
467. 467 if !receipts.has_any_receipt() && !direct_substantive {
468. 468 return Some(BlockReason::VerificationMissing);
469. 469 }
470. 470 // If an execution deliverable was required or a specific file target was requested,
471. 471 // but no files were modified or inspected:
472. 472 let lower_p = prompt.to_ascii_lowercase();
473. 473 let mentions_file_target = lower_p.contains(".md")
474. 474 || lower_p.contains(".txt")
475. 475 || lower_p.contains(".json")
476. 476 || lower_p.contains(".csv")
477. 477 || lower_p.contains("into ")
478. 478 || lower_p.contains("in file")
479. 479 || lower_p.contains("in the file");
480. 480 if (outcome.map(|s| s.requires_execution()).unwrap_or(false)
481. 481 || mentions_file_target)
482. 482 && receipts.files_modified == 0
483. 483 && receipts.read_or_inspected == 0
484. 484 {
485. 485 return Some(BlockReason::VerificationMissing);
486. 486 }
487. 487 // Universal tasks (documentation, research synthesis, lifestyle, notes, recipes,
488. 488 // explanations) where content was inspected, written, or substantively answered
489. 489 // are NOT falsely blocked on non-existent bash commands.
490. 490 }
491. 491
492. 492 // A re-run is owed only when a check was asked for: by the
493. 493 // reading, which decides what completion requires, or by the
494. 494 // request's own words. An edit alone does not owe one — "add a
495. 495 // subtract function" asked for the function — and demanding a
496. 496 // check the surface could not run (a shell needing an approver
497. 497 // nobody could be) looped a live turn until the block cap.
498. 498 let check_owed = outcome.is_none_or(|spec| {
499. 499 spec.stop == vak_intent::StopProfile::Verification || spec.requires_execution()
500. 500 }) || demands_code
501. 501 || Self::demands_verification(prompt);
502. 502 if verification_stale
503. 503 && check_owed
504. 504 && (receipts.code_files_modified > 0 || demands_code)
505. 505 {
506. 506 return Some(BlockReason::VerificationStale);
507. 507 }
508. 508 }
509. 509 None
510. 510 }
511. 511
512. 512 /// Returns Some(reason) when completion should be blocked.
513. 513 pub fn evaluate(
514. 514 &self,
515. 515 prompt: &str,
516. 516 final_text: &str,
517. 517 bash_calls_this_run: u32,
518. 518 ) -> Option<BlockReason> {
519. 519 let receipts = ReceiptSummary {
520. 520 substantive_bash_calls: bash_calls_this_run,
521. 521 successful_tool_calls: if bash_calls_this_run > 0 {
522. 522 bash_calls_this_run
523. 523 } else {
524. 524 0
525. 525 },
526. 526 ..Default::default()
527. 527 };
528. 528 self.evaluate_receipts(prompt, final_text, None, &receipts, false)
529. 529 }
530. 530
531. 531 pub fn evaluate_with_state(
532. 532 &self,
533. 533 prompt: &str,
534. 534 final_text: &str,
535. 535 bash_calls_this_run: u32,
536. 536 verification_stale: bool,
537. 537 ) -> Option<BlockReason> {
538. 538 let receipts = ReceiptSummary {
539. 539 substantive_bash_calls: bash_calls_this_run,
540. 540 code_files_modified: if verification_stale { 1 } else { 0 },
541. 541 files_modified: if verification_stale { 1 } else { 0 },
542. 542 successful_tool_calls: if bash_calls_this_run > 0 {
543. 543 bash_calls_this_run
544. 544 } else {
545. 545 0
546. 546 },
547. 547 ..Default::default()
548. 548 };
549. 549 self.evaluate_receipts(prompt, final_text, None, &receipts, verification_stale)
550. 550 }
551. 551}
552. 552
553. 553/// Checks whether a file path points to an executable, compilable, or script source file
554. 554/// (as opposed to documentation, notes, recipes, data, or content assets).
555. 555pub fn is_code_path(path: &str) -> bool {
556. 556 let p = std::path::Path::new(path);
557. 557 match p
558. 558 .extension()
559. 559 .and_then(|ext| ext.to_str())
560. 560 .map(|ext| ext.to_ascii_lowercase())
561. 561 {
562. 562 Some(ext) => matches!(
563. 563 ext.as_str(),
564. 564 "rs" | "py"
565. 565 | "js"
566. 566 | "mjs"
567. 567 | "cjs"
568. 568 | "ts"
569. 569 | "tsx"
570. 570 | "jsx"
571. 571 | "c"
572. 572 | "cpp"
573. 573 | "cc"
574. 574 | "cxx"
575. 575 | "h"
576. 576 | "hpp"
577. 577 | "go"
578. 578 | "java"
579. 579 | "kt"
580. 580 | "kts"
581. 581 | "rb"
582. 582 | "php"
583. 583 | "swift"
584. 584 | "scala"
585. 585 | "sh"
586. 586 | "bash"
587. 587 | "zsh"
588. 588 | "fish"
589. 589 | "ps1"
590. 590 | "bat"
591. 591 | "cmd"
592. 592 | "lua"
593. 593 | "pl"
594. 594 | "pm"
595. 595 | "r"
596. 596 | "jl"
597. 597 | "dart"
598. 598 | "zig"
599. 599 | "nim"
600. 600 | "sql"
601. 601 ),
602. 602 None => false,
603. 603 }
604. 604}
605. 605
606. 606/// Checks if a shell command is substantive rather than a dummy echo/no-op evasion.
607. 607pub fn is_substantive_command(command: &str) -> bool {
608. 608 let trimmed = command.trim();
609. 609 if trimmed.is_empty() {
610. 610 return false;
611. 611 }
612. 612 // If it writes to a file or pipes to another command, it has side effects or processing
613. 613 if trimmed.contains('>') || trimmed.contains('|') {
614. 614 return true;
615. 615 }
616. 616 // Check if the command line is an explicit evasion claiming verification without doing work
617. 617 let lower = trimmed.to_ascii_lowercase();
618. 618 if (lower.starts_with("echo ") || lower.starts_with("printf "))
619. 619 && (lower.contains("verification")
620. 620 || lower.contains("shell execution path is functional")
621. 621 || lower.contains("verified")
622. 622 || lower.contains("dummy"))
623. 623 {
624. 624 return false;
625. 625 }
626. 626 let first_word = trimmed
627. 627 .split_whitespace()
628. 628 .next()
629. 629 .unwrap_or("")
630. 630 .trim_start_matches("./");
631. 631 let is_noop = matches!(first_word, ":" | "true" | "false" | "exit");
632. 632 !is_noop
633. 633}
634. 634
635. 635#[cfg(test)]
636. 636mod tests {
637. 637 use super::*;
638. 638
639. 639 #[test]
640. 640 fn truncated_shapes_block_normal_prose_passes() {
641. 641 let p = StopPolicy::default();
642. 642 assert!(
643. 643 p.evaluate("", "Working on it:\n- fix parser\n- then", 1)
644. 644 .is_none()
645. 645 || true
646. 646 ); // sanity no-op to keep structure
647. 647
648. 648 // trailing colon line
649. 649 assert!(matches!(
650. 650 p.evaluate("", "Let me check the config:", 3),
651. 651 Some(BlockReason::TruncatedPlan(_))
652. 652 ));
653. 653 // unclosed fence
654. 654 assert!(matches!(
655. 655 p.evaluate("", "here is the patch:\n```rust\nfn a() {}", 3),
656. 656 Some(BlockReason::TruncatedPlan(_))
657. 657 ));
658. 658 // plan-marker final line without terminal punctuation
659. 659 assert!(matches!(
660. 660 p.evaluate(
661. 661 "",
662. 662 "done with part one.\nNow I'll write the store module",
663. 663 3
664. 664 ),
665. 665 Some(BlockReason::TruncatedPlan(_))
666. 666 ));
667. 667 // normal summary passes
668. 668 assert_eq!(
669. 669 p.evaluate("", "All done. Tests pass.\nSummary:\n- added x", 3),
670. 670 None
671. 671 );
672. 672 assert_eq!(
673. 673 p.evaluate("", "Fixed both issues. cargo test green.", 3),
674. 674 None
675. 675 );
676. 676 // headings ending in colon are fine (e.g. 'Summary:')
677. 677 assert_eq!(p.evaluate("", "Results\n# Summary:", 3), None);
678. 678 }
679. 679
680. 680 #[test]
681. 681 fn verify_gate_needs_demand_and_zero_bash() {
682. 682 let p = StopPolicy::default();
683. 683 let prompt = "Create fizzbuzz.py and run it to prove that it works.";
684. 684 assert!(matches!(
685. 685 p.evaluate(prompt, "Created the file.", 0),
686. 686 Some(BlockReason::VerificationMissing)
687. 687 ));
688. 688 assert_eq!(p.evaluate(prompt, "Created the file.", 2), None);
689. 689 // no demand -> never blocks
690. 690 assert_eq!(p.evaluate("Write a haiku about sand.", "Done.", 0), None);
691. 691 }
692. 692
693. 693 #[test]
694. 694 fn verify_gate_recognizes_explicit_run_commands() {
695. 695 let p = StopPolicy::default();
696. 696 assert!(matches!(
697. 697 p.evaluate(
698. 698 "Read README.md, implement the change, then run python3 test_app.py.",
699. 699 "I need more details.",
700. 700 0
701. 701 ),
702. 702 Some(BlockReason::VerificationMissing)
703. 703 ));
704. 704 }
705. 705
706. 706 #[test]
707. 707 fn explicit_until_done_request_requires_user_release() {
708. 708 let p = StopPolicy::default();
709. 709 assert!(matches!(
710. 710 p.evaluate(
711. 711 "Keep improving the project until I say done.",
712. 712 "Improved it.",
713. 713 1
714. 714 ),
715. 715 Some(BlockReason::UserCompletionRequired)
716. 716 ));
717. 717 assert!(StopPolicy::is_done_message("done"));
718. 718 assert!(!StopPolicy::is_done_message(
719. 719 "done, and here is the summary"
720. 720 ));
721. 721 }
722. 722
723. 723 /// An edit the reading did not hold to a check owes none: authoring a
724. 724 /// function into a code file is done when the function is there.
725. 725 #[test]
726. 726 fn an_edit_owes_no_check_the_reading_did_not_ask_for() {
727. 727 let p = StopPolicy::default();
728. 728 let authoring = vak_intent::OutcomeSpec::from_reading(
729. 729 "explain what calc.py does, then add a subtract function to it",
730. 730 &vak_intent::Reading {
731. 731 act: vak_intent::Act::Author,
732. 732 ..vak_intent::Reading::general()
733. 733 },
734. 734 4,
735. 735 );
736. 736 let edited = ReceiptSummary {
737. 737 total_tool_calls: 1,
738. 738 successful_tool_calls: 1,
739. 739 files_modified: 1,
740. 740 code_files_modified: 1,
741. 741 ..Default::default()
742. 742 };
743. 743 assert_eq!(
744. 744 p.evaluate_receipts(
745. 745 "explain what calc.py does, then add a subtract function to it",
746. 746 "calc.py now defines add and subtract.",
747. 747 Some(&authoring),
748. 748 &edited,
749. 749 true,
750. 750 ),
751. 751 None
752. 752 );
753. 753 // A fix is held to a check: it modifies code the reading expects to
754. 754 // be proven.
755. 755 let fixing = vak_intent::OutcomeSpec::from_reading(
756. 756 "fix the off-by-one in calc.py",
757. 757 &vak_intent::Reading {
758. 758 act: vak_intent::Act::Modify,
759. 759 ..vak_intent::Reading::general()
760. 760 },
761. 761 4,
762. 762 );
763. 763 let fixed = ReceiptSummary {
764. 764 substantive_bash_calls: 1,
765. 765 ..edited
766. 766 };
767. 767 assert_eq!(
768. 768 p.evaluate_receipts(
769. 769 "fix the off-by-one in calc.py",
770. 770 "Fixed.",
771. 771 Some(&fixing),
772. 772 &fixed,
773. 773 true,
774. 774 ),
775. 775 Some(BlockReason::VerificationStale)
776. 776 );
777. 777 }
778. 778
779. 779 #[test]
780. 780 fn stale_verification_blocks_after_a_file_change() {
781. 781 let p = StopPolicy::default();
782. 782 assert_eq!(
783. 783 p.evaluate_with_state("implement it and run the tests", "Done.", 1, true),
784. 784 Some(BlockReason::VerificationStale)
785. 785 );
786. 786 assert_eq!(
787. 787 p.evaluate_with_state("implement it and run the tests", "Done.", 1, false),
788. 788 None
789. 789 );
790. 790 }
791. 791
792. 792 #[test]
793. 793 fn empty_final_text_is_left_alone() {
794. 794 let p = StopPolicy::default();
795. 795 assert_eq!(p.evaluate("run the tests", "", 0), None);
796. 796 }
797. 797
798. 798 #[test]
799. 799 fn test_substantive_command_detection() {
800. 800 assert!(!is_substantive_command(
801. 801 "echo \"Verification successful: Shell execution path is functional.\""
802. 802 ));
803. 803 assert!(!is_substantive_command("echo 'verification passed'"));
804. 804 assert!(!is_substantive_command("printf 'verified\\n'"));
805. 805 assert!(!is_substantive_command("true"));
806. 806 assert!(!is_substantive_command(":"));
807. 807 assert!(!is_substantive_command("exit 0"));
808. 808 assert!(!is_substantive_command(""));
809. 809
810. 810 assert!(is_substantive_command("echo ran"));
811. 811 assert!(is_substantive_command("echo 'hello'"));
812. 812 assert!(is_substantive_command("echo 'hello' > index.html"));
813. 813 assert!(is_substantive_command("echo 'hi' | wc -l"));
814. 814 assert!(is_substantive_command("python3 -m unittest"));
815. 815 assert!(is_substantive_command("cargo test"));
816. 816 assert!(is_substantive_command("npm start"));
817. 817 assert!(is_substantive_command("node server.js"));
818. 818 }
819. 819
820. 820 #[test]
821. 821 fn test_sandbox_demands_verification() {
822. 822 let p = StopPolicy::default();
823. 823 let prompt =
824. 824 "make in using react with beautifull design and run them in sandbox and show me";
825. 825 assert!(matches!(
826. 826 p.evaluate(prompt, "Here is the code in a block.", 0),
827. 827 Some(BlockReason::VerificationMissing)
828. 828 ));
829. 829 }
830. 830
831. 831 #[test]
832. 832 fn test_outcome_intent_requires_execution_receipt() {
833. 833 let p = StopPolicy::default();
834. 834 let mut reading = vak_intent::Reading::general();
835. 835 reading.act = vak_intent::Act::Modify;
836. 836 let spec = vak_intent::OutcomeSpec::from_reading("create an svg animation", &reading, 1);
837. 837 assert!(spec.requires_execution());
838. 838
839. 839 // 0 receipts -> blocked
840. 840 let empty_receipts = ReceiptSummary::default();
841. 841 let blocked = p.evaluate_receipts(
842. 842 "create an svg animation",
843. 843 "Here is your svg:\n```xml\n<svg/>\n```",
844. 844 Some(&spec),
845. 845 &empty_receipts,
846. 846 false,
847. 847 );
848. 848 assert!(matches!(
849. 849 blocked,
850. 850 Some(BlockReason::ExecutionReceiptMissing { .. })
851. 851 ));
852. 852
853. 853 // with substantive bash receipt -> allowed
854. 854 let with_bash = ReceiptSummary {
855. 855 substantive_bash_calls: 1,
856. 856 successful_tool_calls: 1,
857. 857 ..Default::default()
858. 858 };
859. 859 assert_eq!(
860. 860 p.evaluate_receipts(
861. 861 "create an svg animation",
862. 862 "Created and verified.",
863. 863 Some(&spec),
864. 864 &with_bash,
865. 865 false
866. 866 ),
867. 867 None
868. 868 );
869. 869
870. 870 // Work done through an integration or a delegated worker is
871. 871 // execution too: the stop gate must not demand a shell receipt for
872. 872 // an email an MCP server sent.
873. 873 let external = ReceiptSummary {
874. 874 external_effects: 1,
875. 875 successful_tool_calls: 1,
876. 876 ..Default::default()
877. 877 };
878. 878 assert!(external.has_execution_receipt());
879. 879 assert_eq!(
880. 880 p.evaluate_receipts(
881. 881 "create an svg animation",
882. 882 "Created and verified.",
883. 883 Some(&spec),
884. 884 &external,
885. 885 false
886. 886 ),
887. 887 None
888. 888 );
889. 889
890. 890 // with file write receipt -> allowed
891. 891 let with_file = ReceiptSummary {
892. 892 files_modified: 1,
893. 893 successful_tool_calls: 1,
894. 894 ..Default::default()
895. 895 };
896. 896 assert_eq!(
897. 897 p.evaluate_receipts(
898. 898 "create an svg animation",
899. 899 "Created file.",
900. 900 Some(&spec),
901. 901 &with_file,
902. 902 false
903. 903 ),
904. 904 None
905. 905 );
906. 906 }
907. 907
908. 908 #[test]
909. 909 fn capped_continuation_counts_prior_saved_file_only_after_successful_inspection() {
910. 910 let policy = StopPolicy::default();
911. 911 let mut reading = vak_intent::Reading::general();
912. 912 reading.act = vak_intent::Act::Modify;
913. 913 let spec = vak_intent::OutcomeSpec::from_reading("create report.csv", &reading, 1);
914. 914 let prompt = "Continue the most recent unfinished task";
915. 915 let answer = "The saved report contains 60 minutes.";
916. 916 let prior_only = ReceiptSummary {
917. 917 continued_saved_file: true,
918. 918 read_or_inspected: 1,
919. 919 ..Default::default()
920. 920 };
921. 921 assert!(matches!(
922. 922 policy.evaluate_receipts(prompt, answer, Some(&spec), &prior_only, false),
923. 923 Some(BlockReason::ExecutionReceiptMissing { .. })
924. 924 ));
925. 925 let inspected = ReceiptSummary {
926. 926 successful_inspections: 1,
927. 927 ..prior_only
928. 928 };
929. 929 assert_eq!(
930. 930 policy.evaluate_receipts(prompt, answer, Some(&spec), &inspected, false),
931. 931 None
932. 932 );
933. 933 }
934. 934
935. 935 #[test]
936. 936 fn test_outcome_conversational_allows_prose_completion() {
937. 937 let p = StopPolicy::default();
938. 938 let mut reading = vak_intent::Reading::general();
939. 939 reading.act = vak_intent::Act::Answer;
940. 940 let spec = vak_intent::OutcomeSpec::from_reading("what is rust?", &reading, 1);
941. 941 assert!(!spec.requires_execution());
942. 942 assert!(!spec.requires_tool());
943. 943
944. 944 let receipts = ReceiptSummary::default();
945. 945 assert_eq!(
946. 946 p.evaluate_receipts(
947. 947 "what is rust?",
948. 948 "Rust is a systems programming language.",
949. 949 Some(&spec),
950. 950 &receipts,
951. 951 false
952. 952 ),
953. 953 None
954. 954 );
955. 955 }
956. 956
957. 957 #[test]
958. 958 fn test_claims_execution_unexecuted_blocks_even_with_outcome_spec() {
959. 959 let p = StopPolicy::default();
960. 960 let reading = vak_intent::Reading::general();
961. 961 let spec = vak_intent::OutcomeSpec::from_reading("can you run it and show", &reading, 1);
962. 962 let receipts = ReceiptSummary::default();
963. 963
964. 964 let blocked = p.evaluate_receipts(
965. 965 "can you run it and show",
966. 966 "I will use the bash tool to execute a Python script:\n```bash\npython3 script.py\n```",
967. 967 Some(&spec),
968. 968 &receipts,
969. 969 false,
970. 970 );
971. 971 assert_eq!(blocked, Some(BlockReason::VerificationMissing));
972. 972 }
973. 973
974. 974 #[test]
975. 975 fn test_unresolved_tool_failure_blocks_unless_reported() {
976. 976 let p = StopPolicy::default();
977. 977 let receipts = ReceiptSummary {
978. 978 total_tool_calls: 1,
979. 979 failed_tool_calls: 1,
980. 980 unresolved_error: Some(("bash".into(), "exit code 1: compile error".into())),
981. 981 ..Default::default()
982. 982 };
983. 983
984. 984 // Model hallucinates success without reporting error -> blocked
985. 985 let blocked = p.evaluate_receipts(
986. 986 "build it",
987. 987 "All done! Everything succeeded.",
988. 988 None,
989. 989 &receipts,
990. 990 false,
991. 991 );
992. 992 assert!(matches!(
993. 993 blocked,
994. 994 Some(BlockReason::UnresolvedToolFailure { .. })
995. 995 ));
996. 996
997. 997 // Model reports the error/blocker -> allowed
998. 998 let reported = p.evaluate_receipts(
999. 999 "build it",
1000. 1000 "The build failed with exit code 1: compile error. Cannot proceed without missing dependency.",