Vakyartha documentation

274 return true;
275 }
276 if p.contains("sandbox")
277 && ["run", "show", "test", "build", "execute", "serve", "start"]
278 .iter()
279 .any(|action| p.contains(action))
280 {
281 return true;
282 }
283 if p.contains("cargo test")
284 || p.contains("pytest")
285 || p.contains("npm test")
286 || p.contains("go test")
287 || p.contains("python -m unittest")
288 {
289 return true;
290 }
291 p.contains("run ")
292 && [
293 "test", "tests", "command", "script", "check", "app", "code", "python", "cargo",
294 "binary",
295 ]
296 .iter()
297 .any(|word| p.contains(word))
298 }
299
300 /// True when the prompt asks for any verification (code or universal/content).
301 pub fn demands_verification(prompt: &str) -> bool {
302 if Self::demands_code_execution(prompt) {
303 return true;
304 }
305 let stripped = if let Some(idx) = prompt.find("[Scheduled-run context:") {
306 &prompt[..idx]
307 } else {
308 prompt
309 };
310 let p = stripped.to_ascii_lowercase();
311 const VERIFY_MARKERS: [&str; 7] = [
312 "verify that",
313 "verify the",
314 "double check",
315 "double-check",
316 "make sure that",
317 "check that",
318 "verify whether",
319 ];
320 VERIFY_MARKERS.iter().any(|m| p.contains(m))
321 }
322
323 /// True when the assistant response claims execution or emits shell scripts without tool calls having run.
324 pub fn claims_execution_unexecuted(final_text: &str) -> bool {
325 let lower = final_text.to_ascii_lowercase();
326 let markers = [
327 "use the bash tool",
328 "use the `bash` tool",
329 "using the bash tool",
330 "using the `bash` tool",
331 "run the python script",
332 "running the python script",
333 "execute the python script",
334 "executing the python script",
335 "execute in the sandbox",
336 "running in the sandbox",
337 "run in the sandbox",
338 "execute in sandbox",
339 "running in sandbox",
340 ];
341 if markers.iter().any(|m| lower.contains(m)) {
342 return true;
343 }
344 if (lower.contains("```bash") || lower.contains("```sh"))
345 && (lower.contains(".vak/scratch")
346 || lower.contains("python3 ")
347 || lower.contains("node ")
348 || lower.contains("cargo "))
349 {
350 return true;
351 }
352 false
353 }
354
355 /// Intent- and receipt-driven evaluation of completion validity.
356 pub fn evaluate_receipts(
357 &self,
358 prompt: &str,
359 final_text: &str,
360 outcome: Option<&vak_intent::OutcomeSpec>,
361 receipts: &ReceiptSummary,
362 verification_stale: bool,
363 ) -> Option<BlockReason> {
364 if final_text.trim().is_empty() {
365 return None;
366 }
367 if Self::requires_user_completion(prompt) {
368 return Some(BlockReason::UserCompletionRequired);
369 }
370 if self.marker_gate
371 && let Some(r) = Self::truncated_plan(final_text)
372 {
373 return Some(r);
374 }
375
376 // If a tool failed and hasn't been repaired or reported in text, block.
377 if let Some((tool, err)) = &receipts.unresolved_error
378 && !reports_blocker(final_text, tool, err)
379 {
380 return Some(BlockReason::UnresolvedToolFailure {
381 tool: tool.clone(),
382 error: err.clone(),
383 });
384 }
385
386 // Intent-driven gate: when outcome specification is available. The
387 // engagement's stop profile (docs/design/47-commitment-kernel.md)
388 // decides first; the typed acts refine it.
389 if let Some(spec) = outcome {
390 use vak_intent::StopProfile;
391 // `Verification`: a checkable result was demanded. An execution
392 // receipt is required, and it must not be stale.
393 if spec.stop == StopProfile::Verification {
394 if !receipts.has_execution_receipt() {
395 return Some(BlockReason::ExecutionReceiptMissing {
396 act: spec.deliverable_act().unwrap_or("verification").to_string(),
397 hint: "run the check that proves this is done".into(),
398 });
399 }
400 if verification_stale && receipts.code_files_modified > 0 {
401 return Some(BlockReason::VerificationStale);
402 }
403 }
404 if spec.stop == StopProfile::Effect || spec.requires_execution() {
405 if !receipts.has_execution_receipt() {
406 let act = spec.deliverable_act().unwrap_or("execution").to_string();
407 return Some(BlockReason::ExecutionReceiptMissing {
408 act,
409 hint: "run code, build, test, or modify files".into(),
410 });
411 }
412 if self.verify_gate
413 && verification_stale
414 && (spec.requires_execution() || Self::demands_verification(prompt))
415 && (receipts.code_files_modified > 0 || Self::demands_code_execution(prompt))
416 {
417 return Some(BlockReason::VerificationStale);
418 }
419 // `Inspection` on its own gates nothing: "something was looked
420 // at" includes the material the request carried (an attachment,
421 // pasted text), which leaves no receipt. Only a `locate` act
422 // demands an inspection receipt.
423 } else if spec.requires_inspection() {
424 let direct_substantive = final_text.trim().len() >= 80
425 && !Self::demands_code_execution(prompt)
426 && !Self::claims_execution_unexecuted(final_text);
427 if !receipts.has_inspection_receipt() && !direct_substantive {
428 let act = spec.deliverable_act().unwrap_or("inspection").to_string();
429 return Some(BlockReason::ExecutionReceiptMissing {
430 act,
431 hint: "read, search, inspect files or data".into(),
432 });
433 }
434 } else if spec.requires_tool() && !receipts.has_any_receipt() {
435 let direct_substantive = final_text.trim().len() >= 80
436 && !Self::demands_code_execution(prompt)
437 && !Self::claims_execution_unexecuted(final_text);
438 if !direct_substantive {
439 return Some(BlockReason::ExecutionReceiptMissing {
440 act: "tool execution".into(),
441 hint: "execute relevant tools".into(),
442 });
443 }
444 }
445 }
446
447 // Verification and execution gate: runs whenever verify_gate is enabled.
448 if self.verify_gate {
449 let demands_code = Self::demands_code_execution(prompt);
450 let claims_exec = Self::claims_execution_unexecuted(final_text);
451
452 if claims_exec {
453 return Some(BlockReason::VerificationMissing);
454 }
455
456 if demands_code && receipts.substantive_bash_calls == 0 {
457 return Some(BlockReason::VerificationMissing);
458 }
459
460 if Self::demands_verification(prompt) && receipts.substantive_bash_calls == 0 {
461 // If code files were touched, verification commands are required.
462 if receipts.code_files_modified > 0 {
463 return Some(BlockReason::VerificationMissing);
464 }
465 // If no tools were called and the text is not a substantive direct answer:
466 let direct_substantive = final_text.trim().len() >= 80;
467 if !receipts.has_any_receipt() && !direct_substantive {
468 return Some(BlockReason::VerificationMissing);
469 }
470 // If an execution deliverable was required or a specific file target was requested,
471 // but no files were modified or inspected:
472 let lower_p = prompt.to_ascii_lowercase();
473 let mentions_file_target = lower_p.contains(".md")
474 || lower_p.contains(".txt")
475 || lower_p.contains(".json")
476 || lower_p.contains(".csv")
477 || lower_p.contains("into ")
478 || lower_p.contains("in file")
479 || lower_p.contains("in the file");
480 if (outcome.map(|s| s.requires_execution()).unwrap_or(false)
481 || mentions_file_target)
482 && receipts.files_modified == 0
483 && receipts.read_or_inspected == 0
484 {
485 return Some(BlockReason::VerificationMissing);
486 }
487 // Universal tasks (documentation, research synthesis, lifestyle, notes, recipes,
488 // explanations) where content was inspected, written, or substantively answered
489 // are NOT falsely blocked on non-existent bash commands.
490 }
491
492 // A re-run is owed only when a check was asked for: by the
493 // reading, which decides what completion requires, or by the
494 // request's own words. An edit alone does not owe one — "add a
495 // subtract function" asked for the function — and demanding a
496 // check the surface could not run (a shell needing an approver
497 // nobody could be) looped a live turn until the block cap.
498 let check_owed = outcome.is_none_or(|spec| {
499 spec.stop == vak_intent::StopProfile::Verification || spec.requires_execution()
500 }) || demands_code
501 || Self::demands_verification(prompt);
502 if verification_stale
503 && check_owed
504 && (receipts.code_files_modified > 0 || demands_code)
505 {
506 return Some(BlockReason::VerificationStale);
507 }
508 }
509 None
510 }
511
512 /// Returns Some(reason) when completion should be blocked.
513 pub fn evaluate(
514 &self,
515 prompt: &str,
516 final_text: &str,
517 bash_calls_this_run: u32,
518 ) -> Option<BlockReason> {
519 let receipts = ReceiptSummary {
520 substantive_bash_calls: bash_calls_this_run,
521 successful_tool_calls: if bash_calls_this_run > 0 {
522 bash_calls_this_run
523 } else {
524 0
525 },
526 ..Default::default()
527 };
528 self.evaluate_receipts(prompt, final_text, None, &receipts, false)
529 }
530
531 pub fn evaluate_with_state(
532 &self,
533 prompt: &str,
534 final_text: &str,
535 bash_calls_this_run: u32,
536 verification_stale: bool,
537 ) -> Option<BlockReason> {
538 let receipts = ReceiptSummary {
539 substantive_bash_calls: bash_calls_this_run,
540 code_files_modified: if verification_stale { 1 } else { 0 },
541 files_modified: if verification_stale { 1 } else { 0 },
542 successful_tool_calls: if bash_calls_this_run > 0 {
543 bash_calls_this_run
544 } else {
545 0
546 },
547 ..Default::default()
548 };
549 self.evaluate_receipts(prompt, final_text, None, &receipts, verification_stale)
550 }
551}
552
553/// Checks whether a file path points to an executable, compilable, or script source file
554/// (as opposed to documentation, notes, recipes, data, or content assets).
555pub fn is_code_path(path: &str) -> bool {
556 let p = std::path::Path::new(path);
557 match p
558 .extension()
559 .and_then(|ext| ext.to_str())
560 .map(|ext| ext.to_ascii_lowercase())
561 {
562 Some(ext) => matches!(
563 ext.as_str(),
564 "rs" | "py"
565 | "js"
566 | "mjs"
567 | "cjs"
568 | "ts"
569 | "tsx"
570 | "jsx"
571 | "c"
572 | "cpp"
573 | "cc"
574 | "cxx"
575 | "h"
576 | "hpp"
577 | "go"
578 | "java"
579 | "kt"
580 | "kts"
581 | "rb"
582 | "php"
583 | "swift"
584 | "scala"
585 | "sh"
586 | "bash"
587 | "zsh"
588 | "fish"
589 | "ps1"
590 | "bat"
591 | "cmd"
592 | "lua"
593 | "pl"
594 | "pm"
595 | "r"
596 | "jl"
597 | "dart"
598 | "zig"
599 | "nim"
600 | "sql"
601 ),
602 None => false,
603 }
604}
605
606/// Checks if a shell command is substantive rather than a dummy echo/no-op evasion.
607pub fn is_substantive_command(command: &str) -> bool {
608 let trimmed = command.trim();
609 if trimmed.is_empty() {
610 return false;
611 }
612 // If it writes to a file or pipes to another command, it has side effects or processing
613 if trimmed.contains('>') || trimmed.contains('|') {
614 return true;
615 }
616 // Check if the command line is an explicit evasion claiming verification without doing work
617 let lower = trimmed.to_ascii_lowercase();
618 if (lower.starts_with("echo ") || lower.starts_with("printf "))
619 && (lower.contains("verification")
620 || lower.contains("shell execution path is functional")
621 || lower.contains("verified")
622 || lower.contains("dummy"))
623 {
624 return false;
625 }
626 let first_word = trimmed
627 .split_whitespace()
628 .next()
629 .unwrap_or("")
630 .trim_start_matches("./");
631 let is_noop = matches!(first_word, ":" | "true" | "false" | "exit");
632 !is_noop
633}
634
635#[cfg(test)]
636mod tests {
637 use super::*;
638
639 #[test]
640 fn truncated_shapes_block_normal_prose_passes() {
641 let p = StopPolicy::default();
642 assert!(
643 p.evaluate("", "Working on it:\n- fix parser\n- then", 1)
644 .is_none()
645 || true
646 ); // sanity no-op to keep structure
647
648 // trailing colon line
649 assert!(matches!(
650 p.evaluate("", "Let me check the config:", 3),
651 Some(BlockReason::TruncatedPlan(_))
652 ));
653 // unclosed fence
654 assert!(matches!(
655 p.evaluate("", "here is the patch:\n```rust\nfn a() {}", 3),
656 Some(BlockReason::TruncatedPlan(_))
657 ));
658 // plan-marker final line without terminal punctuation
659 assert!(matches!(
660 p.evaluate(
661 "",
662 "done with part one.\nNow I'll write the store module",
663 3
664 ),
665 Some(BlockReason::TruncatedPlan(_))
666 ));
667 // normal summary passes
668 assert_eq!(
669 p.evaluate("", "All done. Tests pass.\nSummary:\n- added x", 3),
670 None
671 );
672 assert_eq!(
673 p.evaluate("", "Fixed both issues. cargo test green.", 3),
674 None
675 );
676 // headings ending in colon are fine (e.g. 'Summary:')
677 assert_eq!(p.evaluate("", "Results\n# Summary:", 3), None);
678 }
679
680 #[test]
681 fn verify_gate_needs_demand_and_zero_bash() {
682 let p = StopPolicy::default();
683 let prompt = "Create fizzbuzz.py and run it to prove that it works.";
684 assert!(matches!(
685 p.evaluate(prompt, "Created the file.", 0),
686 Some(BlockReason::VerificationMissing)
687 ));
688 assert_eq!(p.evaluate(prompt, "Created the file.", 2), None);
689 // no demand -> never blocks
690 assert_eq!(p.evaluate("Write a haiku about sand.", "Done.", 0), None);
691 }
692
693 #[test]
694 fn verify_gate_recognizes_explicit_run_commands() {
695 let p = StopPolicy::default();
696 assert!(matches!(
697 p.evaluate(
698 "Read README.md, implement the change, then run python3 test_app.py.",
699 "I need more details.",
700 0
701 ),
702 Some(BlockReason::VerificationMissing)
703 ));
704 }
705
706 #[test]
707 fn explicit_until_done_request_requires_user_release() {
708 let p = StopPolicy::default();
709 assert!(matches!(
710 p.evaluate(
711 "Keep improving the project until I say done.",
712 "Improved it.",
713 1
714 ),
715 Some(BlockReason::UserCompletionRequired)
716 ));
717 assert!(StopPolicy::is_done_message("done"));
718 assert!(!StopPolicy::is_done_message(
719 "done, and here is the summary"
720 ));
721 }
722
723 /// An edit the reading did not hold to a check owes none: authoring a
724 /// function into a code file is done when the function is there.
725 #[test]
726 fn an_edit_owes_no_check_the_reading_did_not_ask_for() {
727 let p = StopPolicy::default();
728 let authoring = vak_intent::OutcomeSpec::from_reading(
729 "explain what calc.py does, then add a subtract function to it",
730 &vak_intent::Reading {
731 act: vak_intent::Act::Author,
732 ..vak_intent::Reading::general()
733 },
734 4,
735 );
736 let edited = ReceiptSummary {
737 total_tool_calls: 1,
738 successful_tool_calls: 1,
739 files_modified: 1,
740 code_files_modified: 1,
741 ..Default::default()
742 };
743 assert_eq!(
744 p.evaluate_receipts(
745 "explain what calc.py does, then add a subtract function to it",
746 "calc.py now defines add and subtract.",
747 Some(&authoring),
748 &edited,
749 true,
750 ),
751 None
752 );
753 // A fix is held to a check: it modifies code the reading expects to
754 // be proven.
755 let fixing = vak_intent::OutcomeSpec::from_reading(
756 "fix the off-by-one in calc.py",
757 &vak_intent::Reading {
758 act: vak_intent::Act::Modify,
759 ..vak_intent::Reading::general()
760 },
761 4,
762 );
763 let fixed = ReceiptSummary {
764 substantive_bash_calls: 1,
765 ..edited
766 };
767 assert_eq!(
768 p.evaluate_receipts(
769 "fix the off-by-one in calc.py",
770 "Fixed.",
771 Some(&fixing),
772 &fixed,
773 true,
774 ),
775 Some(BlockReason::VerificationStale)
776 );
777 }
778
779 #[test]
780 fn stale_verification_blocks_after_a_file_change() {
781 let p = StopPolicy::default();
782 assert_eq!(
783 p.evaluate_with_state("implement it and run the tests", "Done.", 1, true),
784 Some(BlockReason::VerificationStale)
785 );
786 assert_eq!(
787 p.evaluate_with_state("implement it and run the tests", "Done.", 1, false),
788 None
789 );
790 }
791
792 #[test]
793 fn empty_final_text_is_left_alone() {
794 let p = StopPolicy::default();
795 assert_eq!(p.evaluate("run the tests", "", 0), None);
796 }
797
798 #[test]
799 fn test_substantive_command_detection() {
800 assert!(!is_substantive_command(
801 "echo \"Verification successful: Shell execution path is functional.\""
802 ));
803 assert!(!is_substantive_command("echo 'verification passed'"));
804 assert!(!is_substantive_command("printf 'verified\\n'"));
805 assert!(!is_substantive_command("true"));
806 assert!(!is_substantive_command(":"));
807 assert!(!is_substantive_command("exit 0"));
808 assert!(!is_substantive_command(""));
809
810 assert!(is_substantive_command("echo ran"));
811 assert!(is_substantive_command("echo 'hello'"));
812 assert!(is_substantive_command("echo 'hello' > index.html"));
813 assert!(is_substantive_command("echo 'hi' | wc -l"));
814 assert!(is_substantive_command("python3 -m unittest"));
815 assert!(is_substantive_command("cargo test"));
816 assert!(is_substantive_command("npm start"));
817 assert!(is_substantive_command("node server.js"));
818 }
819
820 #[test]
821 fn test_sandbox_demands_verification() {
822 let p = StopPolicy::default();
823 let prompt =
824 "make in using react with beautifull design and run them in sandbox and show me";
825 assert!(matches!(
826 p.evaluate(prompt, "Here is the code in a block.", 0),
827 Some(BlockReason::VerificationMissing)
828 ));
829 }
830
831 #[test]
832 fn test_outcome_intent_requires_execution_receipt() {
833 let p = StopPolicy::default();
834 let mut reading = vak_intent::Reading::general();
835 reading.act = vak_intent::Act::Modify;
836 let spec = vak_intent::OutcomeSpec::from_reading("create an svg animation", &reading, 1);
837 assert!(spec.requires_execution());
838
839 // 0 receipts -> blocked
840 let empty_receipts = ReceiptSummary::default();
841 let blocked = p.evaluate_receipts(
842 "create an svg animation",
843 "Here is your svg:\n```xml\n<svg/>\n```",
844 Some(&spec),
845 &empty_receipts,
846 false,
847 );
848 assert!(matches!(
849 blocked,
850 Some(BlockReason::ExecutionReceiptMissing { .. })
851 ));
852
853 // with substantive bash receipt -> allowed
854 let with_bash = ReceiptSummary {
855 substantive_bash_calls: 1,
856 successful_tool_calls: 1,
857 ..Default::default()
858 };
859 assert_eq!(
860 p.evaluate_receipts(
861 "create an svg animation",
862 "Created and verified.",
863 Some(&spec),
864 &with_bash,
865 false
866 ),
867 None
868 );
869
870 // Work done through an integration or a delegated worker is
871 // execution too: the stop gate must not demand a shell receipt for
872 // an email an MCP server sent.
873 let external = ReceiptSummary {
874 external_effects: 1,
875 successful_tool_calls: 1,
876 ..Default::default()
877 };
878 assert!(external.has_execution_receipt());
879 assert_eq!(
880 p.evaluate_receipts(
881 "create an svg animation",
882 "Created and verified.",
883 Some(&spec),
884 &external,
885 false
886 ),
887 None
888 );
889
890 // with file write receipt -> allowed
891 let with_file = ReceiptSummary {
892 files_modified: 1,
893 successful_tool_calls: 1,
894 ..Default::default()
895 };
896 assert_eq!(
897 p.evaluate_receipts(
898 "create an svg animation",
899 "Created file.",
900 Some(&spec),
901 &with_file,
902 false
903 ),
904 None
905 );
906 }
907
908 #[test]
909 fn capped_continuation_counts_prior_saved_file_only_after_successful_inspection() {
910 let policy = StopPolicy::default();
911 let mut reading = vak_intent::Reading::general();
912 reading.act = vak_intent::Act::Modify;
913 let spec = vak_intent::OutcomeSpec::from_reading("create report.csv", &reading, 1);
914 let prompt = "Continue the most recent unfinished task";
915 let answer = "The saved report contains 60 minutes.";
916 let prior_only = ReceiptSummary {
917 continued_saved_file: true,
918 read_or_inspected: 1,
919 ..Default::default()
920 };
921 assert!(matches!(
922 policy.evaluate_receipts(prompt, answer, Some(&spec), &prior_only, false),
923 Some(BlockReason::ExecutionReceiptMissing { .. })
924 ));
925 let inspected = ReceiptSummary {
926 successful_inspections: 1,
927 ..prior_only
928 };
929 assert_eq!(
930 policy.evaluate_receipts(prompt, answer, Some(&spec), &inspected, false),
931 None
932 );
933 }
934
935 #[test]
936 fn test_outcome_conversational_allows_prose_completion() {
937 let p = StopPolicy::default();
938 let mut reading = vak_intent::Reading::general();
939 reading.act = vak_intent::Act::Answer;
940 let spec = vak_intent::OutcomeSpec::from_reading("what is rust?", &reading, 1);
941 assert!(!spec.requires_execution());
942 assert!(!spec.requires_tool());
943
944 let receipts = ReceiptSummary::default();
945 assert_eq!(
946 p.evaluate_receipts(
947 "what is rust?",
948 "Rust is a systems programming language.",
949 Some(&spec),
950 &receipts,
951 false
952 ),
953 None
954 );
955 }
956
957 #[test]
958 fn test_claims_execution_unexecuted_blocks_even_with_outcome_spec() {
959 let p = StopPolicy::default();
960 let reading = vak_intent::Reading::general();
961 let spec = vak_intent::OutcomeSpec::from_reading("can you run it and show", &reading, 1);
962 let receipts = ReceiptSummary::default();
963
964 let blocked = p.evaluate_receipts(
965 "can you run it and show",
966 "I will use the bash tool to execute a Python script:\n```bash\npython3 script.py\n```",
967 Some(&spec),
968 &receipts,
969 false,
970 );
971 assert_eq!(blocked, Some(BlockReason::VerificationMissing));
972 }
973
974 #[test]
975 fn test_unresolved_tool_failure_blocks_unless_reported() {
976 let p = StopPolicy::default();
977 let receipts = ReceiptSummary {
978 total_tool_calls: 1,
979 failed_tool_calls: 1,
980 unresolved_error: Some(("bash".into(), "exit code 1: compile error".into())),
981 ..Default::default()
982 };
983
984 // Model hallucinates success without reporting error -> blocked
985 let blocked = p.evaluate_receipts(
986 "build it",
987 "All done! Everything succeeded.",
988 None,
989 &receipts,
990 false,
991 );
992 assert!(matches!(
993 blocked,
994 Some(BlockReason::UnresolvedToolFailure { .. })
995 ));
996
997 // Model reports the error/blocker -> allowed
998 let reported = p.evaluate_receipts(
999 "build it",
1000 "The build failed with exit code 1: compile error. Cannot proceed without missing dependency.",
1001 None,
1002 &receipts,
1003 false,
1004 );
1005 assert_eq!(reported, None);
1006 }
1007
1008 #[test]
1009 fn test_direct_substantive_answer_not_blocked_for_inspection_spec() {
1010 let p = StopPolicy::default();
1011 let mut reading = vak_intent::Reading::general();
1012 reading.act = vak_intent::Act::Locate;
1013 let spec = vak_intent::OutcomeSpec::from_reading(
1014 "explain the architectural differences",
1015 &reading,
1016 1,
1017 );
1018 let receipts = ReceiptSummary::default();
1019
1020 // Substantive direct analysis without false tool claims or execution demands -> allowed
1021 let substantive_answer = "Optimistic locking assumes multiple transactions can complete without affecting each other. It verifies no other transaction has modified the data before committing. In contrast, pessimistic locking acquires locks immediately upon reading.";
1022 let blocked = p.evaluate_receipts(
1023 "explain the architectural differences",
1024 substantive_answer,
1025 Some(&spec),
1026 &receipts,
1027 false,
1028 );
1029 assert_eq!(blocked, None);
1030
1031 // Empty or non-substantive answer -> blocked
1032 let blocked_empty = p.evaluate_receipts(
1033 "explain the architectural differences",
1034 "Okay",
1035 Some(&spec),
1036 &receipts,
1037 false,
1038 );
1039 assert!(matches!(
1040 blocked_empty,
1041 Some(BlockReason::ExecutionReceiptMissing { .. })
1042 ));
1043 }
1044
1045 #[test]
1046 fn test_is_code_path_accurately_classifies_code_vs_doc_paths() {
1047 assert!(is_code_path("src/main.rs"));
1048 assert!(is_code_path("backend/app.py"));
1049 assert!(is_code_path("web/index.ts"));
1050 assert!(is_code_path("scripts/deploy.sh"));
1051
1052 assert!(!is_code_path("README.md"));
1053 assert!(!is_code_path("docs/architecture.md"));
1054 assert!(!is_code_path("recipes/sourdough.txt"));
1055 assert!(!is_code_path("data/analysis.csv"));
1056 assert!(!is_code_path("notes.org"));
1057 }
1058
1059 #[test]
1060 fn test_universal_doc_modification_with_verify_not_blocked_on_bash() {
1061 let p = StopPolicy::default();
1062 let prompt = "Update README.md to describe the release steps and verify that all links are formatted correctly.";
1063 let final_text = "Updated README.md with release steps and verified that the Markdown links match the repository structure.";
1064
1065 let receipts = ReceiptSummary {
1066 total_tool_calls: 2,
1067 successful_tool_calls: 2,
1068 files_modified: 1,
1069 code_files_modified: 0,
1070 doc_files_modified: 1,
1071 read_or_inspected: 1,
1072 ..Default::default()
1073 };
1074
1075 // Even though prompt says "verify that", since no code was modified and no code execution was demanded,
1076 // it must NOT block on non-existent bash commands or stale verification!
1077 let blocked = p.evaluate_receipts(prompt, final_text, None, &receipts, false);
1078 assert_eq!(blocked, None);
1079
1080 let blocked_stale = p.evaluate_receipts(prompt, final_text, None, &receipts, true);
1081 assert_eq!(blocked_stale, None);
1082 }
1083
1084 #[test]
1085 fn test_universal_research_and_lifestyle_with_verify_not_blocked() {
1086 let p = StopPolicy::default();
1087 let prompt =
1088 "Compare the top 3 pour-over drippers and verify that the brew ratios are accurate.";
1089 let final_text = "Here is a detailed comparison of Hario V60, Kalita Wave, and Chemex. All brew ratios are verified between 1:15 and 1:17 for balanced extraction across light and medium roasts.";
1090
1091 let receipts = ReceiptSummary {
1092 total_tool_calls: 1,
1093 successful_tool_calls: 1,
1094 read_or_inspected: 1,
1095 ..Default::default()
1096 };
1097
1098 let blocked = p.evaluate_receipts(prompt, final_text, None, &receipts, false);
1099 assert_eq!(blocked, None);
1100 }
1101
1102 #[test]
1103 fn test_code_modification_with_verify_blocked_without_execution() {
1104 let p = StopPolicy::default();
1105 let prompt =
1106 "Fix the off-by-one bug in quicksort.py and verify that the sort works correctly.";
1107 let final_text = "I fixed the index in quicksort.py.";
1108
1109 let receipts = ReceiptSummary {
1110 total_tool_calls: 1,
1111 successful_tool_calls: 1,
1112 files_modified: 1,
1113 code_files_modified: 1,
1114 doc_files_modified: 0,
1115 ..Default::default()
1116 };
1117
1118 // Code was modified and prompt asks to "verify that" -> must block with VerificationMissing
1119 let blocked = p.evaluate_receipts(prompt, final_text, None, &receipts, false);
1120 assert_eq!(blocked, Some(BlockReason::VerificationMissing));
1121
1122 // If code files were modified after test ran, stale verification blocks
1123 let with_bash = ReceiptSummary {
1124 total_tool_calls: 2,
1125 successful_tool_calls: 2,
1126 substantive_bash_calls: 1,
1127 files_modified: 1,
1128 code_files_modified: 1,
1129 ..Default::default()
1130 };
1131 let blocked_stale = p.evaluate_receipts(prompt, final_text, None, &with_bash, true);
1132 assert_eq!(blocked_stale, Some(BlockReason::VerificationStale));
1133 }
1134}
1135

274 return true;
275 }
276 if p.contains("sandbox")
277 && ["run", "show", "test", "build", "execute", "serve", "start"]
278 .iter()
279 .any(|action| p.contains(action))
280 {
281 return true;
282 }
283 if p.contains("cargo test")
284 || p.contains("pytest")
285 || p.contains("npm test")
286 || p.contains("go test")
287 || p.contains("python -m unittest")
288 {
289 return true;
290 }
291 p.contains("run ")
292 && [
293 "test", "tests", "command", "script", "check", "app", "code", "python", "cargo",
294 "binary",
295 ]
296 .iter()
297 .any(|word| p.contains(word))
298 }
299
300 /// True when the prompt asks for any verification (code or universal/content).
301 pub fn demands_verification(prompt: &str) -> bool {
302 if Self::demands_code_execution(prompt) {
303 return true;
304 }
305 let stripped = if let Some(idx) = prompt.find("[Scheduled-run context:") {
306 &prompt[..idx]
307 } else {
308 prompt
309 };
310 let p = stripped.to_ascii_lowercase();
311 const VERIFY_MARKERS: [&str; 7] = [
312 "verify that",
313 "verify the",
314 "double check",
315 "double-check",
316 "make sure that",
317 "check that",
318 "verify whether",
319 ];
320 VERIFY_MARKERS.iter().any(|m| p.contains(m))
321 }
322
323 /// True when the assistant response claims execution or emits shell scripts without tool calls having run.
324 pub fn claims_execution_unexecuted(final_text: &str) -> bool {
325 let lower = final_text.to_ascii_lowercase();
326 let markers = [
327 "use the bash tool",
328 "use the `bash` tool",
329 "using the bash tool",
330 "using the `bash` tool",
331 "run the python script",
332 "running the python script",
333 "execute the python script",
334 "executing the python script",
335 "execute in the sandbox",
336 "running in the sandbox",
337 "run in the sandbox",
338 "execute in sandbox",
339 "running in sandbox",
340 ];
341 if markers.iter().any(|m| lower.contains(m)) {
342 return true;
343 }
344 if (lower.contains("```bash") || lower.contains("```sh"))
345 && (lower.contains(".vak/scratch")
346 || lower.contains("python3 ")
347 || lower.contains("node ")
348 || lower.contains("cargo "))
349 {
350 return true;
351 }
352 false
353 }
354
355 /// Intent- and receipt-driven evaluation of completion validity.
356 pub fn evaluate_receipts(
357 &self,
358 prompt: &str,
359 final_text: &str,
360 outcome: Option<&vak_intent::OutcomeSpec>,
361 receipts: &ReceiptSummary,
362 verification_stale: bool,
363 ) -> Option<BlockReason> {
364 if final_text.trim().is_empty() {
365 return None;
366 }
367 if Self::requires_user_completion(prompt) {
368 return Some(BlockReason::UserCompletionRequired);
369 }
370 if self.marker_gate
371 && let Some(r) = Self::truncated_plan(final_text)
372 {
373 return Some(r);
374 }
375
376 // If a tool failed and hasn't been repaired or reported in text, block.
377 if let Some((tool, err)) = &receipts.unresolved_error
378 && !reports_blocker(final_text, tool, err)
379 {
380 return Some(BlockReason::UnresolvedToolFailure {
381 tool: tool.clone(),
382 error: err.clone(),
383 });
384 }
385
386 // Intent-driven gate: when outcome specification is available. The
387 // engagement's stop profile (docs/design/47-commitment-kernel.md)
388 // decides first; the typed acts refine it.
389 if let Some(spec) = outcome {
390 use vak_intent::StopProfile;
391 // `Verification`: a checkable result was demanded. An execution
392 // receipt is required, and it must not be stale.
393 if spec.stop == StopProfile::Verification {
394 if !receipts.has_execution_receipt() {
395 return Some(BlockReason::ExecutionReceiptMissing {
396 act: spec.deliverable_act().unwrap_or("verification").to_string(),
397 hint: "run the check that proves this is done".into(),
398 });
399 }
400 if verification_stale && receipts.code_files_modified > 0 {
401 return Some(BlockReason::VerificationStale);
402 }
403 }
404 if spec.stop == StopProfile::Effect || spec.requires_execution() {
405 if !receipts.has_execution_receipt() {
406 let act = spec.deliverable_act().unwrap_or("execution").to_string();
407 return Some(BlockReason::ExecutionReceiptMissing {
408 act,
409 hint: "run code, build, test, or modify files".into(),
410 });
411 }
412 if self.verify_gate
413 && verification_stale
414 && (spec.requires_execution() || Self::demands_verification(prompt))
415 && (receipts.code_files_modified > 0 || Self::demands_code_execution(prompt))
416 {
417 return Some(BlockReason::VerificationStale);
418 }
419 // `Inspection` on its own gates nothing: "something was looked
420 // at" includes the material the request carried (an attachment,
421 // pasted text), which leaves no receipt. Only a `locate` act
422 // demands an inspection receipt.
423 } else if spec.requires_inspection() {
424 let direct_substantive = final_text.trim().len() >= 80
425 && !Self::demands_code_execution(prompt)
426 && !Self::claims_execution_unexecuted(final_text);
427 if !receipts.has_inspection_receipt() && !direct_substantive {
428 let act = spec.deliverable_act().unwrap_or("inspection").to_string();
429 return Some(BlockReason::ExecutionReceiptMissing {
430 act,
431 hint: "read, search, inspect files or data".into(),
432 });
433 }
434 } else if spec.requires_tool() && !receipts.has_any_receipt() {
435 let direct_substantive = final_text.trim().len() >= 80
436 && !Self::demands_code_execution(prompt)
437 && !Self::claims_execution_unexecuted(final_text);
438 if !direct_substantive {
439 return Some(BlockReason::ExecutionReceiptMissing {
440 act: "tool execution".into(),
441 hint: "execute relevant tools".into(),
442 });
443 }
444 }
445 }
446
447 // Verification and execution gate: runs whenever verify_gate is enabled.
448 if self.verify_gate {
449 let demands_code = Self::demands_code_execution(prompt);
450 let claims_exec = Self::claims_execution_unexecuted(final_text);
451
452 if claims_exec {
453 return Some(BlockReason::VerificationMissing);
454 }
455
456 if demands_code && receipts.substantive_bash_calls == 0 {
457 return Some(BlockReason::VerificationMissing);
458 }
459
460 if Self::demands_verification(prompt) && receipts.substantive_bash_calls == 0 {
461 // If code files were touched, verification commands are required.
462 if receipts.code_files_modified > 0 {
463 return Some(BlockReason::VerificationMissing);
464 }
465 // If no tools were called and the text is not a substantive direct answer:
466 let direct_substantive = final_text.trim().len() >= 80;
467 if !receipts.has_any_receipt() && !direct_substantive {
468 return Some(BlockReason::VerificationMissing);
469 }
470 // If an execution deliverable was required or a specific file target was requested,
471 // but no files were modified or inspected:
472 let lower_p = prompt.to_ascii_lowercase();
473 let mentions_file_target = lower_p.contains(".md")
474 || lower_p.contains(".txt")
475 || lower_p.contains(".json")
476 || lower_p.contains(".csv")
477 || lower_p.contains("into ")
478 || lower_p.contains("in file")
479 || lower_p.contains("in the file");
480 if (outcome.map(|s| s.requires_execution()).unwrap_or(false)
481 || mentions_file_target)
482 && receipts.files_modified == 0
483 && receipts.read_or_inspected == 0
484 {
485 return Some(BlockReason::VerificationMissing);
486 }
487 // Universal tasks (documentation, research synthesis, lifestyle, notes, recipes,
488 // explanations) where content was inspected, written, or substantively answered
489 // are NOT falsely blocked on non-existent bash commands.
490 }
491
492 // A re-run is owed only when a check was asked for: by the
493 // reading, which decides what completion requires, or by the
494 // request's own words. An edit alone does not owe one — "add a
495 // subtract function" asked for the function — and demanding a
496 // check the surface could not run (a shell needing an approver
497 // nobody could be) looped a live turn until the block cap.
498 let check_owed = outcome.is_none_or(|spec| {
499 spec.stop == vak_intent::StopProfile::Verification || spec.requires_execution()
500 }) || demands_code
501 || Self::demands_verification(prompt);
502 if verification_stale
503 && check_owed
504 && (receipts.code_files_modified > 0 || demands_code)
505 {
506 return Some(BlockReason::VerificationStale);
507 }
508 }
509 None
510 }
511
512 /// Returns Some(reason) when completion should be blocked.
513 pub fn evaluate(
514 &self,
515 prompt: &str,
516 final_text: &str,
517 bash_calls_this_run: u32,
518 ) -> Option<BlockReason> {
519 let receipts = ReceiptSummary {
520 substantive_bash_calls: bash_calls_this_run,
521 successful_tool_calls: if bash_calls_this_run > 0 {
522 bash_calls_this_run
523 } else {
524 0
525 },
526 ..Default::default()
527 };
528 self.evaluate_receipts(prompt, final_text, None, &receipts, false)
529 }
530
531 pub fn evaluate_with_state(
532 &self,
533 prompt: &str,
534 final_text: &str,
535 bash_calls_this_run: u32,
536 verification_stale: bool,
537 ) -> Option<BlockReason> {
538 let receipts = ReceiptSummary {
539 substantive_bash_calls: bash_calls_this_run,
540 code_files_modified: if verification_stale { 1 } else { 0 },
541 files_modified: if verification_stale { 1 } else { 0 },
542 successful_tool_calls: if bash_calls_this_run > 0 {
543 bash_calls_this_run
544 } else {
545 0
546 },
547 ..Default::default()
548 };
549 self.evaluate_receipts(prompt, final_text, None, &receipts, verification_stale)
550 }
551}
552
553/// Checks whether a file path points to an executable, compilable, or script source file
554/// (as opposed to documentation, notes, recipes, data, or content assets).
555pub fn is_code_path(path: &str) -> bool {
556 let p = std::path::Path::new(path);
557 match p
558 .extension()
559 .and_then(|ext| ext.to_str())
560 .map(|ext| ext.to_ascii_lowercase())
561 {
562 Some(ext) => matches!(
563 ext.as_str(),
564 "rs" | "py"
565 | "js"
566 | "mjs"
567 | "cjs"
568 | "ts"
569 | "tsx"
570 | "jsx"
571 | "c"
572 | "cpp"
573 | "cc"
574 | "cxx"
575 | "h"
576 | "hpp"
577 | "go"
578 | "java"
579 | "kt"
580 | "kts"
581 | "rb"
582 | "php"
583 | "swift"
584 | "scala"
585 | "sh"
586 | "bash"
587 | "zsh"
588 | "fish"
589 | "ps1"
590 | "bat"
591 | "cmd"
592 | "lua"
593 | "pl"
594 | "pm"
595 | "r"
596 | "jl"
597 | "dart"
598 | "zig"
599 | "nim"
600 | "sql"
601 ),
602 None => false,
603 }
604}
605
606/// Checks if a shell command is substantive rather than a dummy echo/no-op evasion.
607pub fn is_substantive_command(command: &str) -> bool {
608 let trimmed = command.trim();
609 if trimmed.is_empty() {
610 return false;
611 }
612 // If it writes to a file or pipes to another command, it has side effects or processing
613 if trimmed.contains('>') || trimmed.contains('|') {
614 return true;
615 }
616 // Check if the command line is an explicit evasion claiming verification without doing work
617 let lower = trimmed.to_ascii_lowercase();
618 if (lower.starts_with("echo ") || lower.starts_with("printf "))
619 && (lower.contains("verification")
620 || lower.contains("shell execution path is functional")
621 || lower.contains("verified")
622 || lower.contains("dummy"))
623 {
624 return false;
625 }
626 let first_word = trimmed
627 .split_whitespace()
628 .next()
629 .unwrap_or("")
630 .trim_start_matches("./");
631 let is_noop = matches!(first_word, ":" | "true" | "false" | "exit");
632 !is_noop
633}
634
635#[cfg(test)]
636mod tests {
637 use super::*;
638
639 #[test]
640 fn truncated_shapes_block_normal_prose_passes() {
641 let p = StopPolicy::default();
642 assert!(
643 p.evaluate("", "Working on it:\n- fix parser\n- then", 1)
644 .is_none()
645 || true
646 ); // sanity no-op to keep structure
647
648 // trailing colon line
649 assert!(matches!(
650 p.evaluate("", "Let me check the config:", 3),
651 Some(BlockReason::TruncatedPlan(_))
652 ));
653 // unclosed fence
654 assert!(matches!(
655 p.evaluate("", "here is the patch:\n```rust\nfn a() {}", 3),
656 Some(BlockReason::TruncatedPlan(_))
657 ));
658 // plan-marker final line without terminal punctuation
659 assert!(matches!(
660 p.evaluate(
661 "",
662 "done with part one.\nNow I'll write the store module",
663 3
664 ),
665 Some(BlockReason::TruncatedPlan(_))
666 ));
667 // normal summary passes
668 assert_eq!(
669 p.evaluate("", "All done. Tests pass.\nSummary:\n- added x", 3),
670 None
671 );
672 assert_eq!(
673 p.evaluate("", "Fixed both issues. cargo test green.", 3),
674 None
675 );
676 // headings ending in colon are fine (e.g. 'Summary:')
677 assert_eq!(p.evaluate("", "Results\n# Summary:", 3), None);
678 }
679
680 #[test]
681 fn verify_gate_needs_demand_and_zero_bash() {
682 let p = StopPolicy::default();
683 let prompt = "Create fizzbuzz.py and run it to prove that it works.";
684 assert!(matches!(
685 p.evaluate(prompt, "Created the file.", 0),
686 Some(BlockReason::VerificationMissing)
687 ));
688 assert_eq!(p.evaluate(prompt, "Created the file.", 2), None);
689 // no demand -> never blocks
690 assert_eq!(p.evaluate("Write a haiku about sand.", "Done.", 0), None);
691 }
692
693 #[test]
694 fn verify_gate_recognizes_explicit_run_commands() {
695 let p = StopPolicy::default();
696 assert!(matches!(
697 p.evaluate(
698 "Read README.md, implement the change, then run python3 test_app.py.",
699 "I need more details.",
700 0
701 ),
702 Some(BlockReason::VerificationMissing)
703 ));
704 }
705
706 #[test]
707 fn explicit_until_done_request_requires_user_release() {
708 let p = StopPolicy::default();
709 assert!(matches!(
710 p.evaluate(
711 "Keep improving the project until I say done.",
712 "Improved it.",
713 1
714 ),
715 Some(BlockReason::UserCompletionRequired)
716 ));
717 assert!(StopPolicy::is_done_message("done"));
718 assert!(!StopPolicy::is_done_message(
719 "done, and here is the summary"
720 ));
721 }
722
723 /// An edit the reading did not hold to a check owes none: authoring a
724 /// function into a code file is done when the function is there.
725 #[test]
726 fn an_edit_owes_no_check_the_reading_did_not_ask_for() {
727 let p = StopPolicy::default();
728 let authoring = vak_intent::OutcomeSpec::from_reading(
729 "explain what calc.py does, then add a subtract function to it",
730 &vak_intent::Reading {
731 act: vak_intent::Act::Author,
732 ..vak_intent::Reading::general()
733 },
734 4,
735 );
736 let edited = ReceiptSummary {
737 total_tool_calls: 1,
738 successful_tool_calls: 1,
739 files_modified: 1,
740 code_files_modified: 1,
741 ..Default::default()
742 };
743 assert_eq!(
744 p.evaluate_receipts(
745 "explain what calc.py does, then add a subtract function to it",
746 "calc.py now defines add and subtract.",
747 Some(&authoring),
748 &edited,
749 true,
750 ),
751 None
752 );
753 // A fix is held to a check: it modifies code the reading expects to
754 // be proven.
755 let fixing = vak_intent::OutcomeSpec::from_reading(
756 "fix the off-by-one in calc.py",
757 &vak_intent::Reading {
758 act: vak_intent::Act::Modify,
759 ..vak_intent::Reading::general()
760 },
761 4,
762 );
763 let fixed = ReceiptSummary {
764 substantive_bash_calls: 1,
765 ..edited
766 };
767 assert_eq!(
768 p.evaluate_receipts(
769 "fix the off-by-one in calc.py",
770 "Fixed.",
771 Some(&fixing),
772 &fixed,
773 true,
774 ),
775 Some(BlockReason::VerificationStale)
776 );
777 }
778
779 #[test]
780 fn stale_verification_blocks_after_a_file_change() {
781 let p = StopPolicy::default();
782 assert_eq!(
783 p.evaluate_with_state("implement it and run the tests", "Done.", 1, true),
784 Some(BlockReason::VerificationStale)
785 );
786 assert_eq!(
787 p.evaluate_with_state("implement it and run the tests", "Done.", 1, false),
788 None
789 );
790 }
791
792 #[test]
793 fn empty_final_text_is_left_alone() {
794 let p = StopPolicy::default();
795 assert_eq!(p.evaluate("run the tests", "", 0), None);
796 }
797
798 #[test]
799 fn test_substantive_command_detection() {
800 assert!(!is_substantive_command(
801 "echo \"Verification successful: Shell execution path is functional.\""
802 ));
803 assert!(!is_substantive_command("echo 'verification passed'"));
804 assert!(!is_substantive_command("printf 'verified\\n'"));
805 assert!(!is_substantive_command("true"));
806 assert!(!is_substantive_command(":"));
807 assert!(!is_substantive_command("exit 0"));
808 assert!(!is_substantive_command(""));
809
810 assert!(is_substantive_command("echo ran"));
811 assert!(is_substantive_command("echo 'hello'"));
812 assert!(is_substantive_command("echo 'hello' > index.html"));
813 assert!(is_substantive_command("echo 'hi' | wc -l"));
814 assert!(is_substantive_command("python3 -m unittest"));
815 assert!(is_substantive_command("cargo test"));
816 assert!(is_substantive_command("npm start"));
817 assert!(is_substantive_command("node server.js"));
818 }
819
820 #[test]
821 fn test_sandbox_demands_verification() {
822 let p = StopPolicy::default();
823 let prompt =
824 "make in using react with beautifull design and run them in sandbox and show me";
825 assert!(matches!(
826 p.evaluate(prompt, "Here is the code in a block.", 0),
827 Some(BlockReason::VerificationMissing)
828 ));
829 }
830
831 #[test]
832 fn test_outcome_intent_requires_execution_receipt() {
833 let p = StopPolicy::default();
834 let mut reading = vak_intent::Reading::general();
835 reading.act = vak_intent::Act::Modify;
836 let spec = vak_intent::OutcomeSpec::from_reading("create an svg animation", &reading, 1);
837 assert!(spec.requires_execution());
838
839 // 0 receipts -> blocked
840 let empty_receipts = ReceiptSummary::default();
841 let blocked = p.evaluate_receipts(
842 "create an svg animation",
843 "Here is your svg:\n```xml\n<svg/>\n```",
844 Some(&spec),
845 &empty_receipts,
846 false,
847 );
848 assert!(matches!(
849 blocked,
850 Some(BlockReason::ExecutionReceiptMissing { .. })
851 ));
852
853 // with substantive bash receipt -> allowed
854 let with_bash = ReceiptSummary {
855 substantive_bash_calls: 1,
856 successful_tool_calls: 1,
857 ..Default::default()
858 };
859 assert_eq!(
860 p.evaluate_receipts(
861 "create an svg animation",
862 "Created and verified.",
863 Some(&spec),
864 &with_bash,
865 false
866 ),
867 None
868 );
869
870 // Work done through an integration or a delegated worker is
871 // execution too: the stop gate must not demand a shell receipt for
872 // an email an MCP server sent.
873 let external = ReceiptSummary {
874 external_effects: 1,
875 successful_tool_calls: 1,
876 ..Default::default()
877 };
878 assert!(external.has_execution_receipt());
879 assert_eq!(
880 p.evaluate_receipts(
881 "create an svg animation",
882 "Created and verified.",
883 Some(&spec),
884 &external,
885 false
886 ),
887 None
888 );
889
890 // with file write receipt -> allowed
891 let with_file = ReceiptSummary {
892 files_modified: 1,
893 successful_tool_calls: 1,
894 ..Default::default()
895 };
896 assert_eq!(
897 p.evaluate_receipts(
898 "create an svg animation",
899 "Created file.",
900 Some(&spec),
901 &with_file,
902 false
903 ),
904 None
905 );
906 }
907
908 #[test]
909 fn capped_continuation_counts_prior_saved_file_only_after_successful_inspection() {
910 let policy = StopPolicy::default();
911 let mut reading = vak_intent::Reading::general();
912 reading.act = vak_intent::Act::Modify;
913 let spec = vak_intent::OutcomeSpec::from_reading("create report.csv", &reading, 1);
914 let prompt = "Continue the most recent unfinished task";
915 let answer = "The saved report contains 60 minutes.";
916 let prior_only = ReceiptSummary {
917 continued_saved_file: true,
918 read_or_inspected: 1,
919 ..Default::default()
920 };
921 assert!(matches!(
922 policy.evaluate_receipts(prompt, answer, Some(&spec), &prior_only, false),
923 Some(BlockReason::ExecutionReceiptMissing { .. })
924 ));
925 let inspected = ReceiptSummary {
926 successful_inspections: 1,
927 ..prior_only
928 };
929 assert_eq!(
930 policy.evaluate_receipts(prompt, answer, Some(&spec), &inspected, false),
931 None
932 );
933 }
934
935 #[test]
936 fn test_outcome_conversational_allows_prose_completion() {
937 let p = StopPolicy::default();
938 let mut reading = vak_intent::Reading::general();
939 reading.act = vak_intent::Act::Answer;
940 let spec = vak_intent::OutcomeSpec::from_reading("what is rust?", &reading, 1);
941 assert!(!spec.requires_execution());
942 assert!(!spec.requires_tool());
943
944 let receipts = ReceiptSummary::default();
945 assert_eq!(
946 p.evaluate_receipts(
947 "what is rust?",
948 "Rust is a systems programming language.",
949 Some(&spec),
950 &receipts,
951 false
952 ),
953 None
954 );
955 }
956
957 #[test]
958 fn test_claims_execution_unexecuted_blocks_even_with_outcome_spec() {
959 let p = StopPolicy::default();
960 let reading = vak_intent::Reading::general();
961 let spec = vak_intent::OutcomeSpec::from_reading("can you run it and show", &reading, 1);
962 let receipts = ReceiptSummary::default();
963
964 let blocked = p.evaluate_receipts(
965 "can you run it and show",
966 "I will use the bash tool to execute a Python script:\n```bash\npython3 script.py\n```",
967 Some(&spec),
968 &receipts,
969 false,
970 );
971 assert_eq!(blocked, Some(BlockReason::VerificationMissing));
972 }
973
974 #[test]
975 fn test_unresolved_tool_failure_blocks_unless_reported() {
976 let p = StopPolicy::default();
977 let receipts = ReceiptSummary {
978 total_tool_calls: 1,
979 failed_tool_calls: 1,
980 unresolved_error: Some(("bash".into(), "exit code 1: compile error".into())),
981 ..Default::default()
982 };
983
984 // Model hallucinates success without reporting error -> blocked
985 let blocked = p.evaluate_receipts(
986 "build it",
987 "All done! Everything succeeded.",
988 None,
989 &receipts,
990 false,
991 );
992 assert!(matches!(
993 blocked,
994 Some(BlockReason::UnresolvedToolFailure { .. })
995 ));
996
997 // Model reports the error/blocker -> allowed
998 let reported = p.evaluate_receipts(
999 "build it",
1000 "The build failed with exit code 1: compile error. Cannot proceed without missing dependency.",
1001 None,
1002 &receipts,
1003 false,
1004 );
1005 assert_eq!(reported, None);
1006 }
1007
1008 #[test]
1009 fn test_direct_substantive_answer_not_blocked_for_inspection_spec() {
1010 let p = StopPolicy::default();
1011 let mut reading = vak_intent::Reading::general();
1012 reading.act = vak_intent::Act::Locate;
1013 let spec = vak_intent::OutcomeSpec::from_reading(
1014 "explain the architectural differences",
1015 &reading,
1016 1,
1017 );
1018 let receipts = ReceiptSummary::default();
1019
1020 // Substantive direct analysis without false tool claims or execution demands -> allowed
1021 let substantive_answer = "Optimistic locking assumes multiple transactions can complete without affecting each other. It verifies no other transaction has modified the data before committing. In contrast, pessimistic locking acquires locks immediately upon reading.";
1022 let blocked = p.evaluate_receipts(
1023 "explain the architectural differences",
1024 substantive_answer,
1025 Some(&spec),
1026 &receipts,
1027 false,
1028 );
1029 assert_eq!(blocked, None);
1030
1031 // Empty or non-substantive answer -> blocked
1032 let blocked_empty = p.evaluate_receipts(
1033 "explain the architectural differences",
1034 "Okay",
1035 Some(&spec),
1036 &receipts,
1037 false,
1038 );
1039 assert!(matches!(
1040 blocked_empty,
1041 Some(BlockReason::ExecutionReceiptMissing { .. })
1042 ));
1043 }
1044
1045 #[test]
1046 fn test_is_code_path_accurately_classifies_code_vs_doc_paths() {
1047 assert!(is_code_path("src/main.rs"));
1048 assert!(is_code_path("backend/app.py"));
1049 assert!(is_code_path("web/index.ts"));
1050 assert!(is_code_path("scripts/deploy.sh"));
1051
1052 assert!(!is_code_path("README.md"));
1053 assert!(!is_code_path("docs/architecture.md"));
1054 assert!(!is_code_path("recipes/sourdough.txt"));
1055 assert!(!is_code_path("data/analysis.csv"));
1056 assert!(!is_code_path("notes.org"));
1057 }
1058
1059 #[test]
1060 fn test_universal_doc_modification_with_verify_not_blocked_on_bash() {
1061 let p = StopPolicy::default();
1062 let prompt = "Update README.md to describe the release steps and verify that all links are formatted correctly.";
1063 let final_text = "Updated README.md with release steps and verified that the Markdown links match the repository structure.";
1064
1065 let receipts = ReceiptSummary {
1066 total_tool_calls: 2,
1067 successful_tool_calls: 2,
1068 files_modified: 1,
1069 code_files_modified: 0,
1070 doc_files_modified: 1,
1071 read_or_inspected: 1,
1072 ..Default::default()
1073 };
1074
1075 // Even though prompt says "verify that", since no code was modified and no code execution was demanded,
1076 // it must NOT block on non-existent bash commands or stale verification!
1077 let blocked = p.evaluate_receipts(prompt, final_text, None, &receipts, false);
1078 assert_eq!(blocked, None);
1079
1080 let blocked_stale = p.evaluate_receipts(prompt, final_text, None, &receipts, true);
1081 assert_eq!(blocked_stale, None);
1082 }
1083
1084 #[test]
1085 fn test_universal_research_and_lifestyle_with_verify_not_blocked() {
1086 let p = StopPolicy::default();
1087 let prompt =
1088 "Compare the top 3 pour-over drippers and verify that the brew ratios are accurate.";
1089 let final_text = "Here is a detailed comparison of Hario V60, Kalita Wave, and Chemex. All brew ratios are verified between 1:15 and 1:17 for balanced extraction across light and medium roasts.";
1090
1091 let receipts = ReceiptSummary {
1092 total_tool_calls: 1,
1093 successful_tool_calls: 1,
1094 read_or_inspected: 1,
1095 ..Default::default()
1096 };
1097
1098 let blocked = p.evaluate_receipts(prompt, final_text, None, &receipts, false);
1099 assert_eq!(blocked, None);
1100 }
1101
1102 #[test]
1103 fn test_code_modification_with_verify_blocked_without_execution() {
1104 let p = StopPolicy::default();
1105 let prompt =
1106 "Fix the off-by-one bug in quicksort.py and verify that the sort works correctly.";
1107 let final_text = "I fixed the index in quicksort.py.";
1108
1109 let receipts = ReceiptSummary {
1110 total_tool_calls: 1,
1111 successful_tool_calls: 1,
1112 files_modified: 1,
1113 code_files_modified: 1,
1114 doc_files_modified: 0,
1115 ..Default::default()
1116 };
1117
1118 // Code was modified and prompt asks to "verify that" -> must block with VerificationMissing
1119 let blocked = p.evaluate_receipts(prompt, final_text, None, &receipts, false);
1120 assert_eq!(blocked, Some(BlockReason::VerificationMissing));
1121
1122 // If code files were modified after test ran, stale verification blocks
1123 let with_bash = ReceiptSummary {
1124 total_tool_calls: 2,
1125 successful_tool_calls: 2,
1126 substantive_bash_calls: 1,
1127 files_modified: 1,
1128 code_files_modified: 1,
1129 ..Default::default()
1130 };
1131 let blocked_stale = p.evaluate_receipts(prompt, final_text, None, &with_bash, true);
1132 assert_eq!(blocked_stale, Some(BlockReason::VerificationStale));
1133 }
1134}
1135

Stop Policy

Indexing the workspace…

Stop Policy