Vakyartha documentation

7299 &serde_json::json!({"path": "notes.txt", "content": "x"}),
7300 Mode::WorkspaceWrite,
7301 false,
7302 cwd.path(),
7303 ),
7304 "writing inside the workspace should be auto-approved"
7305 );
7306 let outside = std::env::temp_dir().join("vak-outside.txt");
7307 assert!(
7308 !auto_approve(
7309 ApprovalMode::ApproveSafe,
7310 AskSource::ModeDefault,
7311 "write",
7312 &serde_json::json!({"path": outside.to_string_lossy(), "content": "x"}),
7313 Mode::WorkspaceWrite,
7314 false,
7315 cwd.path(),
7316 ),
7317 "writing outside the workspace must not be auto-approved"
7318 );
7319 }
7320
7321 #[test]
7322 fn approve_safe_denies_non_mode_default_sources() {
7323 let cwd = ws();
7324 // Scope source is auto-approved for workspace-scoped writes.
7325 assert!(auto_approve(
7326 ApprovalMode::ApproveSafe,
7327 AskSource::Scope,
7328 "write",
7329 &serde_json::json!({"path": "ok.txt", "content": "x"}),
7330 Mode::WorkspaceWrite,
7331 false,
7332 cwd.path(),
7333 ));
7334 }
7335
7336 #[test]
7337 fn ask_mode_never_auto_approves_anything() {
7338 let cwd = ws();
7339 assert!(!auto_approve(
7340 ApprovalMode::Ask,
7341 AskSource::ModeDefault,
7342 "read",
7343 &serde_json::json!({}),
7344 Mode::WorkspaceWrite,
7345 true,
7346 cwd.path(),
7347 ));
7348 }
7349
7350 #[test]
7351 fn bash_without_command_arg_still_approved_when_sandboxed() {
7352 // auto_approve for bash keys only on (sandboxed && not FullAccess),
7353 // not on the presence of a command arg — the command arg is
7354 // validated separately by authorize(). This documents that boundary.
7355 let cwd = ws();
7356 assert!(
7357 !auto_approve(
7358 ApprovalMode::ApproveSafe,
7359 AskSource::ModeDefault,
7360 "bash",
7361 &serde_json::json!({}),
7362 Mode::WorkspaceWrite,
7363 false,
7364 cwd.path(),
7365 ),
7366 "un-sandboxed bash without command arg must not be auto-approved"
7367 );
7368 assert!(
7369 auto_approve(
7370 ApprovalMode::ApproveSafe,
7371 AskSource::ModeDefault,
7372 "bash",
7373 &serde_json::json!({}),
7374 Mode::WorkspaceWrite,
7375 true,
7376 cwd.path(),
7377 ),
7378 "sandboxed bash is auto-approved by the sand-boxing contract"
7379 );
7380 }
7381}
7382
7383#[cfg(test)]
7384#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
7385mod tool_call_envelope_tests {
7386 use super::{ContentBlock, PendingToolCall, extract_tool_calls};
7387 use vak_llm::types::{AssistantMessage, StopReason, Usage};
7388 use vak_tools::{Tool, ToolContext, ToolOutput};
7389
7390 struct FakeTool(&'static str);
7391
7392 #[async_trait::async_trait]
7393 impl Tool for FakeTool {
7394 fn name(&self) -> &str {
7395 self.0
7396 }
7397 fn description(&self) -> &str {
7398 "fake"
7399 }
7400 fn schema(&self) -> serde_json::Value {
7401 serde_json::json!({"type": "object"})
7402 }
7403 async fn execute(&self, _args: &serde_json::Value, _ctx: &ToolContext) -> ToolOutput {
7404 ToolOutput::ok(String::new())
7405 }
7406 }
7407
7408 fn loaded() -> Vec<std::sync::Arc<dyn Tool>> {
7409 vec![
7410 std::sync::Arc::new(FakeTool("write")),
7411 std::sync::Arc::new(FakeTool("bash")),
7412 ]
7413 }
7414
7415 fn text_response(text: &str) -> AssistantMessage {
7416 AssistantMessage {
7417 content: vec![ContentBlock::text(text.to_string())],
7418 stop_reason: StopReason::EndTurn,
7419 usage: Usage::default(),
7420 model: "test-model".into(),
7421 response_id: None,
7422 }
7423 }
7424
7425 fn assert_no_calls_and_unchanged(text: &str) {
7426 let mut response = text_response(text);
7427 let original = response.clone();
7428 let calls = extract_tool_calls(&mut response, &loaded());
7429 assert!(calls.is_empty(), "{text:?} must produce no calls");
7430 assert_eq!(
7431 response, original,
7432 "a response with no recognised envelope must be left untouched"
7433 );
7434 }
7435
7436 /// A fenced shell example is prose, not an instruction: illustrating a
7437 /// command must never execute it (live: a ```bash example ran and
7438 /// created a file the model never asked to create).
7439 #[test]
7440 fn prose_with_a_bash_fence_produces_no_calls() {
7441 assert_no_calls_and_unchanged(
7442 "Here is an example:\n```bash\necho FENCE-EXECUTED > marker.txt\n```\n",
7443 );
7444 }
7445
7446 /// The ```json fence path is removed entirely: a model narrating JSON
7447 /// that happens to include a "name" field must not be interpreted as a
7448 /// tool call.
7449 #[test]
7450 fn json_fence_with_a_name_field_produces_no_calls() {
7451 assert_no_calls_and_unchanged(
7452 "For reference, the shape is:\n```json\n{\"name\": \"write\", \"arguments\": {\"path\": \"a\", \"content\": \"b\"}}\n```\n",
7453 );
7454 }
7455
7456 #[test]
7457 fn bash_dash_c_in_prose_produces_no_calls() {
7458 assert_no_calls_and_unchanged(r#"You could run bash -c "echo hi" locally."#);
7459 }
7460
7461 #[test]
7462 fn functional_write_call_in_prose_produces_no_calls() {
7463 assert_no_calls_and_unchanged(
7464 r#"The call looks like write(path="a.txt", content="hello")."#,
7465 );
7466 }
7467
7468 /// An envelope naming a tool that was not loaded this turn is ignored:
7469 /// the fallback must not invoke anything a model happened to spell out.
7470 #[test]
7471 fn envelope_naming_an_unknown_tool_produces_no_call() {
7472 assert_no_calls_and_unchanged(
7473 "<tool_call>{\"name\": \"delete_everything\", \"arguments\": {}}</tool_call>",
7474 );
7475 }
7476
7477 /// An envelope whose `arguments` is present but not a JSON object is
7478 /// malformed and ignored rather than guessed at.
7479 #[test]
7480 fn envelope_with_non_object_arguments_produces_no_call() {
7481 assert_no_calls_and_unchanged(
7482 "<tool_call>{\"name\": \"bash\", \"arguments\": \"not an object\"}</tool_call>",
7483 );
7484 }
7485
7486 /// The XML-style envelope naming a loaded tool produces a ToolUse block
7487 /// in the rewritten response, with a paired PendingToolCall to dispatch,
7488 /// and the envelope text is excised while surrounding prose survives.
7489 #[test]
7490 fn xml_envelope_naming_a_loaded_tool_produces_a_tool_use_block() {
7491 let mut response = text_response(
7492 "Let me check that for you.\n<tool_call>{\"name\": \"bash\", \"arguments\": {\"command\": \"ls\"}}</tool_call>\nDone.",
7493 );
7494 let calls = extract_tool_calls(&mut response, &loaded());
7495 assert_eq!(calls.len(), 1);
7496 assert_eq!(calls[0].name, "bash");
7497 assert_eq!(calls[0].input["command"], "ls");
7498 assert_eq!(response.stop_reason, StopReason::ToolUse);
7499 let tool_use_ids: Vec<&str> = response
7500 .content
7501 .iter()
7502 .filter_map(|b| match b {
7503 ContentBlock::ToolUse { id, name, .. } if name == "bash" => Some(id.as_str()),
7504 _ => None,
7505 })
7506 .collect();
7507 assert_eq!(tool_use_ids, vec![calls[0].id.as_str()]);
7508 let remaining_text = response.text_content();
7509 assert!(remaining_text.contains("Let me check that for you."));
7510 assert!(remaining_text.contains("Done."));
7511 assert!(!remaining_text.contains("tool_call"));
7512 }
7513
7514 /// The fenced `tool_call`/`tool_use` envelope, keyed by `tool` +
7515 /// `input` instead of `name` + `arguments`, is recognised the same way.
7516 #[test]
7517 fn fenced_tool_use_envelope_with_tool_and_input_keys_is_recognised() {
7518 let mut response = text_response(
7519 "```tool_use\n{\"tool\": \"write\", \"input\": {\"path\": \"a.txt\", \"content\": \"hi\"}}\n```",
7520 );
7521 let calls = extract_tool_calls(&mut response, &loaded());
7522 assert_eq!(calls.len(), 1);
7523 assert_eq!(calls[0].name, "write");
7524 assert_eq!(calls[0].input["path"], "a.txt");
7525 assert_eq!(response.stop_reason, StopReason::ToolUse);
7526 }
7527
7528 /// Missing `arguments`/`input`/`parameters` defaults to an empty
7529 /// object rather than being rejected as malformed.
7530 #[test]
7531 fn envelope_with_no_arguments_key_defaults_to_an_empty_object() {
7532 let mut response = text_response("<tool_call>{\"name\": \"bash\"}</tool_call>");
7533 let calls = extract_tool_calls(&mut response, &loaded());
7534 assert_eq!(calls.len(), 1);
7535 assert_eq!(calls[0].input, serde_json::json!({}));
7536 }
7537
7538 /// A structured `tool_use` content block always wins over any text
7539 /// envelope: the fallback is never consulted when the provider already
7540 /// gave a real tool call.
7541 #[test]
7542 fn a_structured_tool_use_block_short_circuits_the_text_fallback() {
7543 let mut response = AssistantMessage {
7544 content: vec![ContentBlock::ToolUse {
7545 id: "real-1".into(),
7546 name: "bash".into(),
7547 input: serde_json::json!({"command": "ls"}),
7548 }],
7549 stop_reason: StopReason::ToolUse,
7550 usage: Usage::default(),
7551 model: "test-model".into(),
7552 response_id: None,
7553 };
7554 let calls = extract_tool_calls(&mut response, &loaded());
7555 assert_eq!(calls.len(), 1);
7556 assert_eq!(calls[0].id, "real-1");
7557 }
7558
7559 #[test]
7560 fn a_pending_tool_call_carries_the_expected_fields() {
7561 // Sanity check on the struct's shape used throughout this module.
7562 let call = PendingToolCall {
7563 id: "x".into(),
7564 name: "bash".into(),
7565 input: serde_json::json!({"command": "ls"}),
7566 };
7567 assert_eq!(call.name, "bash");
7568 }
7569}
7570

7299 &serde_json::json!({"path": "notes.txt", "content": "x"}),
7300 Mode::WorkspaceWrite,
7301 false,
7302 cwd.path(),
7303 ),
7304 "writing inside the workspace should be auto-approved"
7305 );
7306 let outside = std::env::temp_dir().join("vak-outside.txt");
7307 assert!(
7308 !auto_approve(
7309 ApprovalMode::ApproveSafe,
7310 AskSource::ModeDefault,
7311 "write",
7312 &serde_json::json!({"path": outside.to_string_lossy(), "content": "x"}),
7313 Mode::WorkspaceWrite,
7314 false,
7315 cwd.path(),
7316 ),
7317 "writing outside the workspace must not be auto-approved"
7318 );
7319 }
7320
7321 #[test]
7322 fn approve_safe_denies_non_mode_default_sources() {
7323 let cwd = ws();
7324 // Scope source is auto-approved for workspace-scoped writes.
7325 assert!(auto_approve(
7326 ApprovalMode::ApproveSafe,
7327 AskSource::Scope,
7328 "write",
7329 &serde_json::json!({"path": "ok.txt", "content": "x"}),
7330 Mode::WorkspaceWrite,
7331 false,
7332 cwd.path(),
7333 ));
7334 }
7335
7336 #[test]
7337 fn ask_mode_never_auto_approves_anything() {
7338 let cwd = ws();
7339 assert!(!auto_approve(
7340 ApprovalMode::Ask,
7341 AskSource::ModeDefault,
7342 "read",
7343 &serde_json::json!({}),
7344 Mode::WorkspaceWrite,
7345 true,
7346 cwd.path(),
7347 ));
7348 }
7349
7350 #[test]
7351 fn bash_without_command_arg_still_approved_when_sandboxed() {
7352 // auto_approve for bash keys only on (sandboxed && not FullAccess),
7353 // not on the presence of a command arg — the command arg is
7354 // validated separately by authorize(). This documents that boundary.
7355 let cwd = ws();
7356 assert!(
7357 !auto_approve(
7358 ApprovalMode::ApproveSafe,
7359 AskSource::ModeDefault,
7360 "bash",
7361 &serde_json::json!({}),
7362 Mode::WorkspaceWrite,
7363 false,
7364 cwd.path(),
7365 ),
7366 "un-sandboxed bash without command arg must not be auto-approved"
7367 );
7368 assert!(
7369 auto_approve(
7370 ApprovalMode::ApproveSafe,
7371 AskSource::ModeDefault,
7372 "bash",
7373 &serde_json::json!({}),
7374 Mode::WorkspaceWrite,
7375 true,
7376 cwd.path(),
7377 ),
7378 "sandboxed bash is auto-approved by the sand-boxing contract"
7379 );
7380 }
7381}
7382
7383#[cfg(test)]
7384#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
7385mod tool_call_envelope_tests {
7386 use super::{ContentBlock, PendingToolCall, extract_tool_calls};
7387 use vak_llm::types::{AssistantMessage, StopReason, Usage};
7388 use vak_tools::{Tool, ToolContext, ToolOutput};
7389
7390 struct FakeTool(&'static str);
7391
7392 #[async_trait::async_trait]
7393 impl Tool for FakeTool {
7394 fn name(&self) -> &str {
7395 self.0
7396 }
7397 fn description(&self) -> &str {
7398 "fake"
7399 }
7400 fn schema(&self) -> serde_json::Value {
7401 serde_json::json!({"type": "object"})
7402 }
7403 async fn execute(&self, _args: &serde_json::Value, _ctx: &ToolContext) -> ToolOutput {
7404 ToolOutput::ok(String::new())
7405 }
7406 }
7407
7408 fn loaded() -> Vec<std::sync::Arc<dyn Tool>> {
7409 vec![
7410 std::sync::Arc::new(FakeTool("write")),
7411 std::sync::Arc::new(FakeTool("bash")),
7412 ]
7413 }
7414
7415 fn text_response(text: &str) -> AssistantMessage {
7416 AssistantMessage {
7417 content: vec![ContentBlock::text(text.to_string())],
7418 stop_reason: StopReason::EndTurn,
7419 usage: Usage::default(),
7420 model: "test-model".into(),
7421 response_id: None,
7422 }
7423 }
7424
7425 fn assert_no_calls_and_unchanged(text: &str) {
7426 let mut response = text_response(text);
7427 let original = response.clone();
7428 let calls = extract_tool_calls(&mut response, &loaded());
7429 assert!(calls.is_empty(), "{text:?} must produce no calls");
7430 assert_eq!(
7431 response, original,
7432 "a response with no recognised envelope must be left untouched"
7433 );
7434 }
7435
7436 /// A fenced shell example is prose, not an instruction: illustrating a
7437 /// command must never execute it (live: a ```bash example ran and
7438 /// created a file the model never asked to create).
7439 #[test]
7440 fn prose_with_a_bash_fence_produces_no_calls() {
7441 assert_no_calls_and_unchanged(
7442 "Here is an example:\n```bash\necho FENCE-EXECUTED > marker.txt\n```\n",
7443 );
7444 }
7445
7446 /// The ```json fence path is removed entirely: a model narrating JSON
7447 /// that happens to include a "name" field must not be interpreted as a
7448 /// tool call.
7449 #[test]
7450 fn json_fence_with_a_name_field_produces_no_calls() {
7451 assert_no_calls_and_unchanged(
7452 "For reference, the shape is:\n```json\n{\"name\": \"write\", \"arguments\": {\"path\": \"a\", \"content\": \"b\"}}\n```\n",
7453 );
7454 }
7455
7456 #[test]
7457 fn bash_dash_c_in_prose_produces_no_calls() {
7458 assert_no_calls_and_unchanged(r#"You could run bash -c "echo hi" locally."#);
7459 }
7460
7461 #[test]
7462 fn functional_write_call_in_prose_produces_no_calls() {
7463 assert_no_calls_and_unchanged(
7464 r#"The call looks like write(path="a.txt", content="hello")."#,
7465 );
7466 }
7467
7468 /// An envelope naming a tool that was not loaded this turn is ignored:
7469 /// the fallback must not invoke anything a model happened to spell out.
7470 #[test]
7471 fn envelope_naming_an_unknown_tool_produces_no_call() {
7472 assert_no_calls_and_unchanged(
7473 "<tool_call>{\"name\": \"delete_everything\", \"arguments\": {}}</tool_call>",
7474 );
7475 }
7476
7477 /// An envelope whose `arguments` is present but not a JSON object is
7478 /// malformed and ignored rather than guessed at.
7479 #[test]
7480 fn envelope_with_non_object_arguments_produces_no_call() {
7481 assert_no_calls_and_unchanged(
7482 "<tool_call>{\"name\": \"bash\", \"arguments\": \"not an object\"}</tool_call>",
7483 );
7484 }
7485
7486 /// The XML-style envelope naming a loaded tool produces a ToolUse block
7487 /// in the rewritten response, with a paired PendingToolCall to dispatch,
7488 /// and the envelope text is excised while surrounding prose survives.
7489 #[test]
7490 fn xml_envelope_naming_a_loaded_tool_produces_a_tool_use_block() {
7491 let mut response = text_response(
7492 "Let me check that for you.\n<tool_call>{\"name\": \"bash\", \"arguments\": {\"command\": \"ls\"}}</tool_call>\nDone.",
7493 );
7494 let calls = extract_tool_calls(&mut response, &loaded());
7495 assert_eq!(calls.len(), 1);
7496 assert_eq!(calls[0].name, "bash");
7497 assert_eq!(calls[0].input["command"], "ls");
7498 assert_eq!(response.stop_reason, StopReason::ToolUse);
7499 let tool_use_ids: Vec<&str> = response
7500 .content
7501 .iter()
7502 .filter_map(|b| match b {
7503 ContentBlock::ToolUse { id, name, .. } if name == "bash" => Some(id.as_str()),
7504 _ => None,
7505 })
7506 .collect();
7507 assert_eq!(tool_use_ids, vec![calls[0].id.as_str()]);
7508 let remaining_text = response.text_content();
7509 assert!(remaining_text.contains("Let me check that for you."));
7510 assert!(remaining_text.contains("Done."));
7511 assert!(!remaining_text.contains("tool_call"));
7512 }
7513
7514 /// The fenced `tool_call`/`tool_use` envelope, keyed by `tool` +
7515 /// `input` instead of `name` + `arguments`, is recognised the same way.
7516 #[test]
7517 fn fenced_tool_use_envelope_with_tool_and_input_keys_is_recognised() {
7518 let mut response = text_response(
7519 "```tool_use\n{\"tool\": \"write\", \"input\": {\"path\": \"a.txt\", \"content\": \"hi\"}}\n```",
7520 );
7521 let calls = extract_tool_calls(&mut response, &loaded());
7522 assert_eq!(calls.len(), 1);
7523 assert_eq!(calls[0].name, "write");
7524 assert_eq!(calls[0].input["path"], "a.txt");
7525 assert_eq!(response.stop_reason, StopReason::ToolUse);
7526 }
7527
7528 /// Missing `arguments`/`input`/`parameters` defaults to an empty
7529 /// object rather than being rejected as malformed.
7530 #[test]
7531 fn envelope_with_no_arguments_key_defaults_to_an_empty_object() {
7532 let mut response = text_response("<tool_call>{\"name\": \"bash\"}</tool_call>");
7533 let calls = extract_tool_calls(&mut response, &loaded());
7534 assert_eq!(calls.len(), 1);
7535 assert_eq!(calls[0].input, serde_json::json!({}));
7536 }
7537
7538 /// A structured `tool_use` content block always wins over any text
7539 /// envelope: the fallback is never consulted when the provider already
7540 /// gave a real tool call.
7541 #[test]
7542 fn a_structured_tool_use_block_short_circuits_the_text_fallback() {
7543 let mut response = AssistantMessage {
7544 content: vec![ContentBlock::ToolUse {
7545 id: "real-1".into(),
7546 name: "bash".into(),
7547 input: serde_json::json!({"command": "ls"}),
7548 }],
7549 stop_reason: StopReason::ToolUse,
7550 usage: Usage::default(),
7551 model: "test-model".into(),
7552 response_id: None,
7553 };
7554 let calls = extract_tool_calls(&mut response, &loaded());
7555 assert_eq!(calls.len(), 1);
7556 assert_eq!(calls[0].id, "real-1");
7557 }
7558
7559 #[test]
7560 fn a_pending_tool_call_carries_the_expected_fields() {
7561 // Sanity check on the struct's shape used throughout this module.
7562 let call = PendingToolCall {
7563 id: "x".into(),
7564 name: "bash".into(),
7565 input: serde_json::json!({"command": "ls"}),
7566 };
7567 assert_eq!(call.name, "bash");
7568 }
7569}
7570

Lib

Indexing the workspace…

Lib