Vakyartha documentation

1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2
3use std::sync::Arc;
4
5use tokio::sync::mpsc;
6use tokio_util::sync::CancellationToken;
7
8use tempfile::tempdir;
9
10use vak_agent::{Agent, AgentConfig, Approver, AutoApprove, TurnOutcome};
11use vak_llm::Provider;
12use vak_llm::types::{AssistantMessage, ContentBlock, StopReason, Usage};
13use vak_permission::PermissionEngine;
14use vak_session::SessionLog;
15use vak_session::types::{FrozenContract, SessionHeader};
16use vak_tools::bash::BashTool;
17
18fn bash_call(id: &str, cmd: &str) -> AssistantMessage {
19 AssistantMessage {
20 content: vec![ContentBlock::ToolUse {
21 id: id.into(),
22 name: "bash".into(),
23 input: serde_json::json!({"command": cmd}),
24 }],
25 stop_reason: StopReason::ToolUse,
26 usage: Usage::default(),
27 model: "test-model".into(),
28 response_id: None,
29 }
30}
31
32fn text_msg(t: &str) -> AssistantMessage {
33 AssistantMessage {
34 content: vec![ContentBlock::text(t)],
35 stop_reason: StopReason::EndTurn,
36 usage: Usage {
37 input_tokens: 1,
38 output_tokens: 1,
39 ..Default::default()
40 },
41 model: "test-model".into(),
42 response_id: None,
43 }
44}
45
46struct MultiScripted {
47 msgs: std::sync::Mutex<std::collections::VecDeque<AssistantMessage>>,
48}
49
50#[async_trait::async_trait]
51impl Provider for MultiScripted {
52 fn name(&self) -> &str {
53 "scripted"
54 }
55
56 async fn stream(
57 &self,
58 _request: vak_llm::types::ChatRequest,
59 _cancel: CancellationToken,
60 ) -> Result<vak_llm::EventStream, vak_llm::LlmError> {
61 let next = self.msgs.lock().unwrap().pop_front();
62 let (mut sink, rx) = vak_llm::stream::channel(8);
63 if let Some(m) = next {
64 sink.push(vak_llm::stream::StreamEvent::Start { partial: m.clone() });
65 sink.close_message(m).await;
66 } else {
67 sink.close_error(vak_llm::LlmError::Parse("exhausted".into()))
68 .await;
69 }
70 Ok(rx)
71 }
72}
73
74fn multi_setup(
75 responses: Vec<AssistantMessage>,
76 engine: Option<PermissionEngine>,
77 approver: Option<Arc<dyn Approver>>,
78) -> Agent {
79 let dir = tempdir().unwrap();
80 let header = SessionHeader {
81 agent: None,
82 session_id: "s".into(),
83 created_at: chrono::Utc::now(),
84 cwd: dir.path().to_path_buf(),
85 parent_session_id: None,
86 contract_id: None,
87 work_item_id: None,
88 conversation: None,
89 contract: FrozenContract {
90 app_version: "0".into(),
91 provider: "scripted".into(),
92 model: "test-model".into(),
93 route_ladder: Vec::new(),
94 route_objective: String::new(),
95 route_annotations: Vec::new(),
96 system_prompt: "sys".into(),
97 permission_mode: "workspace-write".into(),
98 capabilities: Vec::new(),
99 prompt_layers: Vec::new(),
100 },
101 };
102 let log = SessionLog::create(dir.path().join("s.jsonl"), header).unwrap();
103 let mut cfg = AgentConfig::new("sys");
104 cfg.tools = vec![Arc::new(BashTool)];
105 cfg.permission = engine.map(Arc::new);
106 cfg.approver = approver;
107 cfg.stop_policy = None;
108 std::mem::forget(dir);
109 let provider = MultiScripted {
110 msgs: std::sync::Mutex::new(responses.into_iter().collect()),
111 };
112 Agent::new(Arc::new(provider), log, cfg)
113}
114
115#[tokio::test]
116async fn deny_rule_blocks_execution_and_feeds_reason_back() {
117 let eng = PermissionEngine::from_rule_strings(&["-Bash(rm *)".to_string()]).unwrap();
118 let mut agent = multi_setup(
119 vec![bash_call("t1", "rm -rf /"), text_msg("done")],
120 Some(eng),
121 Some(Arc::new(AutoApprove)),
122 );
123 let outcome = agent
124 .run(
125 "go",
126 &Default::default(),
127 CancellationToken::new(),
128 mpsc::channel(64).0,
129 )
130 .await;
131 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
132 // Raw ledger: the closed turn's result is a trace line in the
133 // projection now (docs/design/68-context-engine.md §10).
134 let session = agent.session.lock().await;
135 let result = session
136 .message_chain()
137 .iter()
138 .flat_map(|(_, m)| m.content.iter())
139 .find_map(|b| match b {
140 ContentBlock::ToolResult {
141 content, is_error, ..
142 } => Some((content.clone(), *is_error)),
143 _ => None,
144 })
145 .expect("tool result exists");
146 assert!(result.1, "denied call must be an error result");
147 assert!(result.0.contains("denied by rule"));
148}
149
150#[tokio::test]
151async fn ask_without_approver_is_denied_with_hint() {
152 let eng = PermissionEngine::default();
153 let mut agent = multi_setup(
154 vec![bash_call("t1", "echo hi"), text_msg("ok")],
155 Some(eng),
156 None,
157 );
158 let outcome = agent
159 .run(
160 "go",
161 &Default::default(),
162 CancellationToken::new(),
163 mpsc::channel(64).0,
164 )
165 .await;
166 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
167 // Raw ledger: see the comment above on the same pattern.
168 let session = agent.session.lock().await;
169 let result = session
170 .message_chain()
171 .iter()
172 .flat_map(|(_, m)| m.content.iter())
173 .find_map(|b| match b {
174 ContentBlock::ToolResult {
175 content, is_error, ..
176 } => Some((content.clone(), *is_error)),
177 _ => None,
178 })
179 .unwrap();
180 assert!(result.0.contains("no approver available"));
181}
182
183#[tokio::test]
184async fn approved_ask_executes_the_tool() {
185 let eng = PermissionEngine::default();
186 let marker = tempdir().unwrap();
187 let cmd = format!("touch {}/marker", marker.path().display());
188 let mut agent = multi_setup(
189 vec![bash_call("t1", &cmd), text_msg("ok")],
190 Some(eng),
191 Some(Arc::new(AutoApprove)),
192 );
193 let outcome = agent
194 .run(
195 "go",
196 &Default::default(),
197 CancellationToken::new(),
198 mpsc::channel(64).0,
199 )
200 .await;
201 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
202 assert!(
203 marker.path().join("marker").exists(),
204 "approved command must have run"
205 );
206}
207
208fn first_tool_result(agent: &Agent) -> (String, bool) {
209 futures::executor::block_on(async {
210 agent
211 .session
212 .lock()
213 .await
214 .message_chain()
215 .iter()
216 .flat_map(|(_, m)| m.content.iter())
217 .find_map(|b| match b {
218 ContentBlock::ToolResult {
219 content, is_error, ..
220 } => Some((content.clone(), *is_error)),
221 _ => None,
222 })
223 .expect("tool result exists")
224 })
225}
226
227/// A live envelope pre-authorizes the calls it covers: the gate that would
228/// have gone to an approver — here, none at all — lets the covered call run.
229#[tokio::test]
230async fn a_covering_envelope_answers_the_gate_without_an_approver() {
231 let marker = tempdir().unwrap();
232 let cmd = format!("touch {}/marker", marker.path().display());
233 let mut agent = multi_setup(
234 vec![bash_call("t1", &cmd), text_msg("ok")],
235 Some(PermissionEngine::default()),
236 None,
237 );
238 agent.config.envelope_check = Some(Arc::new(|tool: &str, _: &serde_json::Value| {
239 (tool == "bash").then(|| "env-1".to_string())
240 }));
241 let outcome = agent
242 .run(
243 "go",
244 &Default::default(),
245 CancellationToken::new(),
246 mpsc::channel(64).0,
247 )
248 .await;
249 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
250 assert!(
251 marker.path().join("marker").exists(),
252 "a covered command runs inside its envelope"
253 );
254}
255
256/// An envelope that does not cover the call buys nothing, and an ask a rule
257/// raised is never one a grant stands in for.
258#[tokio::test]
259async fn an_envelope_never_answers_an_uncovered_call_or_a_rule() {
260 let uncovered = tempdir().unwrap();
261 let mut agent = multi_setup(
262 vec![
263 bash_call(
264 "t1",
265 &format!("touch {}/marker", uncovered.path().display()),
266 ),
267 text_msg("ok"),
268 ],
269 Some(PermissionEngine::default()),
270 None,
271 );
272 agent.config.envelope_check = Some(Arc::new(|_: &str, _: &serde_json::Value| None));
273 agent
274 .run(
275 "go",
276 &Default::default(),
277 CancellationToken::new(),
278 mpsc::channel(64).0,
279 )
280 .await;
281 assert!(!uncovered.path().join("marker").exists());
282 assert!(
283 first_tool_result(&agent)
284 .0
285 .contains("no approver available")
286 );
287
288 let ruled = tempdir().unwrap();
289 let mut agent = multi_setup(
290 vec![
291 bash_call("t1", &format!("touch {}/marker", ruled.path().display())),
292 text_msg("ok"),
293 ],
294 Some(PermissionEngine::from_rule_strings(&["?Bash(touch *)".to_string()]).unwrap()),
295 None,
296 );
297 agent.config.envelope_check = Some(Arc::new(|_: &str, _: &serde_json::Value| {
298 Some("env-1".to_string())
299 }));
300 agent
301 .run(
302 "go",
303 &Default::default(),
304 CancellationToken::new(),
305 mpsc::channel(64).0,
306 )
307 .await;
308 assert!(
309 !ruled.path().join("marker").exists(),
310 "an operator's ask rule reaches a person whatever was granted"
311 );
312}
313

1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2
3use std::sync::Arc;
4
5use tokio::sync::mpsc;
6use tokio_util::sync::CancellationToken;
7
8use tempfile::tempdir;
9
10use vak_agent::{Agent, AgentConfig, Approver, AutoApprove, TurnOutcome};
11use vak_llm::Provider;
12use vak_llm::types::{AssistantMessage, ContentBlock, StopReason, Usage};
13use vak_permission::PermissionEngine;
14use vak_session::SessionLog;
15use vak_session::types::{FrozenContract, SessionHeader};
16use vak_tools::bash::BashTool;
17
18fn bash_call(id: &str, cmd: &str) -> AssistantMessage {
19 AssistantMessage {
20 content: vec![ContentBlock::ToolUse {
21 id: id.into(),
22 name: "bash".into(),
23 input: serde_json::json!({"command": cmd}),
24 }],
25 stop_reason: StopReason::ToolUse,
26 usage: Usage::default(),
27 model: "test-model".into(),
28 response_id: None,
29 }
30}
31
32fn text_msg(t: &str) -> AssistantMessage {
33 AssistantMessage {
34 content: vec![ContentBlock::text(t)],
35 stop_reason: StopReason::EndTurn,
36 usage: Usage {
37 input_tokens: 1,
38 output_tokens: 1,
39 ..Default::default()
40 },
41 model: "test-model".into(),
42 response_id: None,
43 }
44}
45
46struct MultiScripted {
47 msgs: std::sync::Mutex<std::collections::VecDeque<AssistantMessage>>,
48}
49
50#[async_trait::async_trait]
51impl Provider for MultiScripted {
52 fn name(&self) -> &str {
53 "scripted"
54 }
55
56 async fn stream(
57 &self,
58 _request: vak_llm::types::ChatRequest,
59 _cancel: CancellationToken,
60 ) -> Result<vak_llm::EventStream, vak_llm::LlmError> {
61 let next = self.msgs.lock().unwrap().pop_front();
62 let (mut sink, rx) = vak_llm::stream::channel(8);
63 if let Some(m) = next {
64 sink.push(vak_llm::stream::StreamEvent::Start { partial: m.clone() });
65 sink.close_message(m).await;
66 } else {
67 sink.close_error(vak_llm::LlmError::Parse("exhausted".into()))
68 .await;
69 }
70 Ok(rx)
71 }
72}
73
74fn multi_setup(
75 responses: Vec<AssistantMessage>,
76 engine: Option<PermissionEngine>,
77 approver: Option<Arc<dyn Approver>>,
78) -> Agent {
79 let dir = tempdir().unwrap();
80 let header = SessionHeader {
81 agent: None,
82 session_id: "s".into(),
83 created_at: chrono::Utc::now(),
84 cwd: dir.path().to_path_buf(),
85 parent_session_id: None,
86 contract_id: None,
87 work_item_id: None,
88 conversation: None,
89 contract: FrozenContract {
90 app_version: "0".into(),
91 provider: "scripted".into(),
92 model: "test-model".into(),
93 route_ladder: Vec::new(),
94 route_objective: String::new(),
95 route_annotations: Vec::new(),
96 system_prompt: "sys".into(),
97 permission_mode: "workspace-write".into(),
98 capabilities: Vec::new(),
99 prompt_layers: Vec::new(),
100 },
101 };
102 let log = SessionLog::create(dir.path().join("s.jsonl"), header).unwrap();
103 let mut cfg = AgentConfig::new("sys");
104 cfg.tools = vec![Arc::new(BashTool)];
105 cfg.permission = engine.map(Arc::new);
106 cfg.approver = approver;
107 cfg.stop_policy = None;
108 std::mem::forget(dir);
109 let provider = MultiScripted {
110 msgs: std::sync::Mutex::new(responses.into_iter().collect()),
111 };
112 Agent::new(Arc::new(provider), log, cfg)
113}
114
115#[tokio::test]
116async fn deny_rule_blocks_execution_and_feeds_reason_back() {
117 let eng = PermissionEngine::from_rule_strings(&["-Bash(rm *)".to_string()]).unwrap();
118 let mut agent = multi_setup(
119 vec![bash_call("t1", "rm -rf /"), text_msg("done")],
120 Some(eng),
121 Some(Arc::new(AutoApprove)),
122 );
123 let outcome = agent
124 .run(
125 "go",
126 &Default::default(),
127 CancellationToken::new(),
128 mpsc::channel(64).0,
129 )
130 .await;
131 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
132 // Raw ledger: the closed turn's result is a trace line in the
133 // projection now (docs/design/68-context-engine.md §10).
134 let session = agent.session.lock().await;
135 let result = session
136 .message_chain()
137 .iter()
138 .flat_map(|(_, m)| m.content.iter())
139 .find_map(|b| match b {
140 ContentBlock::ToolResult {
141 content, is_error, ..
142 } => Some((content.clone(), *is_error)),
143 _ => None,
144 })
145 .expect("tool result exists");
146 assert!(result.1, "denied call must be an error result");
147 assert!(result.0.contains("denied by rule"));
148}
149
150#[tokio::test]
151async fn ask_without_approver_is_denied_with_hint() {
152 let eng = PermissionEngine::default();
153 let mut agent = multi_setup(
154 vec![bash_call("t1", "echo hi"), text_msg("ok")],
155 Some(eng),
156 None,
157 );
158 let outcome = agent
159 .run(
160 "go",
161 &Default::default(),
162 CancellationToken::new(),
163 mpsc::channel(64).0,
164 )
165 .await;
166 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
167 // Raw ledger: see the comment above on the same pattern.
168 let session = agent.session.lock().await;
169 let result = session
170 .message_chain()
171 .iter()
172 .flat_map(|(_, m)| m.content.iter())
173 .find_map(|b| match b {
174 ContentBlock::ToolResult {
175 content, is_error, ..
176 } => Some((content.clone(), *is_error)),
177 _ => None,
178 })
179 .unwrap();
180 assert!(result.0.contains("no approver available"));
181}
182
183#[tokio::test]
184async fn approved_ask_executes_the_tool() {
185 let eng = PermissionEngine::default();
186 let marker = tempdir().unwrap();
187 let cmd = format!("touch {}/marker", marker.path().display());
188 let mut agent = multi_setup(
189 vec![bash_call("t1", &cmd), text_msg("ok")],
190 Some(eng),
191 Some(Arc::new(AutoApprove)),
192 );
193 let outcome = agent
194 .run(
195 "go",
196 &Default::default(),
197 CancellationToken::new(),
198 mpsc::channel(64).0,
199 )
200 .await;
201 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
202 assert!(
203 marker.path().join("marker").exists(),
204 "approved command must have run"
205 );
206}
207
208fn first_tool_result(agent: &Agent) -> (String, bool) {
209 futures::executor::block_on(async {
210 agent
211 .session
212 .lock()
213 .await
214 .message_chain()
215 .iter()
216 .flat_map(|(_, m)| m.content.iter())
217 .find_map(|b| match b {
218 ContentBlock::ToolResult {
219 content, is_error, ..
220 } => Some((content.clone(), *is_error)),
221 _ => None,
222 })
223 .expect("tool result exists")
224 })
225}
226
227/// A live envelope pre-authorizes the calls it covers: the gate that would
228/// have gone to an approver — here, none at all — lets the covered call run.
229#[tokio::test]
230async fn a_covering_envelope_answers_the_gate_without_an_approver() {
231 let marker = tempdir().unwrap();
232 let cmd = format!("touch {}/marker", marker.path().display());
233 let mut agent = multi_setup(
234 vec![bash_call("t1", &cmd), text_msg("ok")],
235 Some(PermissionEngine::default()),
236 None,
237 );
238 agent.config.envelope_check = Some(Arc::new(|tool: &str, _: &serde_json::Value| {
239 (tool == "bash").then(|| "env-1".to_string())
240 }));
241 let outcome = agent
242 .run(
243 "go",
244 &Default::default(),
245 CancellationToken::new(),
246 mpsc::channel(64).0,
247 )
248 .await;
249 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
250 assert!(
251 marker.path().join("marker").exists(),
252 "a covered command runs inside its envelope"
253 );
254}
255
256/// An envelope that does not cover the call buys nothing, and an ask a rule
257/// raised is never one a grant stands in for.
258#[tokio::test]
259async fn an_envelope_never_answers_an_uncovered_call_or_a_rule() {
260 let uncovered = tempdir().unwrap();
261 let mut agent = multi_setup(
262 vec![
263 bash_call(
264 "t1",
265 &format!("touch {}/marker", uncovered.path().display()),
266 ),
267 text_msg("ok"),
268 ],
269 Some(PermissionEngine::default()),
270 None,
271 );
272 agent.config.envelope_check = Some(Arc::new(|_: &str, _: &serde_json::Value| None));
273 agent
274 .run(
275 "go",
276 &Default::default(),
277 CancellationToken::new(),
278 mpsc::channel(64).0,
279 )
280 .await;
281 assert!(!uncovered.path().join("marker").exists());
282 assert!(
283 first_tool_result(&agent)
284 .0
285 .contains("no approver available")
286 );
287
288 let ruled = tempdir().unwrap();
289 let mut agent = multi_setup(
290 vec![
291 bash_call("t1", &format!("touch {}/marker", ruled.path().display())),
292 text_msg("ok"),
293 ],
294 Some(PermissionEngine::from_rule_strings(&["?Bash(touch *)".to_string()]).unwrap()),
295 None,
296 );
297 agent.config.envelope_check = Some(Arc::new(|_: &str, _: &serde_json::Value| {
298 Some("env-1".to_string())
299 }));
300 agent
301 .run(
302 "go",
303 &Default::default(),
304 CancellationToken::new(),
305 mpsc::channel(64).0,
306 )
307 .await;
308 assert!(
309 !ruled.path().join("marker").exists(),
310 "an operator's ask rule reaches a person whatever was granted"
311 );
312}
313

Permissions

Indexing the workspace…

Permissions