Vakyartha documentation

1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2
3//! Cards are Vak's own display channel: a tool that `presents_cards()` runs in
4//! every permission mode without an approval, and without a denial in
5//! read-only. Real trigger: "Vak wants to use emit_metric_card — this needs
6//! your approval" shown under a card that had already rendered.
7
8use std::collections::VecDeque;
9use std::sync::atomic::{AtomicUsize, Ordering};
10use std::sync::{Arc, Mutex};
11
12use async_trait::async_trait;
13use tokio::sync::mpsc;
14use tokio_util::sync::CancellationToken;
15
16use vak_agent::{Agent, AgentConfig, AgentEvent, TurnOutcome};
17use vak_llm::stream;
18use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
19use vak_llm::{EventStream, LlmError, Provider};
20use vak_permission::{Mode, PermissionEngine};
21use vak_session::types::{FrozenContract, SessionHeader};
22use vak_session::{SessionLog, SessionPath};
23use vak_tools::context::ToolContext;
24use vak_tools::{Tool, ToolOutput};
25
26struct Scripted(Mutex<VecDeque<AssistantMessage>>);
27
28#[async_trait]
29impl Provider for Scripted {
30 fn name(&self) -> &str {
31 "scripted"
32 }
33
34 async fn stream(
35 &self,
36 _request: ChatRequest,
37 _cancel: CancellationToken,
38 ) -> Result<EventStream, LlmError> {
39 let next = self.0.lock().unwrap().pop_front();
40 let (mut sink, rx) = stream::channel(64);
41 match next {
42 Some(m) => {
43 sink.push(stream::StreamEvent::Start { partial: m.clone() });
44 sink.close_message(m).await;
45 }
46 None => sink.close_error(LlmError::Parse("exhausted".into())).await,
47 }
48 Ok(rx)
49 }
50}
51
52struct CardTool {
53 name: &'static str,
54 presents: bool,
55 runs: Arc<AtomicUsize>,
56}
57
58#[async_trait]
59impl Tool for CardTool {
60 fn name(&self) -> &str {
61 self.name
62 }
63 fn description(&self) -> &str {
64 "test stand-in"
65 }
66 fn schema(&self) -> serde_json::Value {
67 serde_json::json!({"type": "object"})
68 }
69 fn presents_cards(&self) -> bool {
70 self.presents
71 }
72 async fn execute(&self, _args: &serde_json::Value, _ctx: &ToolContext) -> ToolOutput {
73 self.runs.fetch_add(1, Ordering::SeqCst);
74 ToolOutput::ok("Card displayed to the user.")
75 }
76}
77
78fn msg(content: Vec<ContentBlock>, stop_reason: StopReason) -> AssistantMessage {
79 AssistantMessage {
80 content,
81 stop_reason,
82 usage: Usage {
83 input_tokens: 1,
84 output_tokens: 1,
85 ..Default::default()
86 },
87 model: "test-model".into(),
88 response_id: None,
89 }
90}
91
92/// Runs one turn that calls `tool` once; returns (executions, approvals asked).
93async fn call_once(mode: Mode, tool: &'static str, presents: bool) -> (usize, usize) {
94 run_calls(mode, tool, presents, 1).await
95}
96
97async fn call_twice(_dir: &tempfile::TempDir) -> (usize, usize) {
98 run_calls(Mode::WorkspaceWrite, "emit_metric_card", true, 2).await
99}
100
101async fn run_calls(mode: Mode, tool: &'static str, presents: bool, calls: usize) -> (usize, usize) {
102 let dir = tempfile::tempdir().unwrap();
103 let home = dir.path().join("home");
104 std::fs::create_dir_all(&home).unwrap();
105 let header = SessionHeader {
106 agent: None,
107 session_id: "card-perm".into(),
108 created_at: chrono::Utc::now(),
109 cwd: dir.path().to_path_buf(),
110 parent_session_id: None,
111 contract_id: None,
112 work_item_id: None,
113 conversation: None,
114 contract: FrozenContract {
115 app_version: "0".into(),
116 provider: "scripted".into(),
117 model: "test-model".into(),
118 route_ladder: Vec::new(),
119 route_objective: String::new(),
120 route_annotations: Vec::new(),
121 system_prompt: "sys".into(),
122 permission_mode: "workspace-write".into(),
123 capabilities: Vec::new(),
124 prompt_layers: Vec::new(),
125 },
126 };
127 let log = SessionLog::create(
128 SessionPath::new_session_file(&home, dir.path(), "card-perm"),
129 header,
130 )
131 .unwrap();
132 let runs = Arc::new(AtomicUsize::new(0));
133 let mut cfg = AgentConfig::new("sys");
134 cfg.model = "test-model".into();
135 cfg.mode = mode;
136 cfg.permission = Some(Arc::new(
137 PermissionEngine::default().with_presenting_tools(["emit_metric_card".to_string()]),
138 ));
139 cfg.tools = vec![Arc::new(CardTool {
140 name: tool,
141 presents,
142 runs: runs.clone(),
143 })];
144 let mut agent = Agent::new(
145 Arc::new(Scripted(Mutex::new({
146 let mut script: VecDeque<AssistantMessage> = (0..calls)
147 .map(|n| {
148 msg(
149 vec![ContentBlock::ToolUse {
150 id: format!("c{n}"),
151 name: tool.into(),
152 input: serde_json::json!({}),
153 }],
154 StopReason::ToolUse,
155 )
156 })
157 .collect();
158 script.push_back(msg(vec![ContentBlock::text("done")], StopReason::EndTurn));
159 script
160 }))),
161 log,
162 cfg,
163 );
164 let (tx, mut rx) = mpsc::channel(256);
165 let outcome = agent
166 .run("show it", &Default::default(), CancellationToken::new(), tx)
167 .await;
168 assert!(
169 matches!(outcome, TurnOutcome::Completed { .. }),
170 "{outcome:?}"
171 );
172 let mut approvals = 0;
173 while let Ok(event) = rx.try_recv() {
174 if matches!(event, AgentEvent::ApprovalRequested { .. }) {
175 approvals += 1;
176 }
177 }
178 (runs.load(Ordering::SeqCst), approvals)
179}
180
181#[tokio::test]
182async fn card_tools_run_without_approval_in_every_mode() {
183 for mode in [Mode::WorkspaceWrite, Mode::ReadOnly, Mode::FullAccess] {
184 assert_eq!(
185 call_once(mode, "emit_metric_card", true).await,
186 (1, 0),
187 "{mode:?}"
188 );
189 }
190}
191
192#[test]
193fn a_tool_the_host_did_not_declare_as_presenting_still_needs_approval() {
194 let engine =
195 PermissionEngine::default().with_presenting_tools(["emit_metric_card".to_string()]);
196 let cwd = std::env::temp_dir();
197 let decision = engine.evaluate(
198 "emit_lookalike_card",
199 &serde_json::json!({}),
200 Mode::WorkspaceWrite,
201 &cwd,
202 );
203 assert!(
204 matches!(decision, vak_permission::Decision::Ask { .. }),
205 "only tools the host declared as presenting are exempt: {decision:?}"
206 );
207}
208
209/// A card whose schema the model cannot satisfy.
210struct StrictCardTool;
211
212#[async_trait]
213impl Tool for StrictCardTool {
214 fn name(&self) -> &str {
215 "emit_metric_card"
216 }
217 fn description(&self) -> &str {
218 "test stand-in with a required payload"
219 }
220 fn schema(&self) -> serde_json::Value {
221 serde_json::json!({
222 "type": "object",
223 "properties": {"payload": {"type": "object"}},
224 "required": ["payload"]
225 })
226 }
227 fn presents_cards(&self) -> bool {
228 true
229 }
230 async fn execute(&self, _args: &serde_json::Value, _ctx: &ToolContext) -> ToolOutput {
231 ToolOutput::ok("Card displayed to the user.")
232 }
233}
234
235/// A card is a presentation of the answer, not part of the work. One that
236/// fails validation is simply not shown; a complete prose answer after it
237/// ends the turn instead of being sent back to repair the card. Measured
238/// live: a small model could not build the card's arguments, and the stop
239/// gate's demand to repair it turned a correct, sourced answer into a
240/// failed turn.
241#[tokio::test]
242async fn a_failed_card_does_not_hold_back_a_complete_answer() {
243 let dir = tempfile::tempdir().unwrap();
244 let home = dir.path().join("home");
245 std::fs::create_dir_all(&home).unwrap();
246 let header = SessionHeader {
247 agent: None,
248 session_id: "card-failed".into(),
249 created_at: chrono::Utc::now(),
250 cwd: dir.path().to_path_buf(),
251 parent_session_id: None,
252 contract_id: None,
253 work_item_id: None,
254 conversation: None,
255 contract: FrozenContract {
256 app_version: "0".into(),
257 provider: "scripted".into(),
258 model: "test-model".into(),
259 route_ladder: Vec::new(),
260 route_objective: String::new(),
261 route_annotations: Vec::new(),
262 system_prompt: "sys".into(),
263 permission_mode: "workspace-write".into(),
264 capabilities: Vec::new(),
265 prompt_layers: Vec::new(),
266 },
267 };
268 let log = SessionLog::create(
269 SessionPath::new_session_file(&home, dir.path(), "card-failed"),
270 header,
271 )
272 .unwrap();
273 let mut cfg = AgentConfig::new("sys");
274 cfg.model = "test-model".into();
275 cfg.mode = Mode::WorkspaceWrite;
276 cfg.permission = Some(Arc::new(
277 PermissionEngine::default().with_presenting_tools(["emit_metric_card".to_string()]),
278 ));
279 cfg.tools = vec![Arc::new(StrictCardTool)];
280 let answer = "Copper trades at about $6.70 per pound, according to the exchange quote.";
281 let mut agent = Agent::new(
282 Arc::new(Scripted(Mutex::new(VecDeque::from([
283 msg(
284 vec![ContentBlock::ToolUse {
285 id: "c0".into(),
286 name: "emit_metric_card".into(),
287 input: serde_json::json!({"metric_data": [1, 2]}),
288 }],
289 StopReason::ToolUse,
290 ),
291 msg(vec![ContentBlock::text(answer)], StopReason::EndTurn),
292 ])))),
293 log,
294 cfg,
295 );
296 let outcome = agent
297 .run(
298 "what is copper trading at",
299 &Default::default(),
300 CancellationToken::new(),
301 mpsc::channel(256).0,
302 )
303 .await;
304 assert!(
305 matches!(&outcome, TurnOutcome::Completed { response } if response.text_content() == answer),
306 "{outcome:?}"
307 );
308 let nudged = agent
309 .session
310 .lock()
311 .await
312 .message_chain()
313 .iter()
314 .flat_map(|(_, message)| message.content.iter())
315 .any(|block| matches!(block, ContentBlock::Text { text } if text.contains("[stop-guard]")));
316 assert!(!nudged, "the answer was sent back to repair a card");
317}
318
319#[tokio::test]
320async fn an_identical_card_call_is_a_quiet_no_op_not_a_prompt_or_a_failure() {
321 // First call runs; the repeat is acked without running, is not an error (which would
322 // force another model turn) and never becomes an approval.
323 let dir = tempfile::tempdir().unwrap();
324 let (runs, approvals) = call_twice(&dir).await;
325 assert_eq!((runs, approvals), (1, 0));
326}
327

1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2
3//! Cards are Vak's own display channel: a tool that `presents_cards()` runs in
4//! every permission mode without an approval, and without a denial in
5//! read-only. Real trigger: "Vak wants to use emit_metric_card — this needs
6//! your approval" shown under a card that had already rendered.
7
8use std::collections::VecDeque;
9use std::sync::atomic::{AtomicUsize, Ordering};
10use std::sync::{Arc, Mutex};
11
12use async_trait::async_trait;
13use tokio::sync::mpsc;
14use tokio_util::sync::CancellationToken;
15
16use vak_agent::{Agent, AgentConfig, AgentEvent, TurnOutcome};
17use vak_llm::stream;
18use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
19use vak_llm::{EventStream, LlmError, Provider};
20use vak_permission::{Mode, PermissionEngine};
21use vak_session::types::{FrozenContract, SessionHeader};
22use vak_session::{SessionLog, SessionPath};
23use vak_tools::context::ToolContext;
24use vak_tools::{Tool, ToolOutput};
25
26struct Scripted(Mutex<VecDeque<AssistantMessage>>);
27
28#[async_trait]
29impl Provider for Scripted {
30 fn name(&self) -> &str {
31 "scripted"
32 }
33
34 async fn stream(
35 &self,
36 _request: ChatRequest,
37 _cancel: CancellationToken,
38 ) -> Result<EventStream, LlmError> {
39 let next = self.0.lock().unwrap().pop_front();
40 let (mut sink, rx) = stream::channel(64);
41 match next {
42 Some(m) => {
43 sink.push(stream::StreamEvent::Start { partial: m.clone() });
44 sink.close_message(m).await;
45 }
46 None => sink.close_error(LlmError::Parse("exhausted".into())).await,
47 }
48 Ok(rx)
49 }
50}
51
52struct CardTool {
53 name: &'static str,
54 presents: bool,
55 runs: Arc<AtomicUsize>,
56}
57
58#[async_trait]
59impl Tool for CardTool {
60 fn name(&self) -> &str {
61 self.name
62 }
63 fn description(&self) -> &str {
64 "test stand-in"
65 }
66 fn schema(&self) -> serde_json::Value {
67 serde_json::json!({"type": "object"})
68 }
69 fn presents_cards(&self) -> bool {
70 self.presents
71 }
72 async fn execute(&self, _args: &serde_json::Value, _ctx: &ToolContext) -> ToolOutput {
73 self.runs.fetch_add(1, Ordering::SeqCst);
74 ToolOutput::ok("Card displayed to the user.")
75 }
76}
77
78fn msg(content: Vec<ContentBlock>, stop_reason: StopReason) -> AssistantMessage {
79 AssistantMessage {
80 content,
81 stop_reason,
82 usage: Usage {
83 input_tokens: 1,
84 output_tokens: 1,
85 ..Default::default()
86 },
87 model: "test-model".into(),
88 response_id: None,
89 }
90}
91
92/// Runs one turn that calls `tool` once; returns (executions, approvals asked).
93async fn call_once(mode: Mode, tool: &'static str, presents: bool) -> (usize, usize) {
94 run_calls(mode, tool, presents, 1).await
95}
96
97async fn call_twice(_dir: &tempfile::TempDir) -> (usize, usize) {
98 run_calls(Mode::WorkspaceWrite, "emit_metric_card", true, 2).await
99}
100
101async fn run_calls(mode: Mode, tool: &'static str, presents: bool, calls: usize) -> (usize, usize) {
102 let dir = tempfile::tempdir().unwrap();
103 let home = dir.path().join("home");
104 std::fs::create_dir_all(&home).unwrap();
105 let header = SessionHeader {
106 agent: None,
107 session_id: "card-perm".into(),
108 created_at: chrono::Utc::now(),
109 cwd: dir.path().to_path_buf(),
110 parent_session_id: None,
111 contract_id: None,
112 work_item_id: None,
113 conversation: None,
114 contract: FrozenContract {
115 app_version: "0".into(),
116 provider: "scripted".into(),
117 model: "test-model".into(),
118 route_ladder: Vec::new(),
119 route_objective: String::new(),
120 route_annotations: Vec::new(),
121 system_prompt: "sys".into(),
122 permission_mode: "workspace-write".into(),
123 capabilities: Vec::new(),
124 prompt_layers: Vec::new(),
125 },
126 };
127 let log = SessionLog::create(
128 SessionPath::new_session_file(&home, dir.path(), "card-perm"),
129 header,
130 )
131 .unwrap();
132 let runs = Arc::new(AtomicUsize::new(0));
133 let mut cfg = AgentConfig::new("sys");
134 cfg.model = "test-model".into();
135 cfg.mode = mode;
136 cfg.permission = Some(Arc::new(
137 PermissionEngine::default().with_presenting_tools(["emit_metric_card".to_string()]),
138 ));
139 cfg.tools = vec![Arc::new(CardTool {
140 name: tool,
141 presents,
142 runs: runs.clone(),
143 })];
144 let mut agent = Agent::new(
145 Arc::new(Scripted(Mutex::new({
146 let mut script: VecDeque<AssistantMessage> = (0..calls)
147 .map(|n| {
148 msg(
149 vec![ContentBlock::ToolUse {
150 id: format!("c{n}"),
151 name: tool.into(),
152 input: serde_json::json!({}),
153 }],
154 StopReason::ToolUse,
155 )
156 })
157 .collect();
158 script.push_back(msg(vec![ContentBlock::text("done")], StopReason::EndTurn));
159 script
160 }))),
161 log,
162 cfg,
163 );
164 let (tx, mut rx) = mpsc::channel(256);
165 let outcome = agent
166 .run("show it", &Default::default(), CancellationToken::new(), tx)
167 .await;
168 assert!(
169 matches!(outcome, TurnOutcome::Completed { .. }),
170 "{outcome:?}"
171 );
172 let mut approvals = 0;
173 while let Ok(event) = rx.try_recv() {
174 if matches!(event, AgentEvent::ApprovalRequested { .. }) {
175 approvals += 1;
176 }
177 }
178 (runs.load(Ordering::SeqCst), approvals)
179}
180
181#[tokio::test]
182async fn card_tools_run_without_approval_in_every_mode() {
183 for mode in [Mode::WorkspaceWrite, Mode::ReadOnly, Mode::FullAccess] {
184 assert_eq!(
185 call_once(mode, "emit_metric_card", true).await,
186 (1, 0),
187 "{mode:?}"
188 );
189 }
190}
191
192#[test]
193fn a_tool_the_host_did_not_declare_as_presenting_still_needs_approval() {
194 let engine =
195 PermissionEngine::default().with_presenting_tools(["emit_metric_card".to_string()]);
196 let cwd = std::env::temp_dir();
197 let decision = engine.evaluate(
198 "emit_lookalike_card",
199 &serde_json::json!({}),
200 Mode::WorkspaceWrite,
201 &cwd,
202 );
203 assert!(
204 matches!(decision, vak_permission::Decision::Ask { .. }),
205 "only tools the host declared as presenting are exempt: {decision:?}"
206 );
207}
208
209/// A card whose schema the model cannot satisfy.
210struct StrictCardTool;
211
212#[async_trait]
213impl Tool for StrictCardTool {
214 fn name(&self) -> &str {
215 "emit_metric_card"
216 }
217 fn description(&self) -> &str {
218 "test stand-in with a required payload"
219 }
220 fn schema(&self) -> serde_json::Value {
221 serde_json::json!({
222 "type": "object",
223 "properties": {"payload": {"type": "object"}},
224 "required": ["payload"]
225 })
226 }
227 fn presents_cards(&self) -> bool {
228 true
229 }
230 async fn execute(&self, _args: &serde_json::Value, _ctx: &ToolContext) -> ToolOutput {
231 ToolOutput::ok("Card displayed to the user.")
232 }
233}
234
235/// A card is a presentation of the answer, not part of the work. One that
236/// fails validation is simply not shown; a complete prose answer after it
237/// ends the turn instead of being sent back to repair the card. Measured
238/// live: a small model could not build the card's arguments, and the stop
239/// gate's demand to repair it turned a correct, sourced answer into a
240/// failed turn.
241#[tokio::test]
242async fn a_failed_card_does_not_hold_back_a_complete_answer() {
243 let dir = tempfile::tempdir().unwrap();
244 let home = dir.path().join("home");
245 std::fs::create_dir_all(&home).unwrap();
246 let header = SessionHeader {
247 agent: None,
248 session_id: "card-failed".into(),
249 created_at: chrono::Utc::now(),
250 cwd: dir.path().to_path_buf(),
251 parent_session_id: None,
252 contract_id: None,
253 work_item_id: None,
254 conversation: None,
255 contract: FrozenContract {
256 app_version: "0".into(),
257 provider: "scripted".into(),
258 model: "test-model".into(),
259 route_ladder: Vec::new(),
260 route_objective: String::new(),
261 route_annotations: Vec::new(),
262 system_prompt: "sys".into(),
263 permission_mode: "workspace-write".into(),
264 capabilities: Vec::new(),
265 prompt_layers: Vec::new(),
266 },
267 };
268 let log = SessionLog::create(
269 SessionPath::new_session_file(&home, dir.path(), "card-failed"),
270 header,
271 )
272 .unwrap();
273 let mut cfg = AgentConfig::new("sys");
274 cfg.model = "test-model".into();
275 cfg.mode = Mode::WorkspaceWrite;
276 cfg.permission = Some(Arc::new(
277 PermissionEngine::default().with_presenting_tools(["emit_metric_card".to_string()]),
278 ));
279 cfg.tools = vec![Arc::new(StrictCardTool)];
280 let answer = "Copper trades at about $6.70 per pound, according to the exchange quote.";
281 let mut agent = Agent::new(
282 Arc::new(Scripted(Mutex::new(VecDeque::from([
283 msg(
284 vec![ContentBlock::ToolUse {
285 id: "c0".into(),
286 name: "emit_metric_card".into(),
287 input: serde_json::json!({"metric_data": [1, 2]}),
288 }],
289 StopReason::ToolUse,
290 ),
291 msg(vec![ContentBlock::text(answer)], StopReason::EndTurn),
292 ])))),
293 log,
294 cfg,
295 );
296 let outcome = agent
297 .run(
298 "what is copper trading at",
299 &Default::default(),
300 CancellationToken::new(),
301 mpsc::channel(256).0,
302 )
303 .await;
304 assert!(
305 matches!(&outcome, TurnOutcome::Completed { response } if response.text_content() == answer),
306 "{outcome:?}"
307 );
308 let nudged = agent
309 .session
310 .lock()
311 .await
312 .message_chain()
313 .iter()
314 .flat_map(|(_, message)| message.content.iter())
315 .any(|block| matches!(block, ContentBlock::Text { text } if text.contains("[stop-guard]")));
316 assert!(!nudged, "the answer was sent back to repair a card");
317}
318
319#[tokio::test]
320async fn an_identical_card_call_is_a_quiet_no_op_not_a_prompt_or_a_failure() {
321 // First call runs; the repeat is acked without running, is not an error (which would
322 // force another model turn) and never becomes an approval.
323 let dir = tempfile::tempdir().unwrap();
324 let (runs, approvals) = call_twice(&dir).await;
325 assert_eq!((runs, approvals), (1, 0));
326}
327

Card Permission

Indexing the workspace…

Card Permission