Vakyartha documentation

1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2
3//! docs/design/68-context-engine.md §10/§3: a TurnCard is written once a
4//! turn closes; a fence-path Presentation is written once and deduped
5//! against a tool-emitted card with the same payload; `recall({ id })`
6//! returns evidence content verbatim.
7
8use std::collections::VecDeque;
9use std::sync::{Arc, Mutex};
10
11use async_trait::async_trait;
12use tokio::sync::mpsc;
13use tokio_util::sync::CancellationToken;
14
15use tempfile::tempdir;
16
17use vak_agent::{Agent, AgentConfig, TurnOutcome};
18use vak_llm::stream;
19use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
20use vak_llm::{EventStream, LlmError, Provider};
21use vak_permission::PermissionEngine;
22use vak_session::types::{FrozenContract, PresentationSource, SessionHeader};
23use vak_session::{SessionLog, SessionPath};
24use vak_tools::PresentationCard;
25use vak_tools::bash::BashTool;
26use vak_tools::context::ToolContext;
27use vak_tools::{Tool, ToolOutput};
28
29struct Scripted {
30 responses: Mutex<VecDeque<AssistantMessage>>,
31}
32
33#[async_trait]
34impl Provider for Scripted {
35 fn name(&self) -> &str {
36 "scripted"
37 }
38
39 async fn stream(
40 &self,
41 _request: ChatRequest,
42 _cancel: CancellationToken,
43 ) -> Result<EventStream, LlmError> {
44 let next = self.responses.lock().unwrap().pop_front();
45 let (mut sink, rx) = stream::channel(64);
46 match next {
47 Some(m) => {
48 sink.push(stream::StreamEvent::Start { partial: m.clone() });
49 sink.close_message(m).await;
50 }
51 None => sink.close_error(LlmError::Parse("exhausted".into())).await,
52 }
53 Ok(rx)
54 }
55}
56
57fn text_msg(t: &str) -> AssistantMessage {
58 AssistantMessage {
59 content: vec![ContentBlock::text(t)],
60 stop_reason: StopReason::EndTurn,
61 usage: Usage {
62 input_tokens: 1,
63 output_tokens: 1,
64 ..Default::default()
65 },
66 model: "test-model".into(),
67 response_id: None,
68 }
69}
70
71fn tool_call_msg(id: &str, name: &str, input: serde_json::Value) -> AssistantMessage {
72 AssistantMessage {
73 content: vec![ContentBlock::ToolUse {
74 id: id.into(),
75 name: name.into(),
76 input,
77 }],
78 stop_reason: StopReason::ToolUse,
79 usage: Usage::default(),
80 model: "test-model".into(),
81 response_id: None,
82 }
83}
84
85/// Stand-in for `vak_core::presentation_tools::EmitCardTool`.
86struct FakeEmitChartCard;
87
88#[async_trait]
89impl Tool for FakeEmitChartCard {
90 fn name(&self) -> &str {
91 "emit_chart_card"
92 }
93 fn description(&self) -> &str {
94 "test stand-in"
95 }
96 fn schema(&self) -> serde_json::Value {
97 serde_json::json!({"type": "object"})
98 }
99 fn presents_cards(&self) -> bool {
100 true
101 }
102 async fn execute(&self, args: &serde_json::Value, _ctx: &ToolContext) -> ToolOutput {
103 let envelope = serde_json::json!({
104 "semantic_type": "chart",
105 "payload": args.get("payload").cloned().unwrap_or(serde_json::json!({})),
106 });
107 ToolOutput::ok(envelope.to_string())
108 }
109}
110
111/// Stand-in for `vak_core::presentation_tools::presentation_info` /
112/// `emit_tool_for`, minimal enough to exercise the fence path without
113/// pulling in vak-core (vak-agent has no dependency on it).
114fn fake_rebuild() -> vak_agent::PresentationRebuild {
115 Arc::new(|_name, input| {
116 let semantic_type = input.get("semantic_type")?.as_str()?.to_string();
117 let payload = vak_session::types::canonicalize_json(input.get("payload")?);
118 let title = payload
119 .get("title")
120 .and_then(|v| v.as_str())
121 .unwrap_or("chart")
122 .to_string();
123 Some(PresentationCard {
124 semantic_type,
125 skill_id: "test".into(),
126 skill_version: "1".into(),
127 schema_version: 1,
128 payload,
129 title,
130 identity_digest: "digest".into(),
131 })
132 })
133}
134
135fn build_agent(
136 dir: &tempfile::TempDir,
137 session_id: &str,
138 responses: Vec<AssistantMessage>,
139 tools: Vec<Arc<dyn Tool>>,
140 with_presentation_rebuild: bool,
141) -> Agent {
142 let header = SessionHeader {
143 agent: None,
144 session_id: session_id.into(),
145 created_at: chrono::Utc::now(),
146 cwd: dir.path().to_path_buf(),
147 parent_session_id: None,
148 contract_id: None,
149 work_item_id: None,
150 conversation: None,
151 contract: FrozenContract {
152 app_version: "0".into(),
153 provider: "scripted".into(),
154 model: "test-model".into(),
155 route_ladder: Vec::new(),
156 route_objective: String::new(),
157 route_annotations: Vec::new(),
158 system_prompt: "sys".into(),
159 permission_mode: "full-access".into(),
160 capabilities: Vec::new(),
161 prompt_layers: Vec::new(),
162 },
163 };
164 let home = dir.path().join("home");
165 std::fs::create_dir_all(&home).unwrap();
166 let log = SessionLog::create(
167 SessionPath::new_session_file(&home, dir.path(), session_id),
168 header,
169 )
170 .unwrap();
171 let mut cfg = AgentConfig::new("sys");
172 cfg.model = "test-model".into();
173 cfg.mode = vak_permission::Mode::FullAccess;
174 cfg.permission = Some(Arc::new(PermissionEngine::default()));
175 cfg.approver = Some(Arc::new(vak_agent::AutoApprove));
176 cfg.tools = tools;
177 if with_presentation_rebuild {
178 cfg.presentation_rebuild = Some(fake_rebuild());
179 }
180 Agent::new(
181 Arc::new(Scripted {
182 responses: Mutex::new(VecDeque::from(responses)),
183 }),
184 log,
185 cfg,
186 )
187}
188
189#[tokio::test]
190async fn turn_card_is_written_at_close() {
191 let dir = tempdir().unwrap();
192 let mut agent = build_agent(&dir, "card-close", vec![text_msg("42")], vec![], false);
193 let outcome = agent
194 .run(
195 "what is six times seven",
196 &Default::default(),
197 CancellationToken::new(),
198 mpsc::channel(64).0,
199 )
200 .await;
201 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
202
203 let session = agent.session.lock().await;
204 let cards = session.turn_cards();
205 assert_eq!(cards.len(), 1, "exactly one TurnCard written at close");
206 assert_eq!(cards[0].1.outcome, "completed");
207 assert_eq!(cards[0].1.answered.narration, "42");
208 assert!(cards[0].1.answered.presentations.is_empty());
209}
210
211#[tokio::test]
212async fn fence_presentation_is_written_once() {
213 let dir = tempdir().unwrap();
214 let fence = "Here's a fresh chart.\n\n```vak\n{\"semantic_type\":\"chart\",\"payload\":{\"title\":\"Fresh\",\"series\":[1,2,3]}}\n```";
215 let mut agent = build_agent(&dir, "fence-write", vec![text_msg(fence)], vec![], true);
216 let outcome = agent
217 .run(
218 "show me a chart",
219 &Default::default(),
220 CancellationToken::new(),
221 mpsc::channel(64).0,
222 )
223 .await;
224 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
225
226 let session = agent.session.lock().await;
227 let presentations = session.presentations();
228 assert_eq!(
229 presentations.len(),
230 1,
231 "the fence must write one presentation"
232 );
233 assert!(matches!(
234 presentations[0].1.source,
235 PresentationSource::Fence { .. }
236 ));
237 assert_eq!(presentations[0].1.title, "Fresh");
238}
239
240#[tokio::test]
241async fn fence_presentation_deduped_against_tool_emitted_card() {
242 let dir = tempdir().unwrap();
243 // The final accepted answer still repeats the just-emitted card as a
244 // `vak` fence with the IDENTICAL payload (the observed real bug this
245 // rule exists for) — the duplicate-card-check repair nudge fires once,
246 // and the model ignores it (attempt two repeats it too), so the run
247 // completes with the duplicate fence still present in the final text.
248 let duplicate_fence = "Here's the chart you asked for.\n\n```vak\n{\"semantic_type\":\"chart\",\"payload\":{\"title\":\"Sales\",\"series\":[1,2,3]}}\n```";
249 let mut agent = build_agent(
250 &dir,
251 "fence-dedup",
252 vec![
253 tool_call_msg(
254 "call_1",
255 "emit_chart_card",
256 serde_json::json!({"semantic_type": "chart", "payload": {"title": "Sales", "series": [1, 2, 3]}}),
257 ),
258 text_msg(duplicate_fence),
259 text_msg(duplicate_fence),
260 ],
261 vec![Arc::new(FakeEmitChartCard)],
262 true,
263 );
264 let outcome = agent
265 .run(
266 "show me the sales chart",
267 &Default::default(),
268 CancellationToken::new(),
269 mpsc::channel(64).0,
270 )
271 .await;
272 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
273
274 let session = agent.session.lock().await;
275 let presentations = session.presentations();
276 assert_eq!(
277 presentations.len(),
278 1,
279 "the duplicate fence must not add a second presentation: {presentations:?}"
280 );
281 assert!(matches!(
282 presentations[0].1.source,
283 PresentationSource::ToolCall { .. }
284 ));
285}
286
287#[tokio::test]
288async fn recall_by_id_returns_the_full_evidence_content() {
289 let dir = tempdir().unwrap();
290 let mut agent = build_agent(
291 &dir,
292 "recall-id",
293 vec![
294 tool_call_msg(
295 "t1",
296 "bash",
297 serde_json::json!({"command": "printf 'HELLO WORLD'"}),
298 ),
299 tool_call_msg("t2", "recall", serde_json::json!({"id": "t1"})),
300 text_msg("done"),
301 ],
302 vec![Arc::new(BashTool)],
303 false,
304 );
305 let outcome = agent
306 .run(
307 "run the command then recall it",
308 &Default::default(),
309 CancellationToken::new(),
310 mpsc::channel(64).0,
311 )
312 .await;
313 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
314
315 let session = agent.session.lock().await;
316 let messages = session.message_chain();
317 let content_for = |id: &str| -> String {
318 messages
319 .iter()
320 .flat_map(|(_, m)| m.content.iter())
321 .find_map(|b| match b {
322 ContentBlock::ToolResult {
323 tool_use_id,
324 content,
325 ..
326 } if tool_use_id == id => Some(content.clone()),
327 _ => None,
328 })
329 .unwrap_or_else(|| panic!("no tool result for {id}"))
330 };
331 let original = content_for("t1");
332 let recalled = content_for("t2");
333 assert!(original.contains("HELLO WORLD"));
334 assert_eq!(
335 recalled, original,
336 "recall({{ id }}) must return the evidence content verbatim"
337 );
338}
339

1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2
3//! docs/design/68-context-engine.md §10/§3: a TurnCard is written once a
4//! turn closes; a fence-path Presentation is written once and deduped
5//! against a tool-emitted card with the same payload; `recall({ id })`
6//! returns evidence content verbatim.
7
8use std::collections::VecDeque;
9use std::sync::{Arc, Mutex};
10
11use async_trait::async_trait;
12use tokio::sync::mpsc;
13use tokio_util::sync::CancellationToken;
14
15use tempfile::tempdir;
16
17use vak_agent::{Agent, AgentConfig, TurnOutcome};
18use vak_llm::stream;
19use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
20use vak_llm::{EventStream, LlmError, Provider};
21use vak_permission::PermissionEngine;
22use vak_session::types::{FrozenContract, PresentationSource, SessionHeader};
23use vak_session::{SessionLog, SessionPath};
24use vak_tools::PresentationCard;
25use vak_tools::bash::BashTool;
26use vak_tools::context::ToolContext;
27use vak_tools::{Tool, ToolOutput};
28
29struct Scripted {
30 responses: Mutex<VecDeque<AssistantMessage>>,
31}
32
33#[async_trait]
34impl Provider for Scripted {
35 fn name(&self) -> &str {
36 "scripted"
37 }
38
39 async fn stream(
40 &self,
41 _request: ChatRequest,
42 _cancel: CancellationToken,
43 ) -> Result<EventStream, LlmError> {
44 let next = self.responses.lock().unwrap().pop_front();
45 let (mut sink, rx) = stream::channel(64);
46 match next {
47 Some(m) => {
48 sink.push(stream::StreamEvent::Start { partial: m.clone() });
49 sink.close_message(m).await;
50 }
51 None => sink.close_error(LlmError::Parse("exhausted".into())).await,
52 }
53 Ok(rx)
54 }
55}
56
57fn text_msg(t: &str) -> AssistantMessage {
58 AssistantMessage {
59 content: vec![ContentBlock::text(t)],
60 stop_reason: StopReason::EndTurn,
61 usage: Usage {
62 input_tokens: 1,
63 output_tokens: 1,
64 ..Default::default()
65 },
66 model: "test-model".into(),
67 response_id: None,
68 }
69}
70
71fn tool_call_msg(id: &str, name: &str, input: serde_json::Value) -> AssistantMessage {
72 AssistantMessage {
73 content: vec![ContentBlock::ToolUse {
74 id: id.into(),
75 name: name.into(),
76 input,
77 }],
78 stop_reason: StopReason::ToolUse,
79 usage: Usage::default(),
80 model: "test-model".into(),
81 response_id: None,
82 }
83}
84
85/// Stand-in for `vak_core::presentation_tools::EmitCardTool`.
86struct FakeEmitChartCard;
87
88#[async_trait]
89impl Tool for FakeEmitChartCard {
90 fn name(&self) -> &str {
91 "emit_chart_card"
92 }
93 fn description(&self) -> &str {
94 "test stand-in"
95 }
96 fn schema(&self) -> serde_json::Value {
97 serde_json::json!({"type": "object"})
98 }
99 fn presents_cards(&self) -> bool {
100 true
101 }
102 async fn execute(&self, args: &serde_json::Value, _ctx: &ToolContext) -> ToolOutput {
103 let envelope = serde_json::json!({
104 "semantic_type": "chart",
105 "payload": args.get("payload").cloned().unwrap_or(serde_json::json!({})),
106 });
107 ToolOutput::ok(envelope.to_string())
108 }
109}
110
111/// Stand-in for `vak_core::presentation_tools::presentation_info` /
112/// `emit_tool_for`, minimal enough to exercise the fence path without
113/// pulling in vak-core (vak-agent has no dependency on it).
114fn fake_rebuild() -> vak_agent::PresentationRebuild {
115 Arc::new(|_name, input| {
116 let semantic_type = input.get("semantic_type")?.as_str()?.to_string();
117 let payload = vak_session::types::canonicalize_json(input.get("payload")?);
118 let title = payload
119 .get("title")
120 .and_then(|v| v.as_str())
121 .unwrap_or("chart")
122 .to_string();
123 Some(PresentationCard {
124 semantic_type,
125 skill_id: "test".into(),
126 skill_version: "1".into(),
127 schema_version: 1,
128 payload,
129 title,
130 identity_digest: "digest".into(),
131 })
132 })
133}
134
135fn build_agent(
136 dir: &tempfile::TempDir,
137 session_id: &str,
138 responses: Vec<AssistantMessage>,
139 tools: Vec<Arc<dyn Tool>>,
140 with_presentation_rebuild: bool,
141) -> Agent {
142 let header = SessionHeader {
143 agent: None,
144 session_id: session_id.into(),
145 created_at: chrono::Utc::now(),
146 cwd: dir.path().to_path_buf(),
147 parent_session_id: None,
148 contract_id: None,
149 work_item_id: None,
150 conversation: None,
151 contract: FrozenContract {
152 app_version: "0".into(),
153 provider: "scripted".into(),
154 model: "test-model".into(),
155 route_ladder: Vec::new(),
156 route_objective: String::new(),
157 route_annotations: Vec::new(),
158 system_prompt: "sys".into(),
159 permission_mode: "full-access".into(),
160 capabilities: Vec::new(),
161 prompt_layers: Vec::new(),
162 },
163 };
164 let home = dir.path().join("home");
165 std::fs::create_dir_all(&home).unwrap();
166 let log = SessionLog::create(
167 SessionPath::new_session_file(&home, dir.path(), session_id),
168 header,
169 )
170 .unwrap();
171 let mut cfg = AgentConfig::new("sys");
172 cfg.model = "test-model".into();
173 cfg.mode = vak_permission::Mode::FullAccess;
174 cfg.permission = Some(Arc::new(PermissionEngine::default()));
175 cfg.approver = Some(Arc::new(vak_agent::AutoApprove));
176 cfg.tools = tools;
177 if with_presentation_rebuild {
178 cfg.presentation_rebuild = Some(fake_rebuild());
179 }
180 Agent::new(
181 Arc::new(Scripted {
182 responses: Mutex::new(VecDeque::from(responses)),
183 }),
184 log,
185 cfg,
186 )
187}
188
189#[tokio::test]
190async fn turn_card_is_written_at_close() {
191 let dir = tempdir().unwrap();
192 let mut agent = build_agent(&dir, "card-close", vec![text_msg("42")], vec![], false);
193 let outcome = agent
194 .run(
195 "what is six times seven",
196 &Default::default(),
197 CancellationToken::new(),
198 mpsc::channel(64).0,
199 )
200 .await;
201 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
202
203 let session = agent.session.lock().await;
204 let cards = session.turn_cards();
205 assert_eq!(cards.len(), 1, "exactly one TurnCard written at close");
206 assert_eq!(cards[0].1.outcome, "completed");
207 assert_eq!(cards[0].1.answered.narration, "42");
208 assert!(cards[0].1.answered.presentations.is_empty());
209}
210
211#[tokio::test]
212async fn fence_presentation_is_written_once() {
213 let dir = tempdir().unwrap();
214 let fence = "Here's a fresh chart.\n\n```vak\n{\"semantic_type\":\"chart\",\"payload\":{\"title\":\"Fresh\",\"series\":[1,2,3]}}\n```";
215 let mut agent = build_agent(&dir, "fence-write", vec![text_msg(fence)], vec![], true);
216 let outcome = agent
217 .run(
218 "show me a chart",
219 &Default::default(),
220 CancellationToken::new(),
221 mpsc::channel(64).0,
222 )
223 .await;
224 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
225
226 let session = agent.session.lock().await;
227 let presentations = session.presentations();
228 assert_eq!(
229 presentations.len(),
230 1,
231 "the fence must write one presentation"
232 );
233 assert!(matches!(
234 presentations[0].1.source,
235 PresentationSource::Fence { .. }
236 ));
237 assert_eq!(presentations[0].1.title, "Fresh");
238}
239
240#[tokio::test]
241async fn fence_presentation_deduped_against_tool_emitted_card() {
242 let dir = tempdir().unwrap();
243 // The final accepted answer still repeats the just-emitted card as a
244 // `vak` fence with the IDENTICAL payload (the observed real bug this
245 // rule exists for) — the duplicate-card-check repair nudge fires once,
246 // and the model ignores it (attempt two repeats it too), so the run
247 // completes with the duplicate fence still present in the final text.
248 let duplicate_fence = "Here's the chart you asked for.\n\n```vak\n{\"semantic_type\":\"chart\",\"payload\":{\"title\":\"Sales\",\"series\":[1,2,3]}}\n```";
249 let mut agent = build_agent(
250 &dir,
251 "fence-dedup",
252 vec![
253 tool_call_msg(
254 "call_1",
255 "emit_chart_card",
256 serde_json::json!({"semantic_type": "chart", "payload": {"title": "Sales", "series": [1, 2, 3]}}),
257 ),
258 text_msg(duplicate_fence),
259 text_msg(duplicate_fence),
260 ],
261 vec![Arc::new(FakeEmitChartCard)],
262 true,
263 );
264 let outcome = agent
265 .run(
266 "show me the sales chart",
267 &Default::default(),
268 CancellationToken::new(),
269 mpsc::channel(64).0,
270 )
271 .await;
272 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
273
274 let session = agent.session.lock().await;
275 let presentations = session.presentations();
276 assert_eq!(
277 presentations.len(),
278 1,
279 "the duplicate fence must not add a second presentation: {presentations:?}"
280 );
281 assert!(matches!(
282 presentations[0].1.source,
283 PresentationSource::ToolCall { .. }
284 ));
285}
286
287#[tokio::test]
288async fn recall_by_id_returns_the_full_evidence_content() {
289 let dir = tempdir().unwrap();
290 let mut agent = build_agent(
291 &dir,
292 "recall-id",
293 vec![
294 tool_call_msg(
295 "t1",
296 "bash",
297 serde_json::json!({"command": "printf 'HELLO WORLD'"}),
298 ),
299 tool_call_msg("t2", "recall", serde_json::json!({"id": "t1"})),
300 text_msg("done"),
301 ],
302 vec![Arc::new(BashTool)],
303 false,
304 );
305 let outcome = agent
306 .run(
307 "run the command then recall it",
308 &Default::default(),
309 CancellationToken::new(),
310 mpsc::channel(64).0,
311 )
312 .await;
313 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
314
315 let session = agent.session.lock().await;
316 let messages = session.message_chain();
317 let content_for = |id: &str| -> String {
318 messages
319 .iter()
320 .flat_map(|(_, m)| m.content.iter())
321 .find_map(|b| match b {
322 ContentBlock::ToolResult {
323 tool_use_id,
324 content,
325 ..
326 } if tool_use_id == id => Some(content.clone()),
327 _ => None,
328 })
329 .unwrap_or_else(|| panic!("no tool result for {id}"))
330 };
331 let original = content_for("t1");
332 let recalled = content_for("t2");
333 assert!(original.contains("HELLO WORLD"));
334 assert_eq!(
335 recalled, original,
336 "recall({{ id }}) must return the evidence content verbatim"
337 );
338}
339

Turn Card And Recall

Indexing the workspace…

Turn Card And Recall