Vakyartha documentation

1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2
3//! Regression coverage for a verified real bug, observed live against the
4//! real local model this app ships with (gemma4:e2b-mlx via Ollama): the
5//! model calls `emit_chart_card` successfully — a card the user already
6//! sees, pushed from the tool result — and then its own next answer *also*
7//! writes out a `vak` fence repeating the same semantic_type, which renders
8//! as a second, duplicate card (vak-server's tool-result projection and the
9//! client's own fence-parsing are independent paths; nothing dedupes across
10//! them). `Agent::run` must give the model one bounded repair turn asking
11//! it to drop the redundant fence, reusing the same turn-loop `continue`
12//! mechanism the grounding-check and malformed-fence repairs use.
13
14use std::collections::VecDeque;
15use std::sync::Arc;
16use std::sync::Mutex;
17
18use async_trait::async_trait;
19use tokio::sync::mpsc;
20use tokio_util::sync::CancellationToken;
21
22use tempfile::tempdir;
23
24use vak_agent::{Agent, AgentConfig, TurnOutcome};
25use vak_llm::stream;
26use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
27use vak_llm::{EventStream, LlmError, Provider};
28use vak_permission::PermissionEngine;
29use vak_session::types::{FrozenContract, SessionHeader};
30use vak_session::{SessionLog, SessionPath};
31use vak_tools::context::ToolContext;
32use vak_tools::{Tool, ToolOutput};
33
34struct Scripted {
35 responses: Mutex<VecDeque<AssistantMessage>>,
36}
37
38#[async_trait]
39impl Provider for Scripted {
40 fn name(&self) -> &str {
41 "scripted"
42 }
43
44 async fn stream(
45 &self,
46 _request: ChatRequest,
47 _cancel: CancellationToken,
48 ) -> Result<EventStream, LlmError> {
49 let next = self.responses.lock().unwrap().pop_front();
50 let (mut sink, rx) = stream::channel(64);
51 match next {
52 Some(m) => {
53 sink.push(stream::StreamEvent::Start { partial: m.clone() });
54 sink.close_message(m).await;
55 }
56 None => sink.close_error(LlmError::Parse("exhausted".into())).await,
57 }
58 Ok(rx)
59 }
60}
61
62/// Stand-in for `vak_core::presentation_tools::EmitCardTool` — returns the
63/// same `{"semantic_type","payload"}` envelope shape the real tool wraps
64/// its arguments in, without pulling in the vak-core dependency.
65struct FakeEmitChartCard;
66
67#[async_trait]
68impl Tool for FakeEmitChartCard {
69 fn name(&self) -> &str {
70 "emit_chart_card"
71 }
72
73 fn description(&self) -> &str {
74 "test stand-in"
75 }
76
77 fn schema(&self) -> serde_json::Value {
78 serde_json::json!({"type": "object"})
79 }
80
81 fn presents_cards(&self) -> bool {
82 true
83 }
84
85 async fn execute(&self, args: &serde_json::Value, _ctx: &ToolContext) -> ToolOutput {
86 let envelope = serde_json::json!({
87 "semantic_type": "chart",
88 "payload": args.get("payload").cloned().unwrap_or(serde_json::json!({})),
89 });
90 ToolOutput::ok(envelope.to_string())
91 }
92}
93
94fn text_msg(t: &str) -> AssistantMessage {
95 AssistantMessage {
96 content: vec![ContentBlock::text(t)],
97 stop_reason: StopReason::EndTurn,
98 usage: Usage {
99 input_tokens: 1,
100 output_tokens: 1,
101 ..Default::default()
102 },
103 model: "test-model".into(),
104 response_id: None,
105 }
106}
107
108fn tool_call_msg(id: &str, name: &str, input: serde_json::Value) -> AssistantMessage {
109 AssistantMessage {
110 content: vec![ContentBlock::ToolUse {
111 id: id.into(),
112 name: name.into(),
113 input,
114 }],
115 stop_reason: StopReason::ToolUse,
116 usage: Usage {
117 input_tokens: 1,
118 output_tokens: 1,
119 ..Default::default()
120 },
121 model: "test-model".into(),
122 response_id: None,
123 }
124}
125
126async fn build_agent(
127 dir: &tempfile::TempDir,
128 session_id: &str,
129 responses: Vec<AssistantMessage>,
130) -> Agent {
131 let header = SessionHeader {
132 agent: None,
133 session_id: session_id.into(),
134 created_at: chrono::Utc::now(),
135 cwd: dir.path().to_path_buf(),
136 parent_session_id: None,
137 contract_id: None,
138 work_item_id: None,
139 conversation: None,
140 contract: FrozenContract {
141 app_version: "0".into(),
142 provider: "scripted".into(),
143 model: "test-model".into(),
144 route_ladder: Vec::new(),
145 route_objective: String::new(),
146 route_annotations: Vec::new(),
147 system_prompt: "sys".into(),
148 permission_mode: "full-access".into(),
149 capabilities: Vec::new(),
150 prompt_layers: Vec::new(),
151 },
152 };
153 let home = dir.path().join("home");
154 std::fs::create_dir_all(&home).unwrap();
155 let log = SessionLog::create(
156 SessionPath::new_session_file(&home, dir.path(), session_id),
157 header,
158 )
159 .unwrap();
160
161 Agent::new(
162 Arc::new(Scripted {
163 responses: Mutex::new(VecDeque::from(responses)),
164 }),
165 log,
166 {
167 let mut cfg = AgentConfig::new("sys");
168 cfg.model = "test-model".into();
169 cfg.mode = vak_permission::Mode::FullAccess;
170 cfg.permission = Some(Arc::new(PermissionEngine::default()));
171 cfg.tools = vec![Arc::new(FakeEmitChartCard)];
172 cfg
173 },
174 )
175}
176
177// Raw ledger, not the model-visible projection: these tests are about the
178// repair loop's mechanics (how many drafts were tried, what a nudge said),
179// which the turn-based projection deliberately no longer preserves once
180// the turn closes — a rejected draft is never projected
181// (docs/design/68-context-engine.md §10), and every step collapses into
182// one trace+narration message in the closed turn's full record.
183fn assistant_texts(agent: &Agent) -> Vec<String> {
184 futures::executor::block_on(async {
185 agent
186 .session
187 .lock()
188 .await
189 .message_chain()
190 .iter()
191 .filter(|(_, m)| m.role == vak_llm::types::Role::Assistant)
192 .flat_map(|(_, m)| m.content.iter())
193 .filter_map(|b| match b {
194 ContentBlock::Text { text } => Some(text.clone()),
195 _ => None,
196 })
197 .collect()
198 })
199}
200
201fn user_texts(agent: &Agent) -> Vec<String> {
202 futures::executor::block_on(async {
203 agent
204 .session
205 .lock()
206 .await
207 .message_chain()
208 .iter()
209 .filter(|(_, m)| m.role == vak_llm::types::Role::User)
210 .flat_map(|(_, m)| m.content.iter())
211 .filter_map(|b| match b {
212 ContentBlock::Text { text } => Some(text.clone()),
213 _ => None,
214 })
215 .collect()
216 })
217}
218
219const DUPLICATE_FENCE: &str = "Here's the chart you asked for.\n\n```vak\n{\"semantic_type\":\"chart\",\"payload\":{\"chart_type\":\"line\",\"series\":[]}}\n```";
220const NARRATION_ONLY: &str = "Here's the chart you asked for.";
221
222#[tokio::test]
223async fn a_fence_repeating_a_just_emitted_card_gets_one_repair_turn() {
224 let dir = tempdir().unwrap();
225 let mut agent = build_agent(
226 &dir,
227 "dup-card-repair",
228 vec![
229 tool_call_msg(
230 "call_1",
231 "emit_chart_card",
232 serde_json::json!({"semantic_type": "chart", "payload": {"chart_type": "line", "series": []}}),
233 ),
234 text_msg(DUPLICATE_FENCE),
235 text_msg(NARRATION_ONLY),
236 ],
237 )
238 .await;
239
240 let outcome = agent
241 .run(
242 "show me a chart",
243 &Default::default(),
244 CancellationToken::new(),
245 mpsc::channel(64).0,
246 )
247 .await;
248 assert!(
249 matches!(outcome, TurnOutcome::Completed { .. }),
250 "got {outcome:?}"
251 );
252
253 let texts = assistant_texts(&agent);
254 assert!(
255 texts.iter().any(|t| t == NARRATION_ONLY),
256 "final state must contain the de-duplicated narration-only answer: {texts:?}"
257 );
258
259 let users = user_texts(&agent);
260 assert!(
261 users
262 .iter()
263 .any(|t| t.contains("[duplicate-card-check]") && t.contains("chart")),
264 "expected a duplicate-card-check repair nudge naming the semantic_type: {users:?}"
265 );
266}
267
268#[tokio::test]
269async fn repair_is_bounded_to_one_attempt_not_a_loop() {
270 let dir = tempdir().unwrap();
271 let mut agent = build_agent(
272 &dir,
273 "dup-card-bounded",
274 vec![
275 tool_call_msg(
276 "call_1",
277 "emit_chart_card",
278 serde_json::json!({"semantic_type": "chart", "payload": {"chart_type": "line", "series": []}}),
279 ),
280 text_msg(DUPLICATE_FENCE),
281 text_msg(DUPLICATE_FENCE), // still duplicated after the nudge
282 ],
283 )
284 .await;
285
286 let outcome = agent
287 .run(
288 "show me a chart",
289 &Default::default(),
290 CancellationToken::new(),
291 mpsc::channel(64).0,
292 )
293 .await;
294 assert!(
295 matches!(outcome, TurnOutcome::Completed { .. }),
296 "got {outcome:?}"
297 );
298
299 let texts = assistant_texts(&agent);
300 assert_eq!(
301 texts.iter().filter(|t| t.contains("```vak")).count(),
302 2,
303 "exactly one retry — not zero, not an infinite loop: {texts:?}"
304 );
305}
306
307#[tokio::test]
308async fn narration_without_a_duplicate_fence_is_never_touched() {
309 let dir = tempdir().unwrap();
310 let mut agent = build_agent(
311 &dir,
312 "dup-card-none",
313 vec![
314 tool_call_msg(
315 "call_1",
316 "emit_chart_card",
317 serde_json::json!({"semantic_type": "chart", "payload": {"chart_type": "line", "series": []}}),
318 ),
319 text_msg(NARRATION_ONLY),
320 ],
321 )
322 .await;
323
324 let outcome = agent
325 .run(
326 "show me a chart",
327 &Default::default(),
328 CancellationToken::new(),
329 mpsc::channel(64).0,
330 )
331 .await;
332 assert!(
333 matches!(outcome, TurnOutcome::Completed { .. }),
334 "got {outcome:?}"
335 );
336
337 let users = user_texts(&agent);
338 assert!(
339 !users.iter().any(|t| t.contains("[duplicate-card-check]")),
340 "an answer with no repeated fence must never trigger a repair nudge: {users:?}"
341 );
342}
343
344#[tokio::test]
345async fn a_fence_with_no_preceding_tool_call_is_never_touched() {
346 let dir = tempdir().unwrap();
347 let mut agent = build_agent(&dir, "dup-card-no-tool", vec![text_msg(DUPLICATE_FENCE)]).await;
348
349 let outcome = agent
350 .run(
351 "show me a chart",
352 &Default::default(),
353 CancellationToken::new(),
354 mpsc::channel(64).0,
355 )
356 .await;
357 assert!(
358 matches!(outcome, TurnOutcome::Completed { .. }),
359 "got {outcome:?}"
360 );
361
362 let users = user_texts(&agent);
363 assert!(
364 !users.iter().any(|t| t.contains("[duplicate-card-check]")),
365 "a fence with no matching tool call this turn is a normal fence-only card, not a duplicate: {users:?}"
366 );
367}
368

1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2
3//! Regression coverage for a verified real bug, observed live against the
4//! real local model this app ships with (gemma4:e2b-mlx via Ollama): the
5//! model calls `emit_chart_card` successfully — a card the user already
6//! sees, pushed from the tool result — and then its own next answer *also*
7//! writes out a `vak` fence repeating the same semantic_type, which renders
8//! as a second, duplicate card (vak-server's tool-result projection and the
9//! client's own fence-parsing are independent paths; nothing dedupes across
10//! them). `Agent::run` must give the model one bounded repair turn asking
11//! it to drop the redundant fence, reusing the same turn-loop `continue`
12//! mechanism the grounding-check and malformed-fence repairs use.
13
14use std::collections::VecDeque;
15use std::sync::Arc;
16use std::sync::Mutex;
17
18use async_trait::async_trait;
19use tokio::sync::mpsc;
20use tokio_util::sync::CancellationToken;
21
22use tempfile::tempdir;
23
24use vak_agent::{Agent, AgentConfig, TurnOutcome};
25use vak_llm::stream;
26use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
27use vak_llm::{EventStream, LlmError, Provider};
28use vak_permission::PermissionEngine;
29use vak_session::types::{FrozenContract, SessionHeader};
30use vak_session::{SessionLog, SessionPath};
31use vak_tools::context::ToolContext;
32use vak_tools::{Tool, ToolOutput};
33
34struct Scripted {
35 responses: Mutex<VecDeque<AssistantMessage>>,
36}
37
38#[async_trait]
39impl Provider for Scripted {
40 fn name(&self) -> &str {
41 "scripted"
42 }
43
44 async fn stream(
45 &self,
46 _request: ChatRequest,
47 _cancel: CancellationToken,
48 ) -> Result<EventStream, LlmError> {
49 let next = self.responses.lock().unwrap().pop_front();
50 let (mut sink, rx) = stream::channel(64);
51 match next {
52 Some(m) => {
53 sink.push(stream::StreamEvent::Start { partial: m.clone() });
54 sink.close_message(m).await;
55 }
56 None => sink.close_error(LlmError::Parse("exhausted".into())).await,
57 }
58 Ok(rx)
59 }
60}
61
62/// Stand-in for `vak_core::presentation_tools::EmitCardTool` — returns the
63/// same `{"semantic_type","payload"}` envelope shape the real tool wraps
64/// its arguments in, without pulling in the vak-core dependency.
65struct FakeEmitChartCard;
66
67#[async_trait]
68impl Tool for FakeEmitChartCard {
69 fn name(&self) -> &str {
70 "emit_chart_card"
71 }
72
73 fn description(&self) -> &str {
74 "test stand-in"
75 }
76
77 fn schema(&self) -> serde_json::Value {
78 serde_json::json!({"type": "object"})
79 }
80
81 fn presents_cards(&self) -> bool {
82 true
83 }
84
85 async fn execute(&self, args: &serde_json::Value, _ctx: &ToolContext) -> ToolOutput {
86 let envelope = serde_json::json!({
87 "semantic_type": "chart",
88 "payload": args.get("payload").cloned().unwrap_or(serde_json::json!({})),
89 });
90 ToolOutput::ok(envelope.to_string())
91 }
92}
93
94fn text_msg(t: &str) -> AssistantMessage {
95 AssistantMessage {
96 content: vec![ContentBlock::text(t)],
97 stop_reason: StopReason::EndTurn,
98 usage: Usage {
99 input_tokens: 1,
100 output_tokens: 1,
101 ..Default::default()
102 },
103 model: "test-model".into(),
104 response_id: None,
105 }
106}
107
108fn tool_call_msg(id: &str, name: &str, input: serde_json::Value) -> AssistantMessage {
109 AssistantMessage {
110 content: vec![ContentBlock::ToolUse {
111 id: id.into(),
112 name: name.into(),
113 input,
114 }],
115 stop_reason: StopReason::ToolUse,
116 usage: Usage {
117 input_tokens: 1,
118 output_tokens: 1,
119 ..Default::default()
120 },
121 model: "test-model".into(),
122 response_id: None,
123 }
124}
125
126async fn build_agent(
127 dir: &tempfile::TempDir,
128 session_id: &str,
129 responses: Vec<AssistantMessage>,
130) -> Agent {
131 let header = SessionHeader {
132 agent: None,
133 session_id: session_id.into(),
134 created_at: chrono::Utc::now(),
135 cwd: dir.path().to_path_buf(),
136 parent_session_id: None,
137 contract_id: None,
138 work_item_id: None,
139 conversation: None,
140 contract: FrozenContract {
141 app_version: "0".into(),
142 provider: "scripted".into(),
143 model: "test-model".into(),
144 route_ladder: Vec::new(),
145 route_objective: String::new(),
146 route_annotations: Vec::new(),
147 system_prompt: "sys".into(),
148 permission_mode: "full-access".into(),
149 capabilities: Vec::new(),
150 prompt_layers: Vec::new(),
151 },
152 };
153 let home = dir.path().join("home");
154 std::fs::create_dir_all(&home).unwrap();
155 let log = SessionLog::create(
156 SessionPath::new_session_file(&home, dir.path(), session_id),
157 header,
158 )
159 .unwrap();
160
161 Agent::new(
162 Arc::new(Scripted {
163 responses: Mutex::new(VecDeque::from(responses)),
164 }),
165 log,
166 {
167 let mut cfg = AgentConfig::new("sys");
168 cfg.model = "test-model".into();
169 cfg.mode = vak_permission::Mode::FullAccess;
170 cfg.permission = Some(Arc::new(PermissionEngine::default()));
171 cfg.tools = vec![Arc::new(FakeEmitChartCard)];
172 cfg
173 },
174 )
175}
176
177// Raw ledger, not the model-visible projection: these tests are about the
178// repair loop's mechanics (how many drafts were tried, what a nudge said),
179// which the turn-based projection deliberately no longer preserves once
180// the turn closes — a rejected draft is never projected
181// (docs/design/68-context-engine.md §10), and every step collapses into
182// one trace+narration message in the closed turn's full record.
183fn assistant_texts(agent: &Agent) -> Vec<String> {
184 futures::executor::block_on(async {
185 agent
186 .session
187 .lock()
188 .await
189 .message_chain()
190 .iter()
191 .filter(|(_, m)| m.role == vak_llm::types::Role::Assistant)
192 .flat_map(|(_, m)| m.content.iter())
193 .filter_map(|b| match b {
194 ContentBlock::Text { text } => Some(text.clone()),
195 _ => None,
196 })
197 .collect()
198 })
199}
200
201fn user_texts(agent: &Agent) -> Vec<String> {
202 futures::executor::block_on(async {
203 agent
204 .session
205 .lock()
206 .await
207 .message_chain()
208 .iter()
209 .filter(|(_, m)| m.role == vak_llm::types::Role::User)
210 .flat_map(|(_, m)| m.content.iter())
211 .filter_map(|b| match b {
212 ContentBlock::Text { text } => Some(text.clone()),
213 _ => None,
214 })
215 .collect()
216 })
217}
218
219const DUPLICATE_FENCE: &str = "Here's the chart you asked for.\n\n```vak\n{\"semantic_type\":\"chart\",\"payload\":{\"chart_type\":\"line\",\"series\":[]}}\n```";
220const NARRATION_ONLY: &str = "Here's the chart you asked for.";
221
222#[tokio::test]
223async fn a_fence_repeating_a_just_emitted_card_gets_one_repair_turn() {
224 let dir = tempdir().unwrap();
225 let mut agent = build_agent(
226 &dir,
227 "dup-card-repair",
228 vec![
229 tool_call_msg(
230 "call_1",
231 "emit_chart_card",
232 serde_json::json!({"semantic_type": "chart", "payload": {"chart_type": "line", "series": []}}),
233 ),
234 text_msg(DUPLICATE_FENCE),
235 text_msg(NARRATION_ONLY),
236 ],
237 )
238 .await;
239
240 let outcome = agent
241 .run(
242 "show me a chart",
243 &Default::default(),
244 CancellationToken::new(),
245 mpsc::channel(64).0,
246 )
247 .await;
248 assert!(
249 matches!(outcome, TurnOutcome::Completed { .. }),
250 "got {outcome:?}"
251 );
252
253 let texts = assistant_texts(&agent);
254 assert!(
255 texts.iter().any(|t| t == NARRATION_ONLY),
256 "final state must contain the de-duplicated narration-only answer: {texts:?}"
257 );
258
259 let users = user_texts(&agent);
260 assert!(
261 users
262 .iter()
263 .any(|t| t.contains("[duplicate-card-check]") && t.contains("chart")),
264 "expected a duplicate-card-check repair nudge naming the semantic_type: {users:?}"
265 );
266}
267
268#[tokio::test]
269async fn repair_is_bounded_to_one_attempt_not_a_loop() {
270 let dir = tempdir().unwrap();
271 let mut agent = build_agent(
272 &dir,
273 "dup-card-bounded",
274 vec![
275 tool_call_msg(
276 "call_1",
277 "emit_chart_card",
278 serde_json::json!({"semantic_type": "chart", "payload": {"chart_type": "line", "series": []}}),
279 ),
280 text_msg(DUPLICATE_FENCE),
281 text_msg(DUPLICATE_FENCE), // still duplicated after the nudge
282 ],
283 )
284 .await;
285
286 let outcome = agent
287 .run(
288 "show me a chart",
289 &Default::default(),
290 CancellationToken::new(),
291 mpsc::channel(64).0,
292 )
293 .await;
294 assert!(
295 matches!(outcome, TurnOutcome::Completed { .. }),
296 "got {outcome:?}"
297 );
298
299 let texts = assistant_texts(&agent);
300 assert_eq!(
301 texts.iter().filter(|t| t.contains("```vak")).count(),
302 2,
303 "exactly one retry — not zero, not an infinite loop: {texts:?}"
304 );
305}
306
307#[tokio::test]
308async fn narration_without_a_duplicate_fence_is_never_touched() {
309 let dir = tempdir().unwrap();
310 let mut agent = build_agent(
311 &dir,
312 "dup-card-none",
313 vec![
314 tool_call_msg(
315 "call_1",
316 "emit_chart_card",
317 serde_json::json!({"semantic_type": "chart", "payload": {"chart_type": "line", "series": []}}),
318 ),
319 text_msg(NARRATION_ONLY),
320 ],
321 )
322 .await;
323
324 let outcome = agent
325 .run(
326 "show me a chart",
327 &Default::default(),
328 CancellationToken::new(),
329 mpsc::channel(64).0,
330 )
331 .await;
332 assert!(
333 matches!(outcome, TurnOutcome::Completed { .. }),
334 "got {outcome:?}"
335 );
336
337 let users = user_texts(&agent);
338 assert!(
339 !users.iter().any(|t| t.contains("[duplicate-card-check]")),
340 "an answer with no repeated fence must never trigger a repair nudge: {users:?}"
341 );
342}
343
344#[tokio::test]
345async fn a_fence_with_no_preceding_tool_call_is_never_touched() {
346 let dir = tempdir().unwrap();
347 let mut agent = build_agent(&dir, "dup-card-no-tool", vec![text_msg(DUPLICATE_FENCE)]).await;
348
349 let outcome = agent
350 .run(
351 "show me a chart",
352 &Default::default(),
353 CancellationToken::new(),
354 mpsc::channel(64).0,
355 )
356 .await;
357 assert!(
358 matches!(outcome, TurnOutcome::Completed { .. }),
359 "got {outcome:?}"
360 );
361
362 let users = user_texts(&agent);
363 assert!(
364 !users.iter().any(|t| t.contains("[duplicate-card-check]")),
365 "a fence with no matching tool call this turn is a normal fence-only card, not a duplicate: {users:?}"
366 );
367}
368

Duplicate Card Check

Indexing the workspace…

Duplicate Card Check