Vakyartha documentation

1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2
3//! Regression coverage for a verified real bug: the model emits a `vak`
4//! card fence whose JSON body doesn't parse (mismatched brackets, an
5//! unquoted key — real failure shapes seen from a small local model in
6//! production), and the answer reaches the user with a broken/invisible
7//! card instead of a working one. `Agent::run` must give the model one
8//! bounded repair turn naming the exact parse error, reusing the same
9//! turn-loop `continue` mechanism the grounding-check repair uses — this
10//! is a new trigger condition on existing infrastructure, not a new system.
11
12use std::collections::VecDeque;
13use std::sync::Arc;
14use std::sync::Mutex;
15
16use async_trait::async_trait;
17use tokio::sync::mpsc;
18use tokio_util::sync::CancellationToken;
19
20use tempfile::tempdir;
21
22use vak_agent::{Agent, AgentConfig, TurnOutcome};
23use vak_llm::stream;
24use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
25use vak_llm::{EventStream, LlmError, Provider};
26use vak_permission::PermissionEngine;
27use vak_session::types::{FrozenContract, SessionHeader};
28use vak_session::{SessionLog, SessionPath};
29
30struct Scripted {
31 responses: Mutex<VecDeque<AssistantMessage>>,
32}
33
34#[async_trait]
35impl Provider for Scripted {
36 fn name(&self) -> &str {
37 "scripted"
38 }
39
40 async fn stream(
41 &self,
42 _request: ChatRequest,
43 _cancel: CancellationToken,
44 ) -> Result<EventStream, LlmError> {
45 let next = self.responses.lock().unwrap().pop_front();
46 let (mut sink, rx) = stream::channel(64);
47 match next {
48 Some(m) => {
49 sink.push(stream::StreamEvent::Start { partial: m.clone() });
50 sink.close_message(m).await;
51 }
52 None => sink.close_error(LlmError::Parse("exhausted".into())).await,
53 }
54 Ok(rx)
55 }
56}
57
58fn text_msg(t: &str) -> AssistantMessage {
59 AssistantMessage {
60 content: vec![ContentBlock::text(t)],
61 stop_reason: StopReason::EndTurn,
62 usage: Usage {
63 input_tokens: 1,
64 output_tokens: 1,
65 ..Default::default()
66 },
67 model: "test-model".into(),
68 response_id: None,
69 }
70}
71
72async fn build_agent(
73 dir: &tempfile::TempDir,
74 session_id: &str,
75 responses: Vec<AssistantMessage>,
76) -> Agent {
77 let header = SessionHeader {
78 agent: None,
79 session_id: session_id.into(),
80 created_at: chrono::Utc::now(),
81 cwd: dir.path().to_path_buf(),
82 parent_session_id: None,
83 contract_id: None,
84 work_item_id: None,
85 conversation: None,
86 contract: FrozenContract {
87 app_version: "0".into(),
88 provider: "scripted".into(),
89 model: "test-model".into(),
90 route_ladder: Vec::new(),
91 route_objective: String::new(),
92 route_annotations: Vec::new(),
93 system_prompt: "sys".into(),
94 permission_mode: "full-access".into(),
95 capabilities: Vec::new(),
96 prompt_layers: Vec::new(),
97 },
98 };
99 let home = dir.path().join("home");
100 std::fs::create_dir_all(&home).unwrap();
101 let log = SessionLog::create(
102 SessionPath::new_session_file(&home, dir.path(), session_id),
103 header,
104 )
105 .unwrap();
106
107 Agent::new(
108 Arc::new(Scripted {
109 responses: Mutex::new(VecDeque::from(responses)),
110 }),
111 log,
112 {
113 let mut cfg = AgentConfig::new("sys");
114 cfg.model = "test-model".into();
115 cfg.mode = vak_permission::Mode::FullAccess;
116 cfg.permission = Some(Arc::new(PermissionEngine::default()));
117 cfg
118 },
119 )
120}
121
122// Raw ledger, not the model-visible projection: these tests are about the
123// repair loop's mechanics, which a closed turn's full record deliberately
124// no longer preserves (docs/design/68-context-engine.md §10) — a rejected
125// draft and a mid-turn nudge are dropped once the turn closes.
126fn assistant_texts(agent: &Agent) -> Vec<String> {
127 futures::executor::block_on(async {
128 agent
129 .session
130 .lock()
131 .await
132 .message_chain()
133 .iter()
134 .filter(|(_, m)| m.role == vak_llm::types::Role::Assistant)
135 .flat_map(|(_, m)| m.content.iter())
136 .filter_map(|b| match b {
137 ContentBlock::Text { text } => Some(text.clone()),
138 _ => None,
139 })
140 .collect()
141 })
142}
143
144fn user_texts(agent: &Agent) -> Vec<String> {
145 futures::executor::block_on(async {
146 agent
147 .session
148 .lock()
149 .await
150 .message_chain()
151 .iter()
152 .filter(|(_, m)| m.role == vak_llm::types::Role::User)
153 .flat_map(|(_, m)| m.content.iter())
154 .filter_map(|b| match b {
155 ContentBlock::Text { text } => Some(text.clone()),
156 _ => None,
157 })
158 .collect()
159 })
160}
161
162// The exact malformed shape from the live bug: a `decision` fence whose
163// payload has a bare (unquoted) key, `precision":1` instead of `"precision":1`.
164const REAL_MALFORMED_FENCE: &str = "```vak\n{\"semantic_type\":\"decision\",\"payload\":{\"title\":\"Capability Confirmation\",\"choices\":[{\"name\":\"Yes\",\"reason\":\"ok\"}],precision\":1}}\n```\n\n### Explanation\n\nYes, I can do that.";
165
166const VALID_FENCE: &str = "```vak\n{\"semantic_type\":\"metric\",\"payload\":{\"label\":\"Uptime\",\"value\":99.9,\"unit\":\"%\"}}\n```";
167
168#[tokio::test]
169async fn malformed_fence_gets_one_repair_turn_naming_the_parse_error() {
170 let dir = tempdir().unwrap();
171 let mut agent = build_agent(
172 &dir,
173 "fence-repair",
174 vec![text_msg(REAL_MALFORMED_FENCE), text_msg(VALID_FENCE)],
175 )
176 .await;
177
178 let outcome = agent
179 .run(
180 "can you make charts",
181 &Default::default(),
182 CancellationToken::new(),
183 mpsc::channel(64).0,
184 )
185 .await;
186 assert!(
187 matches!(outcome, TurnOutcome::Completed { .. }),
188 "got {outcome:?}"
189 );
190
191 let texts = assistant_texts(&agent);
192 assert!(
193 texts
194 .iter()
195 .any(|t| t.contains("\"semantic_type\":\"metric\"")),
196 "final state must contain the repaired, valid fence: {texts:?}"
197 );
198
199 let users = user_texts(&agent);
200 assert!(
201 users
202 .iter()
203 .any(|t| t.contains("[fence-check]") && t.contains("invalid JSON")),
204 "expected a fence-check repair nudge naming the parse failure: {users:?}"
205 );
206}
207
208#[tokio::test]
209async fn repair_is_bounded_to_one_attempt_not_a_loop() {
210 let dir = tempdir().unwrap();
211 let mut agent = build_agent(
212 &dir,
213 "fence-repair-bounded",
214 vec![
215 text_msg(REAL_MALFORMED_FENCE),
216 text_msg(REAL_MALFORMED_FENCE), // still broken after the nudge
217 ],
218 )
219 .await;
220
221 let outcome = agent
222 .run(
223 "can you make charts",
224 &Default::default(),
225 CancellationToken::new(),
226 mpsc::channel(64).0,
227 )
228 .await;
229 assert!(
230 matches!(outcome, TurnOutcome::Completed { .. }),
231 "got {outcome:?}"
232 );
233
234 let texts = assistant_texts(&agent);
235 assert_eq!(
236 texts
237 .iter()
238 .filter(|t| t.contains("Capability Confirmation"))
239 .count(),
240 2,
241 "exactly one retry — not zero, not an infinite loop: {texts:?}"
242 );
243}
244
245#[tokio::test]
246async fn a_valid_fence_is_never_touched() {
247 let dir = tempdir().unwrap();
248 let mut agent = build_agent(&dir, "fence-valid", vec![text_msg(VALID_FENCE)]).await;
249
250 let outcome = agent
251 .run(
252 "what's our uptime",
253 &Default::default(),
254 CancellationToken::new(),
255 mpsc::channel(64).0,
256 )
257 .await;
258 assert!(
259 matches!(outcome, TurnOutcome::Completed { .. }),
260 "got {outcome:?}"
261 );
262
263 let users = user_texts(&agent);
264 assert!(
265 !users.iter().any(|t| t.contains("[fence-check]")),
266 "a syntactically valid fence must never trigger a repair nudge: {users:?}"
267 );
268}
269
270#[tokio::test]
271async fn plain_prose_with_no_fence_is_never_touched() {
272 let dir = tempdir().unwrap();
273 let mut agent = build_agent(
274 &dir,
275 "fence-none",
276 vec![text_msg("Paris is the capital of France.")],
277 )
278 .await;
279
280 let outcome = agent
281 .run(
282 "what's the capital of france",
283 &Default::default(),
284 CancellationToken::new(),
285 mpsc::channel(64).0,
286 )
287 .await;
288 assert!(
289 matches!(outcome, TurnOutcome::Completed { .. }),
290 "got {outcome:?}"
291 );
292
293 let users = user_texts(&agent);
294 assert!(
295 !users.iter().any(|t| t.contains("[fence-check]")),
296 "plain prose with no fence must never trigger a repair nudge: {users:?}"
297 );
298}
299

1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2
3//! Regression coverage for a verified real bug: the model emits a `vak`
4//! card fence whose JSON body doesn't parse (mismatched brackets, an
5//! unquoted key — real failure shapes seen from a small local model in
6//! production), and the answer reaches the user with a broken/invisible
7//! card instead of a working one. `Agent::run` must give the model one
8//! bounded repair turn naming the exact parse error, reusing the same
9//! turn-loop `continue` mechanism the grounding-check repair uses — this
10//! is a new trigger condition on existing infrastructure, not a new system.
11
12use std::collections::VecDeque;
13use std::sync::Arc;
14use std::sync::Mutex;
15
16use async_trait::async_trait;
17use tokio::sync::mpsc;
18use tokio_util::sync::CancellationToken;
19
20use tempfile::tempdir;
21
22use vak_agent::{Agent, AgentConfig, TurnOutcome};
23use vak_llm::stream;
24use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
25use vak_llm::{EventStream, LlmError, Provider};
26use vak_permission::PermissionEngine;
27use vak_session::types::{FrozenContract, SessionHeader};
28use vak_session::{SessionLog, SessionPath};
29
30struct Scripted {
31 responses: Mutex<VecDeque<AssistantMessage>>,
32}
33
34#[async_trait]
35impl Provider for Scripted {
36 fn name(&self) -> &str {
37 "scripted"
38 }
39
40 async fn stream(
41 &self,
42 _request: ChatRequest,
43 _cancel: CancellationToken,
44 ) -> Result<EventStream, LlmError> {
45 let next = self.responses.lock().unwrap().pop_front();
46 let (mut sink, rx) = stream::channel(64);
47 match next {
48 Some(m) => {
49 sink.push(stream::StreamEvent::Start { partial: m.clone() });
50 sink.close_message(m).await;
51 }
52 None => sink.close_error(LlmError::Parse("exhausted".into())).await,
53 }
54 Ok(rx)
55 }
56}
57
58fn text_msg(t: &str) -> AssistantMessage {
59 AssistantMessage {
60 content: vec![ContentBlock::text(t)],
61 stop_reason: StopReason::EndTurn,
62 usage: Usage {
63 input_tokens: 1,
64 output_tokens: 1,
65 ..Default::default()
66 },
67 model: "test-model".into(),
68 response_id: None,
69 }
70}
71
72async fn build_agent(
73 dir: &tempfile::TempDir,
74 session_id: &str,
75 responses: Vec<AssistantMessage>,
76) -> Agent {
77 let header = SessionHeader {
78 agent: None,
79 session_id: session_id.into(),
80 created_at: chrono::Utc::now(),
81 cwd: dir.path().to_path_buf(),
82 parent_session_id: None,
83 contract_id: None,
84 work_item_id: None,
85 conversation: None,
86 contract: FrozenContract {
87 app_version: "0".into(),
88 provider: "scripted".into(),
89 model: "test-model".into(),
90 route_ladder: Vec::new(),
91 route_objective: String::new(),
92 route_annotations: Vec::new(),
93 system_prompt: "sys".into(),
94 permission_mode: "full-access".into(),
95 capabilities: Vec::new(),
96 prompt_layers: Vec::new(),
97 },
98 };
99 let home = dir.path().join("home");
100 std::fs::create_dir_all(&home).unwrap();
101 let log = SessionLog::create(
102 SessionPath::new_session_file(&home, dir.path(), session_id),
103 header,
104 )
105 .unwrap();
106
107 Agent::new(
108 Arc::new(Scripted {
109 responses: Mutex::new(VecDeque::from(responses)),
110 }),
111 log,
112 {
113 let mut cfg = AgentConfig::new("sys");
114 cfg.model = "test-model".into();
115 cfg.mode = vak_permission::Mode::FullAccess;
116 cfg.permission = Some(Arc::new(PermissionEngine::default()));
117 cfg
118 },
119 )
120}
121
122// Raw ledger, not the model-visible projection: these tests are about the
123// repair loop's mechanics, which a closed turn's full record deliberately
124// no longer preserves (docs/design/68-context-engine.md §10) — a rejected
125// draft and a mid-turn nudge are dropped once the turn closes.
126fn assistant_texts(agent: &Agent) -> Vec<String> {
127 futures::executor::block_on(async {
128 agent
129 .session
130 .lock()
131 .await
132 .message_chain()
133 .iter()
134 .filter(|(_, m)| m.role == vak_llm::types::Role::Assistant)
135 .flat_map(|(_, m)| m.content.iter())
136 .filter_map(|b| match b {
137 ContentBlock::Text { text } => Some(text.clone()),
138 _ => None,
139 })
140 .collect()
141 })
142}
143
144fn user_texts(agent: &Agent) -> Vec<String> {
145 futures::executor::block_on(async {
146 agent
147 .session
148 .lock()
149 .await
150 .message_chain()
151 .iter()
152 .filter(|(_, m)| m.role == vak_llm::types::Role::User)
153 .flat_map(|(_, m)| m.content.iter())
154 .filter_map(|b| match b {
155 ContentBlock::Text { text } => Some(text.clone()),
156 _ => None,
157 })
158 .collect()
159 })
160}
161
162// The exact malformed shape from the live bug: a `decision` fence whose
163// payload has a bare (unquoted) key, `precision":1` instead of `"precision":1`.
164const REAL_MALFORMED_FENCE: &str = "```vak\n{\"semantic_type\":\"decision\",\"payload\":{\"title\":\"Capability Confirmation\",\"choices\":[{\"name\":\"Yes\",\"reason\":\"ok\"}],precision\":1}}\n```\n\n### Explanation\n\nYes, I can do that.";
165
166const VALID_FENCE: &str = "```vak\n{\"semantic_type\":\"metric\",\"payload\":{\"label\":\"Uptime\",\"value\":99.9,\"unit\":\"%\"}}\n```";
167
168#[tokio::test]
169async fn malformed_fence_gets_one_repair_turn_naming_the_parse_error() {
170 let dir = tempdir().unwrap();
171 let mut agent = build_agent(
172 &dir,
173 "fence-repair",
174 vec![text_msg(REAL_MALFORMED_FENCE), text_msg(VALID_FENCE)],
175 )
176 .await;
177
178 let outcome = agent
179 .run(
180 "can you make charts",
181 &Default::default(),
182 CancellationToken::new(),
183 mpsc::channel(64).0,
184 )
185 .await;
186 assert!(
187 matches!(outcome, TurnOutcome::Completed { .. }),
188 "got {outcome:?}"
189 );
190
191 let texts = assistant_texts(&agent);
192 assert!(
193 texts
194 .iter()
195 .any(|t| t.contains("\"semantic_type\":\"metric\"")),
196 "final state must contain the repaired, valid fence: {texts:?}"
197 );
198
199 let users = user_texts(&agent);
200 assert!(
201 users
202 .iter()
203 .any(|t| t.contains("[fence-check]") && t.contains("invalid JSON")),
204 "expected a fence-check repair nudge naming the parse failure: {users:?}"
205 );
206}
207
208#[tokio::test]
209async fn repair_is_bounded_to_one_attempt_not_a_loop() {
210 let dir = tempdir().unwrap();
211 let mut agent = build_agent(
212 &dir,
213 "fence-repair-bounded",
214 vec![
215 text_msg(REAL_MALFORMED_FENCE),
216 text_msg(REAL_MALFORMED_FENCE), // still broken after the nudge
217 ],
218 )
219 .await;
220
221 let outcome = agent
222 .run(
223 "can you make charts",
224 &Default::default(),
225 CancellationToken::new(),
226 mpsc::channel(64).0,
227 )
228 .await;
229 assert!(
230 matches!(outcome, TurnOutcome::Completed { .. }),
231 "got {outcome:?}"
232 );
233
234 let texts = assistant_texts(&agent);
235 assert_eq!(
236 texts
237 .iter()
238 .filter(|t| t.contains("Capability Confirmation"))
239 .count(),
240 2,
241 "exactly one retry — not zero, not an infinite loop: {texts:?}"
242 );
243}
244
245#[tokio::test]
246async fn a_valid_fence_is_never_touched() {
247 let dir = tempdir().unwrap();
248 let mut agent = build_agent(&dir, "fence-valid", vec![text_msg(VALID_FENCE)]).await;
249
250 let outcome = agent
251 .run(
252 "what's our uptime",
253 &Default::default(),
254 CancellationToken::new(),
255 mpsc::channel(64).0,
256 )
257 .await;
258 assert!(
259 matches!(outcome, TurnOutcome::Completed { .. }),
260 "got {outcome:?}"
261 );
262
263 let users = user_texts(&agent);
264 assert!(
265 !users.iter().any(|t| t.contains("[fence-check]")),
266 "a syntactically valid fence must never trigger a repair nudge: {users:?}"
267 );
268}
269
270#[tokio::test]
271async fn plain_prose_with_no_fence_is_never_touched() {
272 let dir = tempdir().unwrap();
273 let mut agent = build_agent(
274 &dir,
275 "fence-none",
276 vec![text_msg("Paris is the capital of France.")],
277 )
278 .await;
279
280 let outcome = agent
281 .run(
282 "what's the capital of france",
283 &Default::default(),
284 CancellationToken::new(),
285 mpsc::channel(64).0,
286 )
287 .await;
288 assert!(
289 matches!(outcome, TurnOutcome::Completed { .. }),
290 "got {outcome:?}"
291 );
292
293 let users = user_texts(&agent);
294 assert!(
295 !users.iter().any(|t| t.contains("[fence-check]")),
296 "plain prose with no fence must never trigger a repair nudge: {users:?}"
297 );
298}
299

Malformed Fence Check

Indexing the workspace…

Malformed Fence Check