Vakyartha documentation

1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2
3//! Topic-mismatch enforcement (docs/design/68-context-engine.md §7).
4//! Regression coverage for a real defect found live on the shipped 3.5.0
5//! build: asked "what is the current top news in AI", the model correctly
6//! called `tavily_search`, got real AI-news results back, and then wrote an
7//! `emit_metric_card` for "Noida Weather, 28°C" — a payload copied from an
8//! unrelated, much older turn still sitting in its own context — instead of
9//! answering from the evidence it had just retrieved. The freshness check
10//! did not catch this: a real retrieval HAD succeeded this run, so freshness
11//! had nothing to say; the card's own content was simply unrelated to both
12//! the question and the evidence.
13
14use std::collections::VecDeque;
15use std::sync::{Arc, Mutex};
16
17use async_trait::async_trait;
18use serde_json::Value;
19use tokio::sync::mpsc;
20use tokio_util::sync::CancellationToken;
21
22use tempfile::tempdir;
23
24use vak_agent::{Agent, AgentConfig, TurnOutcome};
25use vak_llm::stream;
26use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
27use vak_llm::{EventStream, LlmError, Provider};
28use vak_permission::PermissionEngine;
29use vak_session::types::{FrozenContract, SessionHeader};
30use vak_session::{SessionLog, SessionPath};
31use vak_tools::context::ToolContext;
32use vak_tools::{Tool, ToolOutput};
33
34struct Scripted {
35 responses: Mutex<VecDeque<AssistantMessage>>,
36}
37
38#[async_trait]
39impl Provider for Scripted {
40 fn name(&self) -> &str {
41 "scripted"
42 }
43
44 async fn stream(
45 &self,
46 _request: ChatRequest,
47 _cancel: CancellationToken,
48 ) -> Result<EventStream, LlmError> {
49 let next = self.responses.lock().unwrap().pop_front();
50 let (mut sink, rx) = stream::channel(16);
51 match next {
52 Some(m) => sink.close_message(m).await,
53 None => sink.close_error(LlmError::Parse("exhausted".into())).await,
54 }
55 Ok(rx)
56 }
57}
58
59struct FakeSearchTool;
60
61#[async_trait]
62impl Tool for FakeSearchTool {
63 fn name(&self) -> &str {
64 "tavily_search"
65 }
66 fn description(&self) -> &str {
67 "fake search tool for tests"
68 }
69 fn schema(&self) -> Value {
70 serde_json::json!({"type": "object", "properties": {"query": {"type": "string"}}})
71 }
72 async fn execute(&self, _args: &Value, _ctx: &ToolContext) -> ToolOutput {
73 ToolOutput::ok(
74 "Title: AI News | Latest News | Insights Powering AI-Driven Business\n\
75 URL: https://www.artificialintelligence-news.example/\n\
76 OpenAI today unveiled GPT-6, its newest model, at a launch event.\n",
77 )
78 }
79}
80
81struct FakeCardTool;
82
83#[async_trait]
84impl Tool for FakeCardTool {
85 fn name(&self) -> &str {
86 "emit_metric_card"
87 }
88 fn description(&self) -> &str {
89 "fake card tool for tests"
90 }
91 fn schema(&self) -> Value {
92 serde_json::json!({"type": "object"})
93 }
94 fn presents_cards(&self) -> bool {
95 true
96 }
97 async fn execute(&self, _args: &Value, _ctx: &ToolContext) -> ToolOutput {
98 ToolOutput::ok("{\"ok\":true}")
99 }
100}
101
102struct FakeEntityTool;
103
104#[async_trait]
105impl Tool for FakeEntityTool {
106 fn name(&self) -> &str {
107 "emit_entity_card"
108 }
109 fn description(&self) -> &str {
110 "fake entity card tool for tests"
111 }
112 fn schema(&self) -> Value {
113 serde_json::json!({"type": "object"})
114 }
115 fn presents_cards(&self) -> bool {
116 true
117 }
118 async fn execute(&self, _args: &Value, _ctx: &ToolContext) -> ToolOutput {
119 ToolOutput::ok("{\"ok\":true}")
120 }
121}
122
123fn search_call(id: &str) -> AssistantMessage {
124 AssistantMessage {
125 content: vec![ContentBlock::ToolUse {
126 id: id.into(),
127 name: "tavily_search".into(),
128 input: serde_json::json!({"query": "current top news in AI"}),
129 }],
130 stop_reason: StopReason::ToolUse,
131 usage: Usage::default(),
132 model: "test-model".into(),
133 response_id: None,
134 }
135}
136
137fn weather_card_call(id: &str) -> AssistantMessage {
138 AssistantMessage {
139 content: vec![ContentBlock::ToolUse {
140 id: id.into(),
141 name: "emit_metric_card".into(),
142 input: serde_json::json!({
143 "semantic_type": "weather",
144 "payload": {"label": "Noida Weather", "unit": "Celsius", "value": "28°C"}
145 }),
146 }],
147 stop_reason: StopReason::ToolUse,
148 usage: Usage::default(),
149 model: "test-model".into(),
150 response_id: None,
151 }
152}
153
154fn ai_entity_card_call(id: &str) -> AssistantMessage {
155 AssistantMessage {
156 content: vec![ContentBlock::ToolUse {
157 id: id.into(),
158 name: "emit_entity_card".into(),
159 input: serde_json::json!({
160 "semantic_type": "entity",
161 "payload": {"title": "OpenAI announces GPT-6", "summary": "newest model launch"}
162 }),
163 }],
164 stop_reason: StopReason::ToolUse,
165 usage: Usage::default(),
166 model: "test-model".into(),
167 response_id: None,
168 }
169}
170
171fn text_msg(t: &str) -> AssistantMessage {
172 AssistantMessage {
173 content: vec![ContentBlock::text(t)],
174 stop_reason: StopReason::EndTurn,
175 usage: Usage {
176 input_tokens: 1,
177 output_tokens: 1,
178 ..Default::default()
179 },
180 model: "test-model".into(),
181 response_id: None,
182 }
183}
184
185async fn build_agent(
186 dir: &tempfile::TempDir,
187 session_id: &str,
188 responses: Vec<AssistantMessage>,
189) -> Agent {
190 let header = SessionHeader {
191 agent: None,
192 session_id: session_id.into(),
193 created_at: chrono::Utc::now(),
194 cwd: dir.path().to_path_buf(),
195 parent_session_id: None,
196 contract_id: None,
197 work_item_id: None,
198 conversation: None,
199 contract: FrozenContract {
200 app_version: "0".into(),
201 provider: "scripted".into(),
202 model: "test-model".into(),
203 route_ladder: Vec::new(),
204 route_objective: String::new(),
205 route_annotations: Vec::new(),
206 system_prompt: "sys".into(),
207 permission_mode: "full-access".into(),
208 capabilities: Vec::new(),
209 prompt_layers: Vec::new(),
210 },
211 };
212 let home = dir.path().join("home");
213 std::fs::create_dir_all(&home).unwrap();
214 let log = SessionLog::create(
215 SessionPath::new_session_file(&home, dir.path(), session_id),
216 header,
217 )
218 .unwrap();
219
220 Agent::new(
221 Arc::new(Scripted {
222 responses: Mutex::new(VecDeque::from(responses)),
223 }),
224 log,
225 {
226 let mut cfg = AgentConfig::new("sys");
227 cfg.model = "test-model".into();
228 cfg.tools = vec![
229 Arc::new(FakeSearchTool),
230 Arc::new(FakeCardTool),
231 Arc::new(FakeEntityTool),
232 ];
233 cfg.retrieval_check = Some(Arc::new(|name: &str, _: &Value| name == "tavily_search"));
234 cfg.mode = vak_permission::Mode::FullAccess;
235 cfg.permission = Some(Arc::new(PermissionEngine::default()));
236 cfg
237 },
238 )
239}
240
241/// The tool_result content for a given `tool_use_id`, raw from the ledger.
242/// This harness never wires `presentation_rebuild` (that hook lives in
243/// `Core`, not `Agent`), so a card call's success or gating is only visible
244/// here — never in `session.presentations()`, which stays empty regardless.
245fn tool_result_for(agent: &Agent, tool_use_id: &str) -> Option<(String, bool)> {
246 futures::executor::block_on(async {
247 agent
248 .session
249 .lock()
250 .await
251 .message_chain()
252 .iter()
253 .flat_map(|(_, m)| m.content.clone())
254 .find_map(|b| match b {
255 ContentBlock::ToolResult {
256 tool_use_id: id,
257 content,
258 is_error,
259 } if id == tool_use_id => Some((content, is_error)),
260 _ => None,
261 })
262 })
263}
264
265fn user_texts(agent: &Agent) -> Vec<String> {
266 futures::executor::block_on(async {
267 agent
268 .session
269 .lock()
270 .await
271 .message_chain()
272 .iter()
273 .filter(|(_, m)| m.role == vak_llm::types::Role::User)
274 .flat_map(|(_, m)| m.content.iter())
275 .filter_map(|b| match b {
276 ContentBlock::Text { text } => Some(text.clone()),
277 _ => None,
278 })
279 .collect()
280 })
281}
282
283#[tokio::test]
284async fn the_real_regression_is_gated_and_evidence_is_shown_instead() {
285 let dir = tempdir().unwrap();
286 let mut agent = build_agent(
287 &dir,
288 "topic-mismatch-real-bug",
289 vec![
290 // Exactly what really happened: correct search, then an
291 // unrelated card written from an older turn, twice.
292 search_call("s1"),
293 weather_card_call("c1"),
294 weather_card_call("c2"),
295 ],
296 )
297 .await;
298
299 let outcome = agent
300 .run(
301 "what is the current top news in AI",
302 &Default::default(),
303 CancellationToken::new(),
304 mpsc::channel(64).0,
305 )
306 .await;
307
308 // The first weather-card attempt is refused, not executed, by the
309 // topic gate specifically.
310 let (content, is_error) = tool_result_for(&agent, "c1").expect("a result for c1");
311 assert!(is_error, "c1 must be refused, not executed: {content}");
312 assert!(
313 content.starts_with("[topic-mismatch]"),
314 "c1 must be refused by the topic gate specifically: {content}"
315 );
316 // The second attempt is the same mismatched card again: the turn ends
317 // there (matching the freshness gate's own precedent) before a second
318 // tool_result is ever recorded for it.
319 assert!(tool_result_for(&agent, "c2").is_none());
320
321 match &outcome {
322 TurnOutcome::Completed { response } => {
323 let text = response.text_content();
324 assert!(
325 text.contains("OpenAI") || text.contains("GPT-6") || text.contains("AI News"),
326 "the real evidence gathered this run must surface instead of nothing: {text}"
327 );
328 assert!(
329 !text.contains("Noida"),
330 "the wrong card's content must not leak through: {text}"
331 );
332 }
333 other => panic!("expected a completed turn, got {other:?}"),
334 }
335
336 let nudges = user_texts(&agent);
337 assert_eq!(
338 nudges
339 .iter()
340 .filter(|t| t.starts_with("[topic-mismatch]"))
341 .count(),
342 0,
343 "the gate answers with a tool-result error, not a session-visible nudge in this path: {nudges:?}"
344 );
345}
346
347#[tokio::test]
348async fn a_correctly_retrieved_card_that_renames_the_topic_is_not_gated() {
349 let dir = tempdir().unwrap();
350 let mut agent = build_agent(
351 &dir,
352 "topic-mismatch-paraphrase-safe",
353 vec![
354 search_call("s1"),
355 // Titled from what the search actually said, not from the
356 // user's own words — must NOT be gated.
357 ai_entity_card_call("c1"),
358 text_msg("OpenAI just announced GPT-6."),
359 ],
360 )
361 .await;
362
363 let outcome = agent
364 .run(
365 "what is the current top news in AI",
366 &Default::default(),
367 CancellationToken::new(),
368 mpsc::channel(64).0,
369 )
370 .await;
371
372 assert!(
373 matches!(&outcome, TurnOutcome::Completed { .. }),
374 "got {outcome:?}"
375 );
376 let (content, is_error) =
377 tool_result_for(&agent, "c1").expect("a result for the entity card call");
378 assert!(
379 !is_error,
380 "the correctly-derived, differently-titled card must not be gated: {content}"
381 );
382 assert_eq!(content, "{\"ok\":true}");
383}
384

1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2
3//! Topic-mismatch enforcement (docs/design/68-context-engine.md §7).
4//! Regression coverage for a real defect found live on the shipped 3.5.0
5//! build: asked "what is the current top news in AI", the model correctly
6//! called `tavily_search`, got real AI-news results back, and then wrote an
7//! `emit_metric_card` for "Noida Weather, 28°C" — a payload copied from an
8//! unrelated, much older turn still sitting in its own context — instead of
9//! answering from the evidence it had just retrieved. The freshness check
10//! did not catch this: a real retrieval HAD succeeded this run, so freshness
11//! had nothing to say; the card's own content was simply unrelated to both
12//! the question and the evidence.
13
14use std::collections::VecDeque;
15use std::sync::{Arc, Mutex};
16
17use async_trait::async_trait;
18use serde_json::Value;
19use tokio::sync::mpsc;
20use tokio_util::sync::CancellationToken;
21
22use tempfile::tempdir;
23
24use vak_agent::{Agent, AgentConfig, TurnOutcome};
25use vak_llm::stream;
26use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
27use vak_llm::{EventStream, LlmError, Provider};
28use vak_permission::PermissionEngine;
29use vak_session::types::{FrozenContract, SessionHeader};
30use vak_session::{SessionLog, SessionPath};
31use vak_tools::context::ToolContext;
32use vak_tools::{Tool, ToolOutput};
33
34struct Scripted {
35 responses: Mutex<VecDeque<AssistantMessage>>,
36}
37
38#[async_trait]
39impl Provider for Scripted {
40 fn name(&self) -> &str {
41 "scripted"
42 }
43
44 async fn stream(
45 &self,
46 _request: ChatRequest,
47 _cancel: CancellationToken,
48 ) -> Result<EventStream, LlmError> {
49 let next = self.responses.lock().unwrap().pop_front();
50 let (mut sink, rx) = stream::channel(16);
51 match next {
52 Some(m) => sink.close_message(m).await,
53 None => sink.close_error(LlmError::Parse("exhausted".into())).await,
54 }
55 Ok(rx)
56 }
57}
58
59struct FakeSearchTool;
60
61#[async_trait]
62impl Tool for FakeSearchTool {
63 fn name(&self) -> &str {
64 "tavily_search"
65 }
66 fn description(&self) -> &str {
67 "fake search tool for tests"
68 }
69 fn schema(&self) -> Value {
70 serde_json::json!({"type": "object", "properties": {"query": {"type": "string"}}})
71 }
72 async fn execute(&self, _args: &Value, _ctx: &ToolContext) -> ToolOutput {
73 ToolOutput::ok(
74 "Title: AI News | Latest News | Insights Powering AI-Driven Business\n\
75 URL: https://www.artificialintelligence-news.example/\n\
76 OpenAI today unveiled GPT-6, its newest model, at a launch event.\n",
77 )
78 }
79}
80
81struct FakeCardTool;
82
83#[async_trait]
84impl Tool for FakeCardTool {
85 fn name(&self) -> &str {
86 "emit_metric_card"
87 }
88 fn description(&self) -> &str {
89 "fake card tool for tests"
90 }
91 fn schema(&self) -> Value {
92 serde_json::json!({"type": "object"})
93 }
94 fn presents_cards(&self) -> bool {
95 true
96 }
97 async fn execute(&self, _args: &Value, _ctx: &ToolContext) -> ToolOutput {
98 ToolOutput::ok("{\"ok\":true}")
99 }
100}
101
102struct FakeEntityTool;
103
104#[async_trait]
105impl Tool for FakeEntityTool {
106 fn name(&self) -> &str {
107 "emit_entity_card"
108 }
109 fn description(&self) -> &str {
110 "fake entity card tool for tests"
111 }
112 fn schema(&self) -> Value {
113 serde_json::json!({"type": "object"})
114 }
115 fn presents_cards(&self) -> bool {
116 true
117 }
118 async fn execute(&self, _args: &Value, _ctx: &ToolContext) -> ToolOutput {
119 ToolOutput::ok("{\"ok\":true}")
120 }
121}
122
123fn search_call(id: &str) -> AssistantMessage {
124 AssistantMessage {
125 content: vec![ContentBlock::ToolUse {
126 id: id.into(),
127 name: "tavily_search".into(),
128 input: serde_json::json!({"query": "current top news in AI"}),
129 }],
130 stop_reason: StopReason::ToolUse,
131 usage: Usage::default(),
132 model: "test-model".into(),
133 response_id: None,
134 }
135}
136
137fn weather_card_call(id: &str) -> AssistantMessage {
138 AssistantMessage {
139 content: vec![ContentBlock::ToolUse {
140 id: id.into(),
141 name: "emit_metric_card".into(),
142 input: serde_json::json!({
143 "semantic_type": "weather",
144 "payload": {"label": "Noida Weather", "unit": "Celsius", "value": "28°C"}
145 }),
146 }],
147 stop_reason: StopReason::ToolUse,
148 usage: Usage::default(),
149 model: "test-model".into(),
150 response_id: None,
151 }
152}
153
154fn ai_entity_card_call(id: &str) -> AssistantMessage {
155 AssistantMessage {
156 content: vec![ContentBlock::ToolUse {
157 id: id.into(),
158 name: "emit_entity_card".into(),
159 input: serde_json::json!({
160 "semantic_type": "entity",
161 "payload": {"title": "OpenAI announces GPT-6", "summary": "newest model launch"}
162 }),
163 }],
164 stop_reason: StopReason::ToolUse,
165 usage: Usage::default(),
166 model: "test-model".into(),
167 response_id: None,
168 }
169}
170
171fn text_msg(t: &str) -> AssistantMessage {
172 AssistantMessage {
173 content: vec![ContentBlock::text(t)],
174 stop_reason: StopReason::EndTurn,
175 usage: Usage {
176 input_tokens: 1,
177 output_tokens: 1,
178 ..Default::default()
179 },
180 model: "test-model".into(),
181 response_id: None,
182 }
183}
184
185async fn build_agent(
186 dir: &tempfile::TempDir,
187 session_id: &str,
188 responses: Vec<AssistantMessage>,
189) -> Agent {
190 let header = SessionHeader {
191 agent: None,
192 session_id: session_id.into(),
193 created_at: chrono::Utc::now(),
194 cwd: dir.path().to_path_buf(),
195 parent_session_id: None,
196 contract_id: None,
197 work_item_id: None,
198 conversation: None,
199 contract: FrozenContract {
200 app_version: "0".into(),
201 provider: "scripted".into(),
202 model: "test-model".into(),
203 route_ladder: Vec::new(),
204 route_objective: String::new(),
205 route_annotations: Vec::new(),
206 system_prompt: "sys".into(),
207 permission_mode: "full-access".into(),
208 capabilities: Vec::new(),
209 prompt_layers: Vec::new(),
210 },
211 };
212 let home = dir.path().join("home");
213 std::fs::create_dir_all(&home).unwrap();
214 let log = SessionLog::create(
215 SessionPath::new_session_file(&home, dir.path(), session_id),
216 header,
217 )
218 .unwrap();
219
220 Agent::new(
221 Arc::new(Scripted {
222 responses: Mutex::new(VecDeque::from(responses)),
223 }),
224 log,
225 {
226 let mut cfg = AgentConfig::new("sys");
227 cfg.model = "test-model".into();
228 cfg.tools = vec![
229 Arc::new(FakeSearchTool),
230 Arc::new(FakeCardTool),
231 Arc::new(FakeEntityTool),
232 ];
233 cfg.retrieval_check = Some(Arc::new(|name: &str, _: &Value| name == "tavily_search"));
234 cfg.mode = vak_permission::Mode::FullAccess;
235 cfg.permission = Some(Arc::new(PermissionEngine::default()));
236 cfg
237 },
238 )
239}
240
241/// The tool_result content for a given `tool_use_id`, raw from the ledger.
242/// This harness never wires `presentation_rebuild` (that hook lives in
243/// `Core`, not `Agent`), so a card call's success or gating is only visible
244/// here — never in `session.presentations()`, which stays empty regardless.
245fn tool_result_for(agent: &Agent, tool_use_id: &str) -> Option<(String, bool)> {
246 futures::executor::block_on(async {
247 agent
248 .session
249 .lock()
250 .await
251 .message_chain()
252 .iter()
253 .flat_map(|(_, m)| m.content.clone())
254 .find_map(|b| match b {
255 ContentBlock::ToolResult {
256 tool_use_id: id,
257 content,
258 is_error,
259 } if id == tool_use_id => Some((content, is_error)),
260 _ => None,
261 })
262 })
263}
264
265fn user_texts(agent: &Agent) -> Vec<String> {
266 futures::executor::block_on(async {
267 agent
268 .session
269 .lock()
270 .await
271 .message_chain()
272 .iter()
273 .filter(|(_, m)| m.role == vak_llm::types::Role::User)
274 .flat_map(|(_, m)| m.content.iter())
275 .filter_map(|b| match b {
276 ContentBlock::Text { text } => Some(text.clone()),
277 _ => None,
278 })
279 .collect()
280 })
281}
282
283#[tokio::test]
284async fn the_real_regression_is_gated_and_evidence_is_shown_instead() {
285 let dir = tempdir().unwrap();
286 let mut agent = build_agent(
287 &dir,
288 "topic-mismatch-real-bug",
289 vec![
290 // Exactly what really happened: correct search, then an
291 // unrelated card written from an older turn, twice.
292 search_call("s1"),
293 weather_card_call("c1"),
294 weather_card_call("c2"),
295 ],
296 )
297 .await;
298
299 let outcome = agent
300 .run(
301 "what is the current top news in AI",
302 &Default::default(),
303 CancellationToken::new(),
304 mpsc::channel(64).0,
305 )
306 .await;
307
308 // The first weather-card attempt is refused, not executed, by the
309 // topic gate specifically.
310 let (content, is_error) = tool_result_for(&agent, "c1").expect("a result for c1");
311 assert!(is_error, "c1 must be refused, not executed: {content}");
312 assert!(
313 content.starts_with("[topic-mismatch]"),
314 "c1 must be refused by the topic gate specifically: {content}"
315 );
316 // The second attempt is the same mismatched card again: the turn ends
317 // there (matching the freshness gate's own precedent) before a second
318 // tool_result is ever recorded for it.
319 assert!(tool_result_for(&agent, "c2").is_none());
320
321 match &outcome {
322 TurnOutcome::Completed { response } => {
323 let text = response.text_content();
324 assert!(
325 text.contains("OpenAI") || text.contains("GPT-6") || text.contains("AI News"),
326 "the real evidence gathered this run must surface instead of nothing: {text}"
327 );
328 assert!(
329 !text.contains("Noida"),
330 "the wrong card's content must not leak through: {text}"
331 );
332 }
333 other => panic!("expected a completed turn, got {other:?}"),
334 }
335
336 let nudges = user_texts(&agent);
337 assert_eq!(
338 nudges
339 .iter()
340 .filter(|t| t.starts_with("[topic-mismatch]"))
341 .count(),
342 0,
343 "the gate answers with a tool-result error, not a session-visible nudge in this path: {nudges:?}"
344 );
345}
346
347#[tokio::test]
348async fn a_correctly_retrieved_card_that_renames_the_topic_is_not_gated() {
349 let dir = tempdir().unwrap();
350 let mut agent = build_agent(
351 &dir,
352 "topic-mismatch-paraphrase-safe",
353 vec![
354 search_call("s1"),
355 // Titled from what the search actually said, not from the
356 // user's own words — must NOT be gated.
357 ai_entity_card_call("c1"),
358 text_msg("OpenAI just announced GPT-6."),
359 ],
360 )
361 .await;
362
363 let outcome = agent
364 .run(
365 "what is the current top news in AI",
366 &Default::default(),
367 CancellationToken::new(),
368 mpsc::channel(64).0,
369 )
370 .await;
371
372 assert!(
373 matches!(&outcome, TurnOutcome::Completed { .. }),
374 "got {outcome:?}"
375 );
376 let (content, is_error) =
377 tool_result_for(&agent, "c1").expect("a result for the entity card call");
378 assert!(
379 !is_error,
380 "the correctly-derived, differently-titled card must not be gated: {content}"
381 );
382 assert_eq!(content, "{\"ok\":true}");
383}
384

Topic Mismatch Check

Indexing the workspace…

Topic Mismatch Check