1. 1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2. 2
3. 3//! The presentation check: an answer the app's own signal/recipe detection
4. 4//! says should be a card, written as prose with no card emitted, gets exactly
5. 5//! one nudge. Real trigger: a gpt-5.6-luna answer to "how did the Indian stock
6. 6//! market perform last week" that was a markdown table plus bullets, with the
7. 7//! `emit_*_card` tools offered and unused.
8. 8
9. 9use std::collections::VecDeque;
10. 10use std::sync::Arc;
11. 11use std::sync::Mutex;
12. 12
13. 13use async_trait::async_trait;
14. 14use tokio::sync::mpsc;
15. 15use tokio_util::sync::CancellationToken;
16. 16
17. 17use tempfile::tempdir;
18. 18
19. 19use vak_agent::{Agent, AgentConfig, TurnOutcome};
20. 20use vak_llm::stream;
21. 21use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
22. 22use vak_llm::{EventStream, LlmError, Provider};
23. 23use vak_permission::PermissionEngine;
24. 24use vak_session::types::{FrozenContract, SessionHeader};
25. 25use vak_session::{SessionLog, SessionPath};
26. 26use vak_tools::context::ToolContext;
27. 27use vak_tools::{Tool, ToolOutput};
28. 28
29. 29type Requests = Arc<Mutex<Vec<Vec<String>>>>;
30. 30
31. 31struct Scripted {
32. 32 responses: Mutex<VecDeque<AssistantMessage>>,
33. 33 /// The tool names each request actually declared.
34. 34 requests: Requests,
35. 35}
36. 36
37. 37#[async_trait]
38. 38impl Provider for Scripted {
39. 39 fn name(&self) -> &str {
40. 40 "scripted"
41. 41 }
42. 42
43. 43 async fn stream(
44. 44 &self,
45. 45 request: ChatRequest,
46. 46 _cancel: CancellationToken,
47. 47 ) -> Result<EventStream, LlmError> {
48. 48 self.requests
49. 49 .lock()
50. 50 .unwrap()
51. 51 .push(request.tools.iter().map(|t| t.name.clone()).collect());
52. 52 let next = self.responses.lock().unwrap().pop_front();
53. 53 let (mut sink, rx) = stream::channel(64);
54. 54 match next {
55. 55 Some(m) => {
56. 56 sink.push(stream::StreamEvent::Start { partial: m.clone() });
57. 57 sink.close_message(m).await;
58. 58 }
59. 59 None => sink.close_error(LlmError::Parse("exhausted".into())).await,
60. 60 }
61. 61 Ok(rx)
62. 62 }
63. 63}
64. 64
65. 65/// Stand-in for `vak_core::presentation_tools::EmitCardTool` — returns the
66. 66/// same `{"semantic_type","payload"}` envelope shape the real tool wraps
67. 67/// its arguments in, without pulling in the vak-core dependency.
68. 68struct FakeEmitChartCard;
69. 69
70. 70#[async_trait]
71. 71impl Tool for FakeEmitChartCard {
72. 72 fn name(&self) -> &str {
73. 73 "emit_chart_card"
74. 74 }
75. 75
76. 76 fn description(&self) -> &str {
77. 77 "test stand-in"
78. 78 }
79. 79
80. 80 fn schema(&self) -> serde_json::Value {
81. 81 serde_json::json!({"type": "object"})
82. 82 }
83. 83
84. 84 fn presents_cards(&self) -> bool {
85. 85 true
86. 86 }
87. 87
88. 88 async fn execute(&self, args: &serde_json::Value, _ctx: &ToolContext) -> ToolOutput {
89. 89 let envelope = serde_json::json!({
90. 90 "semantic_type": "chart",
91. 91 "payload": args.get("payload").cloned().unwrap_or(serde_json::json!({})),
92. 92 });
93. 93 ToolOutput::ok(envelope.to_string())
94. 94 }
95. 95}
96. 96
97. 97fn text_msg(t: &str) -> AssistantMessage {
98. 98 AssistantMessage {
99. 99 content: vec![ContentBlock::text(t)],
100. 100 stop_reason: StopReason::EndTurn,
101. 101 usage: Usage {
102. 102 input_tokens: 1,
103. 103 output_tokens: 1,
104. 104 ..Default::default()
105. 105 },
106. 106 model: "test-model".into(),
107. 107 response_id: None,
108. 108 }
109. 109}
110. 110
111. 111fn tool_call_msg(id: &str, name: &str, input: serde_json::Value) -> AssistantMessage {
112. 112 AssistantMessage {
113. 113 content: vec![ContentBlock::ToolUse {
114. 114 id: id.into(),
115. 115 name: name.into(),
116. 116 input,
117. 117 }],
118. 118 stop_reason: StopReason::ToolUse,
119. 119 usage: Usage {
120. 120 input_tokens: 1,
121. 121 output_tokens: 1,
122. 122 ..Default::default()
123. 123 },
124. 124 model: "test-model".into(),
125. 125 response_id: None,
126. 126 }
127. 127}
128. 128
129. 129async fn build_agent(
130. 130 dir: &tempfile::TempDir,
131. 131 session_id: &str,
132. 132 responses: Vec<AssistantMessage>,
133. 133 with_check: bool,
134. 134) -> (Agent, Requests) {
135. 135 let header = SessionHeader {
136. 136 agent: None,
137. 137 session_id: session_id.into(),
138. 138 created_at: chrono::Utc::now(),
139. 139 cwd: dir.path().to_path_buf(),
140. 140 parent_session_id: None,
141. 141 contract_id: None,
142. 142 work_item_id: None,
143. 143 conversation: None,
144. 144 contract: FrozenContract {
145. 145 app_version: "0".into(),
146. 146 provider: "scripted".into(),
147. 147 model: "test-model".into(),
148. 148 route_ladder: Vec::new(),
149. 149 route_objective: String::new(),
150. 150 route_annotations: Vec::new(),
151. 151 system_prompt: "sys".into(),
152. 152 permission_mode: "full-access".into(),
153. 153 capabilities: Vec::new(),
154. 154 prompt_layers: Vec::new(),
155. 155 },
156. 156 };
157. 157 let home = dir.path().join("home");
158. 158 std::fs::create_dir_all(&home).unwrap();
159. 159 let log = SessionLog::create(
160. 160 SessionPath::new_session_file(&home, dir.path(), session_id),
161. 161 header,
162. 162 )
163. 163 .unwrap();
164. 164
165. 165 let requests = Requests::default();
166. 166 let agent = Agent::new(
167. 167 Arc::new(Scripted {
168. 168 responses: Mutex::new(VecDeque::from(responses)),
169. 169 requests: requests.clone(),
170. 170 }),
171. 171 log,
172. 172 {
173. 173 let mut cfg = AgentConfig::new("sys");
174. 174 cfg.model = "test-model".into();
175. 175 cfg.mode = vak_permission::Mode::FullAccess;
176. 176 cfg.permission = Some(Arc::new(PermissionEngine::default()));
177. 177 cfg.tools = vec![Arc::new(FakeEmitChartCard)];
178. 178 // As in production: a card tool the request did not predict is
179. 179 // deferred, so it is not declared until something loads it.
180. 180 cfg.tool_definitions = Some(vec![
181. 181 vak_llm::ToolDefinition::new(
182. 182 "emit_chart_card",
183. 183 "test stand-in",
184. 184 serde_json::json!({"type": "object"}),
185. 185 )
186. 186 .deferred(),
187. 187 ]);
188. 188 if with_check {
189. 189 cfg.presentation_check = Some(Arc::new(|text, offered| {
190. 190 (text.contains("TABLE") && offered.iter().any(|t| t == "emit_chart_card")).then(
191. 191 || vak_agent::PresentationNudge {
192. 192 tool: "emit_chart_card".into(),
193. 193 text:
194. 194 "[presentation-check]: this reads as a card; call emit_chart_card."
195. 195 .into(),
196. 196 },
197. 197 )
198. 198 }));
199. 199 }
200. 200 cfg
201. 201 },
202. 202 );
203. 203 (agent, requests)
204. 204}
205. 205
206. 206// Raw ledger, not the model-visible projection: these tests are about the
207. 207// repair loop's mechanics, which a closed turn's full record deliberately
208. 208// no longer preserves (docs/design/68-context-engine.md §10) — a rejected
209. 209// draft and a mid-turn nudge are dropped once the turn closes.
210. 210fn assistant_texts(agent: &Agent) -> Vec<String> {
211. 211 futures::executor::block_on(async {
212. 212 agent
213. 213 .session
214. 214 .lock()
215. 215 .await
216. 216 .message_chain()
217. 217 .iter()
218. 218 .filter(|(_, m)| m.role == vak_llm::types::Role::Assistant)
219. 219 .flat_map(|(_, m)| m.content.iter())
220. 220 .filter_map(|b| match b {
221. 221 ContentBlock::Text { text } => Some(text.clone()),
222. 222 _ => None,
223. 223 })
224. 224 .collect()
225. 225 })
226. 226}
227. 227
228. 228fn user_texts(agent: &Agent) -> Vec<String> {
229. 229 futures::executor::block_on(async {
230. 230 agent
231. 231 .session
232. 232 .lock()
233. 233 .await
234. 234 .message_chain()
235. 235 .iter()
236. 236 .filter(|(_, m)| m.role == vak_llm::types::Role::User)
237. 237 .flat_map(|(_, m)| m.content.iter())
238. 238 .filter_map(|b| match b {
239. 239 ContentBlock::Text { text } => Some(text.clone()),
240. 240 _ => None,
241. 241 })
242. 242 .collect()
243. 243 })
244. 244}
245. 245
246. 246const PROSE: &str = "Weekly moves as a TABLE:\n\n| Index | Change |\n|---|---|\n| Nifty | -0.22% |";
247. 247const NARRATION: &str = "The chart is shown above.";
248. 248const CHART_INPUT: fn() -> serde_json::Value = || serde_json::json!({"semantic_type": "chart", "payload": {"chart_type": "line", "series": []}});
249. 249
250. 250async fn run(agent: &mut Agent) -> TurnOutcome {
251. 251 agent
252. 252 .run(
253. 253 "how did the market do",
254. 254 &Default::default(),
255. 255 CancellationToken::new(),
256. 256 mpsc::channel(64).0,
257. 257 )
258. 258 .await
259. 259}
260. 260
261. 261#[tokio::test]
262. 262async fn prose_that_reads_as_a_card_gets_one_nudge_and_the_model_can_then_emit_it() {
263. 263 let dir = tempdir().unwrap();
264. 264 let (mut agent, requests) = build_agent(
265. 265 &dir,
266. 266 "pc-nudge",
267. 267 vec![
268. 268 text_msg(PROSE),
269. 269 tool_call_msg("c1", "emit_chart_card", CHART_INPUT()),
270. 270 text_msg(NARRATION),
271. 271 ],
272. 272 true,
273. 273 )
274. 274 .await;
275. 275 assert!(matches!(
276. 276 run(&mut agent).await,
277. 277 TurnOutcome::Completed { .. }
278. 278 ));
279. 279 let users = user_texts(&agent);
280. 280 assert_eq!(
281. 281 users
282. 282 .iter()
283. 283 .filter(|t| t.contains("[presentation-check]"))
284. 284 .count(),
285. 285 1,
286. 286 "{users:?}"
287. 287 );
288. 288 assert!(assistant_texts(&agent).iter().any(|t| t == NARRATION));
289. 289 let requests = requests.lock().unwrap();
290. 290 assert!(
291. 291 !requests[0].contains(&"emit_chart_card".to_string()),
292. 292 "the deferred card tool is not declared before the nudge"
293. 293 );
294. 294 assert!(
295. 295 requests[1].contains(&"emit_chart_card".to_string()),
296. 296 "the nudge loads the card tool it asks for, on a non-Anthropic leg"
297. 297 );
298. 298}
299. 299
300. 300#[tokio::test]
301. 301async fn the_nudge_is_one_shot_and_the_model_may_decline_by_resending() {
302. 302 let dir = tempdir().unwrap();
303. 303 let (mut agent, _) = build_agent(
304. 304 &dir,
305. 305 "pc-once",
306. 306 vec![text_msg(PROSE), text_msg(PROSE)],
307. 307 true,
308. 308 )
309. 309 .await;
310. 310 assert!(matches!(
311. 311 run(&mut agent).await,
312. 312 TurnOutcome::Completed { .. }
313. 313 ));
314. 314 let users = user_texts(&agent);
315. 315 assert_eq!(
316. 316 users
317. 317 .iter()
318. 318 .filter(|t| t.contains("[presentation-check]"))
319. 319 .count(),
320. 320 1,
321. 321 "{users:?}"
322. 322 );
323. 323 assert_eq!(
324. 324 assistant_texts(&agent)
325. 325 .iter()
326. 326 .filter(|t| *t == PROSE)
327. 327 .count(),
328. 328 2
329. 329 );
330. 330}
331. 331
332. 332#[tokio::test]
333. 333async fn no_nudge_when_a_card_was_already_emitted_this_run() {
334. 334 let dir = tempdir().unwrap();
335. 335 let (mut agent, _) = build_agent(
336. 336 &dir,
337. 337 "pc-emitted",
338. 338 vec![
339. 339 tool_call_msg("c1", "emit_chart_card", CHART_INPUT()),
340. 340 text_msg(PROSE),
341. 341 ],
342. 342 true,
343. 343 )
344. 344 .await;
345. 345 assert!(matches!(
346. 346 run(&mut agent).await,
347. 347 TurnOutcome::Completed { .. }
348. 348 ));
349. 349 assert!(
350. 350 !user_texts(&agent)
351. 351 .iter()
352. 352 .any(|t| t.contains("[presentation-check]"))
353. 353 );
354. 354}
355. 355
356. 356#[tokio::test]
357. 357async fn no_nudge_when_the_answer_already_carries_an_inline_card() {
358. 358 let dir = tempdir().unwrap();
359. 359 let with_fence = format!(
360. 360 "{PROSE}\n\n```vak\n{{\"semantic_type\":\"metric\",\"payload\":{{\"label\":\"x\",\"value\":1}}}}\n```"
361. 361 );
362. 362 let (mut agent, _) = build_agent(&dir, "pc-fence", vec![text_msg(&with_fence)], true).await;
363. 363 assert!(matches!(
364. 364 run(&mut agent).await,
365. 365 TurnOutcome::Completed { .. }
366. 366 ));
367. 367 assert!(
368. 368 !user_texts(&agent)
369. 369 .iter()
370. 370 .any(|t| t.contains("[presentation-check]"))
371. 371 );
372. 372}
373. 373
374. 374#[tokio::test]
375. 375async fn with_no_check_configured_the_loop_is_untouched() {
376. 376 let dir = tempdir().unwrap();
377. 377 let (mut agent, _) = build_agent(&dir, "pc-off", vec![text_msg(PROSE)], false).await;
378. 378 assert!(matches!(
379. 379 run(&mut agent).await,
380. 380 TurnOutcome::Completed { .. }
381. 381 ));
382. 382 assert!(
383. 383 !user_texts(&agent)
384. 384 .iter()
385. 385 .any(|t| t.contains("[presentation-check]"))
386. 386 );
387. 387}
388. 388