1. 1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2. 2
3. 3//! Freshness and empty-step enforcement (docs/design/68-context-engine.md
4. 4//! §7). Freshness: a directive
5. 5//! whose reading carries the `live-data` domain asks for a value as it
6. 6//! stands now. If the model answers — in prose or with a card — without any
7. 7//! retrieval-shaped call succeeding in the run, the answer can only repeat
8. 8//! what an earlier turn found, so it gets exactly one `[freshness-check]`
9. 9//! redo. Observed live: six replays of "what is the current weather in new
10. 10//! delhi" on gemma4:e2b-mlx emitted a metric card five times without
11. 11//! searching, carrying a temperature from a previous turn's answer.
12. 12
13. 13use std::collections::VecDeque;
14. 14use std::sync::{Arc, Mutex};
15. 15
16. 16use async_trait::async_trait;
17. 17use serde_json::Value;
18. 18use tokio::sync::mpsc;
19. 19use tokio_util::sync::CancellationToken;
20. 20
21. 21use tempfile::tempdir;
22. 22
23. 23use vak_agent::{Agent, AgentConfig, TurnOutcome};
24. 24use vak_llm::stream;
25. 25use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
26. 26use vak_llm::{EventStream, LlmError, Provider};
27. 27use vak_permission::PermissionEngine;
28. 28use vak_session::types::{FrozenContract, IntentRecord, SessionHeader};
29. 29use vak_session::{SessionLog, SessionPath};
30. 30use vak_tools::context::ToolContext;
31. 31use vak_tools::{Tool, ToolOutput};
32. 32
33. 33struct Scripted {
34. 34 responses: Mutex<VecDeque<AssistantMessage>>,
35. 35}
36. 36
37. 37#[async_trait]
38. 38impl Provider for Scripted {
39. 39 fn name(&self) -> &str {
40. 40 "scripted"
41. 41 }
42. 42
43. 43 async fn stream(
44. 44 &self,
45. 45 _request: ChatRequest,
46. 46 _cancel: CancellationToken,
47. 47 ) -> Result<EventStream, LlmError> {
48. 48 let next = self.responses.lock().unwrap().pop_front();
49. 49 let (mut sink, rx) = stream::channel(16);
50. 50 match next {
51. 51 Some(m) => sink.close_message(m).await,
52. 52 None => sink.close_error(LlmError::Parse("exhausted".into())).await,
53. 53 }
54. 54 Ok(rx)
55. 55 }
56. 56}
57. 57
58. 58struct FakeSearchTool;
59. 59
60. 60#[async_trait]
61. 61impl Tool for FakeSearchTool {
62. 62 fn name(&self) -> &str {
63. 63 "search"
64. 64 }
65. 65 fn description(&self) -> &str {
66. 66 "fake search tool for tests"
67. 67 }
68. 68 fn schema(&self) -> Value {
69. 69 serde_json::json!({"type": "object", "properties": {"query": {"type": "string"}}})
70. 70 }
71. 71 async fn execute(&self, _args: &Value, _ctx: &ToolContext) -> ToolOutput {
72. 72 ToolOutput::ok(
73. 73 "Title: Delhi observations 05:30\nURL: https://example-met.in/delhi\nTemperature: 26.4 C\n",
74. 74 )
75. 75 }
76. 76}
77. 77
78. 78fn search_call(id: &str) -> AssistantMessage {
79. 79 AssistantMessage {
80. 80 content: vec![ContentBlock::ToolUse {
81. 81 id: id.into(),
82. 82 name: "search".into(),
83. 83 input: serde_json::json!({"query": "delhi observations now"}),
84. 84 }],
85. 85 stop_reason: StopReason::ToolUse,
86. 86 usage: Usage::default(),
87. 87 model: "test-model".into(),
88. 88 response_id: None,
89. 89 }
90. 90}
91. 91
92. 92fn text_msg(t: &str) -> AssistantMessage {
93. 93 AssistantMessage {
94. 94 content: vec![ContentBlock::text(t)],
95. 95 stop_reason: StopReason::EndTurn,
96. 96 usage: Usage {
97. 97 input_tokens: 1,
98. 98 output_tokens: 1,
99. 99 ..Default::default()
100. 100 },
101. 101 model: "test-model".into(),
102. 102 response_id: None,
103. 103 }
104. 104}
105. 105
106. 106fn thinking_only() -> AssistantMessage {
107. 107 AssistantMessage {
108. 108 content: vec![ContentBlock::Thinking {
109. 109 text: "Final Plan: 1. Use search to get the current reading. 2. Answer.".into(),
110. 110 signature: None,
111. 111 }],
112. 112 stop_reason: StopReason::EndTurn,
113. 113 usage: Usage::default(),
114. 114 model: "test-model".into(),
115. 115 response_id: None,
116. 116 }
117. 117}
118. 118
119. 119/// Seeds the reading vak-core would have written at turn start for a
120. 120/// directive with temporal deixis.
121. 121fn live_data_intent() -> IntentRecord {
122. 122 let mut reading = vak_intent::Reading::general();
123. 123 reading.domains.insert("live-data".into());
124. 124 IntentRecord {
125. 125 reading,
126. 126 engagement: vak_intent::Engagement::general(),
127. 127 provenance: vak_intent::Provenance::new(vak_intent::Tier::Signals, 1, Vec::new()),
128. 128 outcome: None,
129. 129 model_visible: None,
130. 130 commitment_id: None,
131. 131 strands: Vec::new(),
132. 132 strand_commitments: Default::default(),
133. 133 }
134. 134}
135. 135
136. 136async fn build_agent(
137. 137 dir: &tempfile::TempDir,
138. 138 session_id: &str,
139. 139 responses: Vec<AssistantMessage>,
140. 140 live_data: bool,
141. 141) -> Agent {
142. 142 let header = SessionHeader {
143. 143 agent: None,
144. 144 session_id: session_id.into(),
145. 145 created_at: chrono::Utc::now(),
146. 146 cwd: dir.path().to_path_buf(),
147. 147 parent_session_id: None,
148. 148 contract_id: None,
149. 149 work_item_id: None,
150. 150 conversation: None,
151. 151 contract: FrozenContract {
152. 152 app_version: "0".into(),
153. 153 provider: "scripted".into(),
154. 154 model: "test-model".into(),
155. 155 route_ladder: Vec::new(),
156. 156 route_objective: String::new(),
157. 157 route_annotations: Vec::new(),
158. 158 system_prompt: "sys".into(),
159. 159 permission_mode: "full-access".into(),
160. 160 capabilities: Vec::new(),
161. 161 prompt_layers: Vec::new(),
162. 162 },
163. 163 };
164. 164 let home = dir.path().join("home");
165. 165 std::fs::create_dir_all(&home).unwrap();
166. 166 let mut log = SessionLog::create(
167. 167 SessionPath::new_session_file(&home, dir.path(), session_id),
168. 168 header,
169. 169 )
170. 170 .unwrap();
171. 171 if live_data {
172. 172 log.append_intent(live_data_intent()).unwrap();
173. 173 }
174. 174
175. 175 Agent::new(
176. 176 Arc::new(Scripted {
177. 177 responses: Mutex::new(VecDeque::from(responses)),
178. 178 }),
179. 179 log,
180. 180 {
181. 181 let mut cfg = AgentConfig::new("sys");
182. 182 cfg.model = "test-model".into();
183. 183 cfg.tools = vec![Arc::new(FakeSearchTool)];
184. 184 cfg.retrieval_check = Some(Arc::new(|name: &str, _: &Value| name == "search"));
185. 185 cfg.mode = vak_permission::Mode::FullAccess;
186. 186 cfg.permission = Some(Arc::new(PermissionEngine::default()));
187. 187 cfg
188. 188 },
189. 189 )
190. 190}
191. 191
192. 192fn user_texts(agent: &Agent) -> Vec<String> {
193. 193 futures::executor::block_on(async {
194. 194 agent
195. 195 .session
196. 196 .lock()
197. 197 .await
198. 198 .message_chain()
199. 199 .iter()
200. 200 .filter(|(_, m)| m.role == vak_llm::types::Role::User)
201. 201 .flat_map(|(_, m)| m.content.iter())
202. 202 .filter_map(|b| match b {
203. 203 ContentBlock::Text { text } => Some(text.clone()),
204. 204 _ => None,
205. 205 })
206. 206 .collect()
207. 207 })
208. 208}
209. 209
210. 210#[tokio::test]
211. 211async fn a_live_data_answer_without_retrieval_gets_one_freshness_redo() {
212. 212 let dir = tempdir().unwrap();
213. 213 let mut agent = build_agent(
214. 214 &dir,
215. 215 "freshness-redo",
216. 216 vec![
217. 217 // Answers from memory of an earlier turn — no retrieval.
218. 218 text_msg("It is 29.1°C in New Delhi."),
219. 219 // After the nudge: retrieves, then answers from the result as a
220. 220 // cited card (the grounding check then has nothing to add).
221. 221 search_call("s1"),
222. 222 text_msg(
223. 223 "```vak\n{\"semantic_type\":\"metric\",\"payload\":{\"label\":\"Delhi 05:30\",\"value\":26.4,\"unit\":\"C\",\"source\":\"https://example-met.in/delhi\"}}\n```",
224. 224 ),
225. 225 ],
226. 226 true,
227. 227 )
228. 228 .await;
229. 229
230. 230 let outcome = agent
231. 231 .run(
232. 232 "what is the current weather in new delhi",
233. 233 &Default::default(),
234. 234 CancellationToken::new(),
235. 235 mpsc::channel(64).0,
236. 236 )
237. 237 .await;
238. 238 assert!(
239. 239 matches!(&outcome, TurnOutcome::Completed { response } if response.text_content().contains("26.4")),
240. 240 "got {outcome:?}"
241. 241 );
242. 242
243. 243 let nudges = user_texts(&agent);
244. 244 assert_eq!(
245. 245 nudges
246. 246 .iter()
247. 247 .filter(|t| t.starts_with("[freshness-check]"))
248. 248 .count(),
249. 249 1,
250. 250 "exactly one freshness nudge: {nudges:?}"
251. 251 );
252. 252}
253. 253
254. 254#[tokio::test]
255. 255async fn admitting_no_live_data_is_accepted_without_a_redo() {
256. 256 let dir = tempdir().unwrap();
257. 257 let mut agent = build_agent(
258. 258 &dir,
259. 259 "freshness-admit",
260. 260 vec![text_msg(
261. 261 "I don't have live data for New Delhi right now; the last reading I saw was from an earlier turn.",
262. 262 )],
263. 263 true,
264. 264 )
265. 265 .await;
266. 266 let outcome = agent
267. 267 .run(
268. 268 "what is the current weather in new delhi",
269. 269 &Default::default(),
270. 270 CancellationToken::new(),
271. 271 mpsc::channel(64).0,
272. 272 )
273. 273 .await;
274. 274 assert!(
275. 275 matches!(outcome, TurnOutcome::Completed { .. }),
276. 276 "got {outcome:?}"
277. 277 );
278. 278 assert!(
279. 279 !user_texts(&agent)
280. 280 .iter()
281. 281 .any(|t| t.starts_with("[freshness-check]"))
282. 282 );
283. 283}
284. 284
285. 285#[tokio::test]
286. 286async fn a_timeless_directive_is_never_nudged() {
287. 287 let dir = tempdir().unwrap();
288. 288 let mut agent = build_agent(
289. 289 &dir,
290. 290 "freshness-timeless",
291. 291 vec![text_msg("Copper is refined by electrolysis.")],
292. 292 false,
293. 293 )
294. 294 .await;
295. 295 let outcome = agent
296. 296 .run(
297. 297 "explain how copper is refined",
298. 298 &Default::default(),
299. 299 CancellationToken::new(),
300. 300 mpsc::channel(64).0,
301. 301 )
302. 302 .await;
303. 303 assert!(
304. 304 matches!(outcome, TurnOutcome::Completed { .. }),
305. 305 "got {outcome:?}"
306. 306 );
307. 307 assert!(
308. 308 !user_texts(&agent)
309. 309 .iter()
310. 310 .any(|t| t.starts_with("[freshness-check]"))
311. 311 );
312. 312}
313. 313
314. 314#[tokio::test]
315. 315async fn a_thinking_only_step_gets_one_redo_to_act() {
316. 316 let dir = tempdir().unwrap();
317. 317 let mut agent = build_agent(
318. 318 &dir,
319. 319 "empty-step-redo",
320. 320 vec![
321. 321 thinking_only(),
322. 322 text_msg("Copper is refined by electrolysis."),
323. 323 ],
324. 324 false,
325. 325 )
326. 326 .await;
327. 327 let outcome = agent
328. 328 .run(
329. 329 "explain how copper is refined",
330. 330 &Default::default(),
331. 331 CancellationToken::new(),
332. 332 mpsc::channel(64).0,
333. 333 )
334. 334 .await;
335. 335 assert!(
336. 336 matches!(&outcome, TurnOutcome::Completed { response } if response.text_content().contains("electrolysis")),
337. 337 "got {outcome:?}"
338. 338 );
339. 339 let nudges = user_texts(&agent);
340. 340 assert_eq!(
341. 341 nudges
342. 342 .iter()
343. 343 .filter(|t| t.starts_with("[empty-step]"))
344. 344 .count(),
345. 345 1,
346. 346 "exactly one empty-step nudge: {nudges:?}"
347. 347 );
348. 348 assert!(
349. 349 nudges.iter().any(|text| {
350. 350 text.starts_with("[empty-step]") && text.contains("explain how copper is refined")
351. 351 }),
352. 352 "empty-step repair must retain the already-admitted target: {nudges:?}"
353. 353 );
354. 354}
355. 355
356. 356#[tokio::test]
357. 357async fn a_tool_call_starts_a_fresh_bounded_empty_step_boundary() {
358. 358 let dir = tempdir().unwrap();
359. 359 let mut agent = build_agent(
360. 360 &dir,
361. 361 "empty-step-after-tool",
362. 362 vec![
363. 363 thinking_only(),
364. 364 search_call("s1"),
365. 365 thinking_only(),
366. 366 text_msg("The retrieved observation reports 26.4 C in Delhi."),
367. 367 ],
368. 368 false,
369. 369 )
370. 370 .await;
371. 371 let outcome = agent
372. 372 .run(
373. 373 "find the Delhi observation and answer",
374. 374 &Default::default(),
375. 375 CancellationToken::new(),
376. 376 mpsc::channel(64).0,
377. 377 )
378. 378 .await;
379. 379 assert!(
380. 380 matches!(&outcome, TurnOutcome::Completed { response } if response.text_content().contains("26.4")),
381. 381 "got {outcome:?}"
382. 382 );
383. 383 assert_eq!(
384. 384 user_texts(&agent)
385. 385 .iter()
386. 386 .filter(|text| text.starts_with("[empty-step]"))
387. 387 .count(),
388. 388 2,
389. 389 "each action boundary gets one bounded repair"
390. 390 );
391. 391}
392. 392
393. 393#[tokio::test]
394. 394async fn two_thinking_only_steps_fail_instead_of_completing_without_work() {
395. 395 let dir = tempdir().unwrap();
396. 396 let mut agent = build_agent(
397. 397 &dir,
398. 398 "empty-step-failure",
399. 399 vec![thinking_only(), thinking_only()],
400. 400 false,
401. 401 )
402. 402 .await;
403. 403 let outcome = agent
404. 404 .run(
405. 405 "create a draft",
406. 406 &Default::default(),
407. 407 CancellationToken::new(),
408. 408 mpsc::channel(64).0,
409. 409 )
410. 410 .await;
411. 411 assert!(
412. 412 matches!(&outcome, TurnOutcome::Failed { error: vak_llm::LlmError::Parse(message) }
413. 413 if message.contains("no visible answer")),
414. 414 "got {outcome:?}"
415. 415 );
416. 416 assert_eq!(
417. 417 user_texts(&agent)
418. 418 .iter()
419. 419 .filter(|text| text.starts_with("[empty-step]"))
420. 420 .count(),
421. 421 1
422. 422 );
423. 423}
424. 424
425. 425fn card_call(id: &str, value: &str) -> AssistantMessage {
426. 426 AssistantMessage {
427. 427 content: vec![ContentBlock::ToolUse {
428. 428 id: id.into(),
429. 429 name: "emit_metric_card".into(),
430. 430 input: serde_json::json!({"semantic_type":"metric","payload":{"label":"Delhi","value":value,"unit":"C"}}),
431. 431 }],
432. 432 stop_reason: StopReason::ToolUse,
433. 433 usage: Usage::default(),
434. 434 model: "test-model".into(),
435. 435 response_id: None,
436. 436 }
437. 437}
438. 438
439. 439struct FakeCardTool;
440. 440
441. 441#[async_trait]
442. 442impl Tool for FakeCardTool {
443. 443 fn name(&self) -> &str {
444. 444 "emit_metric_card"
445. 445 }
446. 446 fn description(&self) -> &str {
447. 447 "fake card tool for tests"
448. 448 }
449. 449 fn schema(&self) -> Value {
450. 450 serde_json::json!({"type": "object"})
451. 451 }
452. 452 fn presents_cards(&self) -> bool {
453. 453 true
454. 454 }
455. 455 async fn execute(&self, _args: &Value, _ctx: &ToolContext) -> ToolOutput {
456. 456 ToolOutput::ok("{\"ok\":true}")
457. 457 }
458. 458}
459. 459
460. 460#[tokio::test]
461. 461async fn a_card_in_a_live_data_turn_is_gated_until_something_is_retrieved() {
462. 462 let dir = tempdir().unwrap();
463. 463 let mut agent = build_agent(
464. 464 &dir,
465. 465 "freshness-card-gate",
466. 466 vec![
467. 467 // Card straight from memory: not executed, error value back.
468. 468 card_call("c1", "29.1"),
469. 469 // Repairs: retrieves, then the card from the result is shown.
470. 470 search_call("s1"),
471. 471 card_call("c2", "26.4"),
472. 472 text_msg("Delhi is at 26.4 C (example-met.in, 05:30)."),
473. 473 ],
474. 474 true,
475. 475 )
476. 476 .await;
477. 477 agent.config.tools.push(Arc::new(FakeCardTool));
478. 478 let outcome = agent
479. 479 .run(
480. 480 "what is the current weather in new delhi",
481. 481 &Default::default(),
482. 482 CancellationToken::new(),
483. 483 mpsc::channel(64).0,
484. 484 )
485. 485 .await;
486. 486 assert!(
487. 487 matches!(&outcome, TurnOutcome::Completed { .. }),
488. 488 "got {outcome:?}"
489. 489 );
490. 490
491. 491 let results: Vec<(String, bool)> = futures::executor::block_on(async {
492. 492 agent
493. 493 .session
494. 494 .lock()
495. 495 .await
496. 496 .message_chain()
497. 497 .iter()
498. 498 .flat_map(|(_, m)| m.content.clone())
499. 499 .filter_map(|b| match b {
500. 500 ContentBlock::ToolResult {
501. 501 content, is_error, ..
502. 502 } => Some((content, is_error)),
503. 503 _ => None,
504. 504 })
505. 505 .collect()
506. 506 });
507. 507 assert!(
508. 508 results
509. 509 .iter()
510. 510 .any(|(c, e)| *e && c.starts_with("[freshness-check]")),
511. 511 "the first card must come back as a freshness error: {results:?}"
512. 512 );
513. 513 assert!(
514. 514 !user_texts(&agent)
515. 515 .iter()
516. 516 .any(|t| t.starts_with("[freshness-check]")),
517. 517 "the gate fired at the card, so no second nudge on the final text"
518. 518 );
519. 519}
520. 520
521. 521#[tokio::test]
522. 522async fn a_second_stale_card_fails_closed_with_an_honest_answer() {
523. 523 let dir = tempdir().unwrap();
524. 524 let mut agent = build_agent(
525. 525 &dir,
526. 526 "freshness-fail-closed",
527. 527 vec![
528. 528 card_call("c1", "29.1"),
529. 529 // The "repair" is another carried-over figure, still no retrieval.
530. 530 card_call("c2", "28"),
531. 531 text_msg("unreachable"),
532. 532 ],
533. 533 true,
534. 534 )
535. 535 .await;
536. 536 agent.config.tools.push(Arc::new(FakeCardTool));
537. 537 let outcome = agent
538. 538 .run(
539. 539 "what is the current weather in new delhi",
540. 540 &Default::default(),
541. 541 CancellationToken::new(),
542. 542 mpsc::channel(64).0,
543. 543 )
544. 544 .await;
545. 545 match &outcome {
546. 546 TurnOutcome::Completed { response } => {
547. 547 let text = response.text_content();
548. 548 assert!(
549. 549 text.contains("not presenting a carried-over figure as current"),
550. 550 "fail-closed answer expected, got: {text}"
551. 551 );
552. 552 assert!(!text.contains("unreachable"));
553. 553 }
554. 554 other => panic!("expected a completed turn, got {other:?}"),
555. 555 }
556. 556 let presented =
557. 557 futures::executor::block_on(async { agent.session.lock().await.presentations().len() });
558. 558 assert_eq!(
559. 559 presented, 0,
560. 560 "no stale card may reach the ledger as a presentation"
561. 561 );
562. 562 // The system-authored answer is on the ledger, so the turn is closed
563. 563 // and carded like any other.
564. 564 let (last_is_answer, carded) = futures::executor::block_on(async {
565. 565 let session = agent.session.lock().await;
566. 566 let last = session
567. 567 .message_chain()
568. 568 .last()
569. 569 .map(|(_, m)| m.text_content().contains("carried-over figure"))
570. 570 .unwrap_or(false);
571. 571 (last, session.turn_cards().len())
572. 572 });
573. 573 assert!(last_is_answer, "the fail-closed answer must be logged");
574. 574 assert_eq!(carded, 1, "the turn closes with a TurnCard");
575. 575}
576. 576
577. 577struct FakeListTool;
578. 578
579. 579#[async_trait]
580. 580impl Tool for FakeListTool {
581. 581 fn name(&self) -> &str {
582. 582 "glob"
583. 583 }
584. 584 fn description(&self) -> &str {
585. 585 "fake directory listing for tests"
586. 586 }
587. 587 fn schema(&self) -> Value {
588. 588 serde_json::json!({"type": "object", "properties": {"pattern": {"type": "string"}}})
589. 589 }
590. 590 async fn execute(&self, _args: &Value, _ctx: &ToolContext) -> ToolOutput {
591. 591 ToolOutput::ok("Cargo.toml\nsrc/main.rs\n")
592. 592 }
593. 593}
594. 594
595. 595/// A current value in the workspace is observed by looking at the workspace:
596. 596/// "what's in the current directory?" is answered by listing it. A local
597. 597/// observation satisfies the freshness check exactly as a retrieval does,
598. 598/// with no demand for a web search that could not answer it.
599. 599#[tokio::test]
600. 600async fn a_local_observation_satisfies_the_freshness_check() {
601. 601 let dir = tempdir().unwrap();
602. 602 let list_call = AssistantMessage {
603. 603 content: vec![ContentBlock::ToolUse {
604. 604 id: "g1".into(),
605. 605 name: "glob".into(),
606. 606 input: serde_json::json!({"pattern": "*"}),
607. 607 }],
608. 608 stop_reason: StopReason::ToolUse,
609. 609 usage: Usage::default(),
610. 610 model: "test-model".into(),
611. 611 response_id: None,
612. 612 };
613. 613 let mut agent = build_agent(
614. 614 &dir,
615. 615 "freshness-local",
616. 616 vec![
617. 617 list_call,
618. 618 text_msg("The current directory holds Cargo.toml and src/main.rs."),
619. 619 ],
620. 620 true,
621. 621 )
622. 622 .await;
623. 623 agent.config.tools.push(Arc::new(FakeListTool));
624. 624 agent.config.observation_check = Some(Arc::new(|name: &str, _: &Value| name == "glob"));
625. 625 let outcome = agent
626. 626 .run(
627. 627 "what's in the current directory?",
628. 628 &Default::default(),
629. 629 CancellationToken::new(),
630. 630 mpsc::channel(64).0,
631. 631 )
632. 632 .await;
633. 633 assert!(
634. 634 matches!(&outcome, TurnOutcome::Completed { response } if response.text_content().contains("Cargo.toml")),
635. 635 "got {outcome:?}"
636. 636 );
637. 637 assert!(
638. 638 !user_texts(&agent)
639. 639 .iter()
640. 640 .any(|t| t.starts_with("[freshness-check]")),
641. 641 "a listing is an observation; no redo was owed"
642. 642 );
643. 643}
644. 644