1. 1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2. 2
3. 3//! General-purpose (non-coding) scenario coverage. The same harness
4. 4//! machinery — steering, abort, stop gate, compaction, MCP, permissions —
5. 5//! exercised through research, writing, planning, and data-analysis flows
6. 6//! instead of code tasks.
7. 7
8. 8use std::collections::{HashMap, VecDeque};
9. 9use std::path::Path;
10. 10use std::sync::{Arc, Mutex};
11. 11use std::time::Duration;
12. 12
13. 13use tokio::sync::mpsc;
14. 14use tokio_util::sync::CancellationToken;
15. 15
16. 16use tempfile::tempdir;
17. 17
18. 18use vak_agent::{Agent, AgentConfig, AgentEvent, SteeringQueues, TurnOutcome};
19. 19use vak_llm::stream;
20. 20use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
21. 21use vak_llm::{EventStream, LlmError, Provider};
22. 22use vak_mcp::{McpManager, McpTool, ServerConfig};
23. 23use vak_permission::{Mode, PermissionEngine};
24. 24use vak_session::types::{FrozenContract, SessionHeader};
25. 25use vak_session::{SessionLog, SessionPath};
26. 26use vak_tools::bash::BashTool;
27. 27use vak_tools::read::ReadTool;
28. 28use vak_tools::write::WriteTool;
29. 29
30. 30struct Scripted {
31. 31 responses: std::sync::Mutex<VecDeque<AssistantMessage>>,
32. 32 requests: std::sync::Mutex<Vec<ChatRequest>>,
33. 33}
34. 34
35. 35#[async_trait::async_trait]
36. 36impl Provider for Scripted {
37. 37 fn name(&self) -> &str {
38. 38 "scripted-general"
39. 39 }
40. 40
41. 41 async fn stream(
42. 42 &self,
43. 43 request: ChatRequest,
44. 44 _cancel: CancellationToken,
45. 45 ) -> Result<EventStream, LlmError> {
46. 46 self.requests.lock().unwrap().push(request);
47. 47 let next = self.responses.lock().unwrap().pop_front();
48. 48 let (mut sink, rx) = stream::channel(64);
49. 49 match next {
50. 50 Some(m) => {
51. 51 sink.push(stream::StreamEvent::Start { partial: m.clone() });
52. 52 sink.close_message(m).await;
53. 53 }
54. 54 None => {
55. 55 sink.close_error(LlmError::Parse("script exhausted".into()))
56. 56 .await
57. 57 }
58. 58 }
59. 59 Ok(rx)
60. 60 }
61. 61}
62. 62
63. 63fn text_msg(t: &str) -> AssistantMessage {
64. 64 AssistantMessage {
65. 65 content: vec![ContentBlock::text(t)],
66. 66 stop_reason: StopReason::EndTurn,
67. 67 usage: Usage {
68. 68 input_tokens: 1,
69. 69 output_tokens: 1,
70. 70 ..Default::default()
71. 71 },
72. 72 model: "test-model".into(),
73. 73 response_id: None,
74. 74 }
75. 75}
76. 76
77. 77fn tool_call(id: &str, name: &str, input: serde_json::Value) -> AssistantMessage {
78. 78 AssistantMessage {
79. 79 content: vec![ContentBlock::ToolUse {
80. 80 id: id.into(),
81. 81 name: name.into(),
82. 82 input,
83. 83 }],
84. 84 stop_reason: StopReason::ToolUse,
85. 85 usage: Usage::default(),
86. 86 model: "test-model".into(),
87. 87 response_id: None,
88. 88 }
89. 89}
90. 90
91. 91fn setup(
92. 92 responses: Vec<AssistantMessage>,
93. 93 tools: Vec<Arc<dyn vak_tools::Tool>>,
94. 94 customize: impl FnOnce(&mut AgentConfig),
95. 95) -> (Agent, Arc<Scripted>, tempfile::TempDir) {
96. 96 let dir = tempdir().unwrap();
97. 97 let cwd = dir.path().to_path_buf();
98. 98 let header = SessionHeader {
99. 99 agent: None,
100. 100 session_id: "general-flow".into(),
101. 101 created_at: chrono::Utc::now(),
102. 102 cwd: cwd.clone(),
103. 103 parent_session_id: None,
104. 104 contract_id: None,
105. 105 work_item_id: None,
106. 106 conversation: None,
107. 107 contract: FrozenContract {
108. 108 app_version: "0".into(),
109. 109 provider: "scripted".into(),
110. 110 model: "test-model".into(),
111. 111 route_ladder: Vec::new(),
112. 112 route_objective: String::new(),
113. 113 route_annotations: Vec::new(),
114. 114 system_prompt: "sys".into(),
115. 115 permission_mode: "full-access".into(),
116. 116 capabilities: Vec::new(),
117. 117 prompt_layers: Vec::new(),
118. 118 },
119. 119 };
120. 120 let home = cwd.join(".vak-home");
121. 121 std::fs::create_dir_all(&home).unwrap();
122. 122 let log = SessionLog::create(
123. 123 SessionPath::new_session_file(&home, &cwd, "general-flow"),
124. 124 header,
125. 125 )
126. 126 .unwrap();
127. 127 let provider = Arc::new(Scripted {
128. 128 responses: std::sync::Mutex::new(responses.into_iter().collect()),
129. 129 requests: std::sync::Mutex::new(Vec::new()),
130. 130 });
131. 131 let mut cfg = AgentConfig::new("sys");
132. 132 cfg.model = "test-model".into();
133. 133 cfg.tools = tools;
134. 134 cfg.mode = Mode::FullAccess;
135. 135 cfg.permission = Some(Arc::new(PermissionEngine::default()));
136. 136 cfg.approver = Some(Arc::new(vak_agent::AutoApprove));
137. 137 customize(&mut cfg);
138. 138 let agent = Agent::new(provider.clone(), log, cfg);
139. 139 (agent, provider, dir)
140. 140}
141. 141
142. 142fn spawn_collector(mut rx: mpsc::Receiver<AgentEvent>) -> tokio::task::JoinHandle<Vec<AgentEvent>> {
143. 143 tokio::spawn(async move {
144. 144 let mut out = Vec::new();
145. 145 while let Some(ev) = rx.recv().await {
146. 146 out.push(ev);
147. 147 }
148. 148 out
149. 149 })
150. 150}
151. 151
152. 152/// The user steers mid-run while the first step is still executing; the
153. 153/// steer must enter the ledger before the next model call and shape the
154. 154/// final artifact.
155. 155#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
156. 156async fn steering_redirects_trip_plan_mid_run() {
157. 157 let (mut agent, _provider, dir) = setup(
158. 158 vec![
159. 159 tool_call("t1", "bash", serde_json::json!({"command": "sleep 0.6"})),
160. 160 tool_call(
161. 161 "t2",
162. 162 "write",
163. 163 serde_json::json!({
164. 164 "path": "trip-plan.md",
165. 165 "content": "# Trip Plan\n\nTotal budget stays under $1500, with two full days in Kyoto.\n"
166. 166 }),
167. 167 ),
168. 168 text_msg("plan finalized"),
169. 169 ],
170. 170 vec![Arc::new(BashTool), Arc::new(WriteTool)],
171. 171 |_| {},
172. 172 );
173. 173
174. 174 let steering = Arc::new(SteeringQueues::new());
175. 175 let pusher = {
176. 176 let steering = steering.clone();
177. 177 tokio::spawn(async move {
178. 178 tokio::time::sleep(Duration::from_millis(150)).await;
179. 179 steering.push_steering("Budget is under $1500 total, and include two days in Kyoto.");
180. 180 })
181. 181 };
182. 182
183. 183 let (ev_tx, ev_rx) = mpsc::channel(256);
184. 184 drop(spawn_collector(ev_rx));
185. 185
186. 186 let outcome = agent
187. 187 .run(
188. 188 "Plan our Japan trip and save it to trip-plan.md.",
189. 189 &steering,
190. 190 CancellationToken::new(),
191. 191 ev_tx,
192. 192 )
193. 193 .await;
194. 194 pusher.await.unwrap();
195. 195
196. 196 assert!(
197. 197 matches!(outcome, TurnOutcome::Completed { .. }),
198. 198 "got {outcome:?}"
199. 199 );
200. 200
201. 201 // Model-visible input must be reconstructable from the ledger.
202. 202 let steered = agent
203. 203 .session
204. 204 .lock()
205. 205 .await
206. 206 .derive_messages()
207. 207 .iter()
208. 208 .any(|m| m.text_content().contains("under $1500"));
209. 209 assert!(steered, "steering message must be logged in the session");
210. 210
211. 211 let draft = std::fs::read_to_string(dir.path().join("trip-plan.md")).unwrap();
212. 212 assert!(
213. 213 draft.contains("$1500") && draft.to_lowercase().contains("kyoto"),
214. 214 "final artifact must reflect the steering"
215. 215 );
216. 216}
217. 217
218. 218/// Cancelling mid-run keeps everything the agent already wrote to disk.
219. 219#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
220. 220async fn abort_preserves_partial_research_notes() {
221. 221 let cancel = CancellationToken::new();
222. 222 let (mut agent, _provider, dir) = setup(
223. 223 vec![
224. 224 tool_call(
225. 225 "t1",
226. 226 "write",
227. 227 serde_json::json!({
228. 228 "path": "notes-draft.md",
229. 229 "content": "# Research Notes\n\n- source A reviewed\n"
230. 230 }),
231. 231 ),
232. 232 tool_call("t2", "bash", serde_json::json!({"command": "sleep 30"})),
233. 233 text_msg("never reached"),
234. 234 ],
235. 235 vec![Arc::new(WriteTool), Arc::new(BashTool)],
236. 236 |_| {},
237. 237 );
238. 238
239. 239 let watcher_cancel = cancel.clone();
240. 240 let watch_path = dir.path().join("notes-draft.md");
241. 241 let watcher = tokio::spawn(async move {
242. 242 loop {
243. 243 if watch_path.exists() {
244. 244 tokio::time::sleep(Duration::from_millis(50)).await;
245. 245 watcher_cancel.cancel();
246. 246 break;
247. 247 }
248. 248 tokio::time::sleep(Duration::from_millis(10)).await;
249. 249 }
250. 250 });
251. 251
252. 252 let outcome = agent
253. 253 .run(
254. 254 "Compile research notes into notes-draft.md.",
255. 255 &Default::default(),
256. 256 cancel,
257. 257 mpsc::channel(64).0,
258. 258 )
259. 259 .await;
260. 260 watcher.abort();
261. 261
262. 262 assert!(
263. 263 matches!(outcome, TurnOutcome::Aborted { .. }),
264. 264 "expected abort, got {outcome:?}"
265. 265 );
266. 266 let partial =
267. 267 std::fs::read_to_string(dir.path().join("notes-draft.md")).expect("partial file survives");
268. 268 assert!(partial.contains("source A reviewed"));
269. 269}
270. 270
271. 271/// A report task that demands verification cannot finish with zero executed
272. 272/// commands; the built-in stop gate forces one continuation, then the run
273. 273/// completes only after real verification ran.
274. 274#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
275. 275async fn stop_gate_blocks_premature_report_until_verified() {
276. 276 let (mut agent, _provider, dir) = setup(
277. 277 vec![
278. 278 text_msg("Done. The report is ready."),
279. 279 tool_call(
280. 280 "t1",
281. 281 "bash",
282. 282 serde_json::json!({
283. 283 "command": "printf 'Q1 total: 4200\\n' > report.md && grep -q 4200 report.md"
284. 284 }),
285. 285 ),
286. 286 text_msg("Verified. Q1 total recorded as 4200 in report.md."),
287. 287 ],
288. 288 vec![Arc::new(BashTool)],
289. 289 |_| {},
290. 290 );
291. 291
292. 292 let (ev_tx, ev_rx) = mpsc::channel(256);
293. 293 let collector = spawn_collector(ev_rx);
294. 294
295. 295 let outcome = agent
296. 296 .run(
297. 297 "Compute the totals into report.md and verify the number appears in the file before you finish.",
298. 298 &Default::default(),
299. 299 CancellationToken::new(),
300. 300 ev_tx,
301. 301 )
302. 302 .await;
303. 303 let events = collector.await.unwrap();
304. 304
305. 305 match outcome {
306. 306 TurnOutcome::Completed { response } => {
307. 307 assert!(response.text_content().contains("Verified"));
308. 308 }
309. 309 other => panic!("expected completed after gate continuation, got {other:?}"),
310. 310 }
311. 311
312. 312 let continuations = events
313. 313 .iter()
314. 314 .filter(|e| matches!(e, AgentEvent::StopHookContinuation { .. }))
315. 315 .count();
316. 316 assert_eq!(continuations, 1, "gate must block exactly once");
317. 317
318. 318 // Raw ledger: a control nudge is scaffolding for the turn still in
319. 319 // progress and is dropped once the turn closes
320. 320 // (docs/design/68-context-engine.md §10) — this checks it was recorded
321. 321 // at all, not that the final projection still carries it.
322. 322 let guard_msgs = agent
323. 323 .session
324. 324 .lock()
325. 325 .await
326. 326 .message_chain()
327. 327 .iter()
328. 328 .filter(|(_, m)| m.text_content().contains("[stop-guard]"))
329. 329 .count();
330. 330 assert_eq!(guard_msgs, 1, "guard continuation must be logged once");
331. 331
332. 332 // Agent bash works in the workspace, where the file tools read, so the
333. 333 // verified report is where the user and the next tool call look for it.
334. 334 let report = std::fs::read_to_string(dir.path().join("report.md"))
335. 335 .expect("verified report is in the workspace");
336. 336 assert!(report.contains("4200"));
337. 337}
338. 338
339. 339/// A long research session crosses the context trigger; compaction
340. 340/// summarizes older turns into a ledger entry (never deleting them), the
341. 341/// projection carries the summary forward, and the run completes.
342. 342#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
343. 343async fn compaction_during_long_research_session() {
344. 344 // Compaction works in whole closed turns
345. 345 // (docs/design/68-context-engine.md §10: a turn is never split), so
346. 346 // there has to be at least one closed turn for it to summarize away —
347. 347 // the still-open research turn below is never itself a compaction
348. 348 // candidate (§10: "current turn: every result verbatim, always"), so
349. 349 // the source files stay modest here; what pushes the budget over is
350. 350 // the seeded prior research, which compaction then drops.
351. 351 let big_a = "Research note on climate data points.\n".repeat(6);
352. 352 let big_b = "Research note on energy market shifts.\n".repeat(6);
353. 353
354. 354 let dir = tempdir().unwrap();
355. 355 let cwd = dir.path().to_path_buf();
356. 356 let header = SessionHeader {
357. 357 agent: None,
358. 358 session_id: "general-flow".into(),
359. 359 created_at: chrono::Utc::now(),
360. 360 cwd: cwd.clone(),
361. 361 parent_session_id: None,
362. 362 contract_id: None,
363. 363 work_item_id: None,
364. 364 conversation: None,
365. 365 contract: FrozenContract {
366. 366 app_version: "0".into(),
367. 367 provider: "scripted".into(),
368. 368 model: "test-model".into(),
369. 369 route_ladder: Vec::new(),
370. 370 route_objective: String::new(),
371. 371 route_annotations: Vec::new(),
372. 372 system_prompt: "sys".into(),
373. 373 permission_mode: "full-access".into(),
374. 374 capabilities: Vec::new(),
375. 375 prompt_layers: Vec::new(),
376. 376 },
377. 377 };
378. 378 let home = cwd.join(".vak-home");
379. 379 std::fs::create_dir_all(&home).unwrap();
380. 380 let mut log = SessionLog::create(
381. 381 SessionPath::new_session_file(&home, &cwd, "general-flow"),
382. 382 header,
383. 383 )
384. 384 .unwrap();
385. 385 // Big enough on their own to already be over the trigger threshold
386. 386 // before the real research turn starts, so incremental compaction
387. 387 // (docs/design/68-context-engine.md §4) fires on the very first plan —
388. 388 // consuming the FIRST scripted response below (the compaction summary)
389. 389 // rather than one meant for the real turn. Each seeded turn gets a
390. 390 // real `TurnCard` (as the turn-close hook would write): the planner
391. 391 // only ever collapses carded turns into a packet.
392. 392 let seed_filler = "prior research finding ".repeat(220);
393. 393 for i in 0..3 {
394. 394 let turn_id = log
395. 395 .append_message(vak_session::types::MessageRecord {
396. 396 message: vak_llm::types::Message::user_text(format!("earlier note {i}")),
397. 397 meta: None,
398. 398 })
399. 399 .unwrap()
400. 400 .id;
401. 401 let answer = format!("acknowledged note {i}: {seed_filler}");
402. 402 log.append_message(vak_session::types::MessageRecord {
403. 403 message: vak_llm::types::Message::assistant(vec![ContentBlock::text(&answer)]),
404. 404 meta: None,
405. 405 })
406. 406 .unwrap();
407. 407 let card = vak_session::TurnIndex::from_log(&log)
408. 408 .turn_by_id(&turn_id)
409. 409 .unwrap()
410. 410 .build_card("completed", answer, &|s| s.len() as u64 / 4);
411. 411 log.append_turn_card(vak_session::types::TurnCardRecord { turn_id, card })
412. 412 .unwrap();
413. 413 }
414. 414 let provider = Arc::new(Scripted {
415. 415 responses: std::sync::Mutex::new(VecDeque::from(vec![
416. 416 text_msg(
417. 417 "Summary: prior research condensed; key climate and energy findings retained for the brief.",
418. 418 ),
419. 419 tool_call("r1", "read", serde_json::json!({"path": "big-a.txt"})),
420. 420 tool_call("r2", "read", serde_json::json!({"path": "big-b.txt"})),
421. 421 tool_call(
422. 422 "w1",
423. 423 "write",
424. 424 serde_json::json!({
425. 425 "path": "digest.txt",
426. 426 "content": "Digest: sources A and B agree on the retained findings.\n"
427. 427 }),
428. 428 ),
429. 429 text_msg("digest written"),
430. 430 ])),
431. 431 requests: std::sync::Mutex::new(Vec::new()),
432. 432 });
433. 433 let mut cfg = AgentConfig::new("sys");
434. 434 cfg.model = "test-model".into();
435. 435 cfg.tools = vec![Arc::new(ReadTool), Arc::new(WriteTool)];
436. 436 cfg.mode = Mode::FullAccess;
437. 437 cfg.permission = Some(Arc::new(PermissionEngine::default()));
438. 438 cfg.approver = Some(Arc::new(vak_agent::AutoApprove));
439. 439 // Small enough that the three heavily-padded seeded turns above cannot
440. 440 // all fit even as cards, forcing a packet on the very first plan.
441. 441 cfg.declared_window = 1500;
442. 442 cfg.max_output = 100;
443. 443 // The handoff-reset rescue is a different mechanism (Phase H) from
444. 444 // incremental compaction and would consume its own scripted response
445. 445 // if it fired; keep this test isolated to compaction alone.
446. 446 cfg.handoff_reset = false;
447. 447 let mut agent = Agent::new(provider.clone(), log, cfg);
448. 448
449. 449 for (name, content) in [("big-a.txt", &big_a), ("big-b.txt", &big_b)] {
450. 450 std::fs::write(dir.path().join(name), content).unwrap();
451. 451 }
452. 452
453. 453 let (ev_tx, ev_rx) = mpsc::channel(4096);
454. 454 let collector = spawn_collector(ev_rx);
455. 455
456. 456 let outcome = agent
457. 457 .run(
458. 458 "Read both source files and write digest.txt summarizing their key facts.",
459. 459 &Default::default(),
460. 460 CancellationToken::new(),
461. 461 ev_tx,
462. 462 )
463. 463 .await;
464. 464 let events = collector.await.unwrap();
465. 465
466. 466 match outcome {
467. 467 TurnOutcome::Completed { response } => {
468. 468 assert_eq!(response.text_content(), "digest written");
469. 469 }
470. 470 other => panic!("expected completed after compaction, got {other:?}"),
471. 471 }
472. 472 let continuations = events
473. 473 .iter()
474. 474 .filter(|e| matches!(e, AgentEvent::ContextCompacting { .. }))
475. 475 .count();
476. 476 assert_eq!(continuations, 1, "compaction must trigger exactly once");
477. 477 let compacted = events.iter().find_map(|e| match e {
478. 478 AgentEvent::ContextCompacted {
479. 479 before_tokens,
480. 480 after_tokens,
481. 481 ..
482. 482 } => Some((*before_tokens, *after_tokens)),
483. 483 _ => None,
484. 484 });
485. 485 let (before, after) = compacted.expect("compacted event");
486. 486 assert!(after < before, "compaction must shrink the estimate");
487. 487
488. 488 // The summarizer ran as its own model call against the transcript
489. 489 // BEFORE the real research turn dispatched at all (the seeded prior
490. 490 // research alone was already over budget), and the whole run consumed
491. 491 // exactly the scripted trajectory: compaction + two reads + write +
492. 492 // final.
493. 493 {
494. 494 let reqs = provider.requests.lock().unwrap();
495. 495 assert_eq!(reqs.len(), 5, "compaction + two reads + write + final");
496. 496 assert!(
497. 497 reqs[0]
498. 498 .system
499. 499 .as_deref()
500. 500 .unwrap_or("")
501. 501 .contains("compactor"),
502. 502 "first request must be the compaction call"
503. 503 );
504. 504 }
505. 505
506. 506 // Append-only invariant: raw ledger still holds pre-compaction history
507. 507 // plus exactly one compaction entry; projection carries the summary.
508. 508 let session_file =
509. 509 SessionPath::new_session_file(&dir.path().join(".vak-home"), dir.path(), "general-flow");
510. 510 let raw = std::fs::read_to_string(&session_file).unwrap();
511. 511 let compaction_lines = raw
512. 512 .lines()
513. 513 .filter(|l| l.contains("\"kind\":\"compaction\""))
514. 514 .count();
515. 515 assert_eq!(compaction_lines, 1, "exactly one compaction entry expected");
516. 516 assert!(
517. 517 raw.contains("prior research finding"),
518. 518 "original history must never be deleted from the ledger"
519. 519 );
520. 520
521. 521 // The packet reached the model: every request after the compaction
522. 522 // call led with the summary. The plan-free projection, by contrast,
523. 523 // still carries the raw history — a packet is a cache for the plan
524. 524 // that asked for it, never a boundary in the ledger.
525. 525 {
526. 526 let reqs = provider.requests.lock().unwrap();
527. 527 for req in &reqs[1..] {
528. 528 assert!(
529. 529 req.messages[0]
530. 530 .text_content()
531. 531 .starts_with("<context_summary>"),
532. 532 "every request after compaction must lead with the packet"
533. 533 );
534. 534 }
535. 535 }
536. 536 let projected = agent.session.lock().await.derive_messages();
537. 537 assert!(
538. 538 projected
539. 539 .iter()
540. 540 .any(|m| m.text_content().contains("prior research finding")),
541. 541 "the packet must not hide history from the plan-free projection"
542. 542 );
543. 543
544. 544 let digest = std::fs::read_to_string(dir.path().join("digest.txt")).unwrap();
545. 545 assert!(digest.contains("Digest:"));
546. 546}
547. 547
548. 548/// A non-code lookup through the MCP meta-tool (list → call → artifact).
549. 549#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
550. 550async fn mcp_notes_lookup_informs_planning_answer() {
551. 551 let Ok(probe) = std::process::Command::new("python3")
552. 552 .arg("--version")
553. 553 .output()
554. 554 else {
555. 555 eprintln!("skipping: python3 unavailable");
556. 556 return;
557. 557 };
558. 558 assert!(probe.status.success(), "python3 must be runnable");
559. 559
560. 560 let script = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../scripts/fake_mcp_server.py");
561. 561 let mut servers = HashMap::new();
562. 562 servers.insert(
563. 563 "notes".to_string(),
564. 564 ServerConfig {
565. 565 command: "python3".into(),
566. 566 args: vec![script.display().to_string()],
567. 567 env: Vec::new(),
568. 568 network: false,
569. 569 },
570. 570 );
571. 571 let manager = Arc::new(McpManager::new(servers, dir_safe_cwd()));
572. 572 let aliases = Arc::new(Mutex::new(HashMap::new()));
573. 573 let aliases_after_list = aliases.clone();
574. 574 let mcp_tool: Arc<dyn vak_tools::Tool> = Arc::new(McpTool::new(manager).with_catalog_observer(
575. 575 Arc::new(move |catalog| {
576. 576 let mut registered = aliases_after_list.lock().unwrap();
577. 577 for (server, tools) in catalog {
578. 578 for tool in tools {
579. 579 registered.insert(tool.name.clone(), server.clone());
580. 580 }
581. 581 }
582. 582 }),
583. 583 ));
584. 584
585. 585 let (mut agent, _provider, dir) = setup(
586. 586 vec![
587. 587 tool_call("m1", "mcp", serde_json::json!({"action": "list"})),
588. 588 tool_call("m2", "echo", serde_json::json!({"text": "flights booked"})),
589. 589 tool_call(
590. 590 "w1",
591. 591 "write",
592. 592 serde_json::json!({
593. 593 "path": "answer.md",
594. 594 "content": "# Status\n\nMCP says: echo: flights booked\n"
595. 595 }),
596. 596 ),
597. 597 text_msg("answered via mcp"),
598. 598 ],
599. 599 vec![mcp_tool, Arc::new(WriteTool)],
600. 600 |config| {
601. 601 config.mcp_tool_index = aliases.clone();
602. 602 },
603. 603 );
604. 604
605. 605 let outcome = agent
606. 606 .run(
607. 607 "Ask the notes service about our booking and record the reply in answer.md.",
608. 608 &Default::default(),
609. 609 CancellationToken::new(),
610. 610 mpsc::channel(256).0,
611. 611 )
612. 612 .await;
613. 613
614. 614 assert!(
615. 615 matches!(outcome, TurnOutcome::Completed { .. }),
616. 616 "got {outcome:?}"
617. 617 );
618. 618 let answer = std::fs::read_to_string(dir.path().join("answer.md")).unwrap();
619. 619 assert!(
620. 620 answer.contains("echo: flights booked"),
621. 621 "artifact must carry the MCP result"
622. 622 );
623. 623}
624. 624
625. 625#[tokio::test]
626. 626async fn mcp_call_omitting_server_auto_resolves_and_completes() {
627. 627 let Ok(probe) = std::process::Command::new("python3")
628. 628 .arg("--version")
629. 629 .output()
630. 630 else {
631. 631 eprintln!("skipping: python3 unavailable");
632. 632 return;
633. 633 };
634. 634 assert!(probe.status.success(), "python3 must be runnable");
635. 635
636. 636 let script = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../scripts/fake_mcp_server.py");
637. 637 let mut servers = HashMap::new();
638. 638 servers.insert(
639. 639 "notes".to_string(),
640. 640 ServerConfig {
641. 641 command: "python3".into(),
642. 642 args: vec![script.display().to_string()],
643. 643 env: Vec::new(),
644. 644 network: false,
645. 645 },
646. 646 );
647. 647 let manager = Arc::new(McpManager::new(servers, dir_safe_cwd()));
648. 648 let aliases = Arc::new(Mutex::new(HashMap::new()));
649. 649 let aliases_map = aliases.clone();
650. 650 let mcp_tool: Arc<dyn vak_tools::Tool> = Arc::new(McpTool::new(manager).with_catalog_observer(
651. 651 Arc::new(move |catalog| {
652. 652 let mut registered = aliases_map.lock().unwrap();
653. 653 for (server, tools) in catalog {
654. 654 for tool in tools {
655. 655 registered.insert(tool.name.clone(), server.clone());
656. 656 }
657. 657 }
658. 658 }),
659. 659 ));
660. 660
661. 661 // Pre-populate alias as would be done from admitted capability inventory
662. 662 aliases
663. 663 .lock()
664. 664 .unwrap()
665. 665 .insert("echo".into(), "notes".into());
666. 666
667. 667 let (mut agent, _provider, dir) = setup(
668. 668 vec![
669. 669 // Model calls `mcp` with action=call and tool=echo, but completely OMITS `server`!
670. 670 tool_call(
671. 671 "m1",
672. 672 "mcp",
673. 673 serde_json::json!({
674. 674 "action": "call",
675. 675 "tool": "echo",
676. 676 "arguments": {"text": "flights booked"}
677. 677 }),
678. 678 ),
679. 679 tool_call(
680. 680 "w1",
681. 681 "write",
682. 682 serde_json::json!({
683. 683 "path": "auto_resolved.md",
684. 684 "content": "# Result\n\nAuto-resolved echo succeeded\n"
685. 685 }),
686. 686 ),
687. 687 text_msg("auto-resolved and done"),
688. 688 ],
689. 689 vec![mcp_tool, Arc::new(WriteTool)],
690. 690 |config| {
691. 691 config.mcp_tool_index = aliases.clone();
692. 692 },
693. 693 );
694. 694
695. 695 let outcome = agent
696. 696 .run(
697. 697 "Echo flights booked without specifying server",
698. 698 &Default::default(),
699. 699 CancellationToken::new(),
700. 700 mpsc::channel(256).0,
701. 701 )
702. 702 .await;
703. 703
704. 704 assert!(
705. 705 matches!(outcome, TurnOutcome::Completed { .. }),
706. 706 "got {outcome:?}"
707. 707 );
708. 708 assert!(dir.path().join("auto_resolved.md").is_file());
709. 709}
710. 710
711. 711fn dir_safe_cwd() -> std::path::PathBuf {
712. 712 std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from("."))
713. 713}
714. 714
715. 715/// In read-only mode a note-taking write is denied with a typed reason; the
716. 716/// model adapts by delivering the content inline and nothing hits disk.
717. 717#[tokio::test]
718. 718async fn read_only_mode_denies_note_writes_and_agent_reports_inline() {
719. 719 let (mut agent, _provider, dir) = setup(
720. 720 vec![
721. 721 tool_call(
722. 722 "t1",
723. 723 "write",
724. 724 serde_json::json!({
725. 725 "path": "field-notes.md",
726. 726 "content": "# Field Notes\n\nObservation one recorded on site.\n"
727. 727 }),
728. 728 ),
729. 729 text_msg(
730. 730 "Saving is unavailable in read-only mode. Field notes inline: Observation one recorded on site.",
731. 731 ),
732. 732 ],
733. 733 vec![Arc::new(WriteTool)],
734. 734 |cfg| {
735. 735 cfg.mode = Mode::ReadOnly;
736. 736 },
737. 737 );
738. 738
739. 739 let outcome = agent
740. 740 .run(
741. 741 "Record today's field observation in field-notes.md.",
742. 742 &Default::default(),
743. 743 CancellationToken::new(),
744. 744 mpsc::channel(64).0,
745. 745 )
746. 746 .await;
747. 747
748. 748 match outcome {
749. 749 TurnOutcome::Completed { response } => {
750. 750 assert!(response.text_content().contains("inline"));
751. 751 }
752. 752 other => panic!("denied write must not fail the run, got {other:?}"),
753. 753 }
754. 754 assert!(
755. 755 !dir.path().join("field-notes.md").exists(),
756. 756 "denied write must never touch disk"
757. 757 );
758. 758 // Raw ledger: the closed turn's result is a trace line in the
759. 759 // projection now (docs/design/68-context-engine.md §10); this checks
760. 760 // what actually got recorded.
761. 761 let denied = agent
762. 762 .session
763. 763 .lock()
764. 764 .await
765. 765 .message_chain()
766. 766 .iter()
767. 767 .flat_map(|(_, m)| m.content.iter())
768. 768 .find_map(|b| match b {
769. 769 ContentBlock::ToolResult {
770. 770 content, is_error, ..
771. 771 } => Some((content.clone(), *is_error)),
772. 772 _ => None,
773. 773 })
774. 774 .expect("denied call must produce a tool result");
775. 775 assert!(denied.1, "denied write is an error result");
776. 776 assert!(denied.0.contains("read-only mode denies"));
777. 777}
778. 778