Vakyartha documentation

1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2
3use std::collections::VecDeque;
4use std::sync::{Arc, Mutex};
5
6use tokio::sync::mpsc;
7use tokio_util::sync::CancellationToken;
8
9use tempfile::tempdir;
10
11use vak_agent::{Agent, AgentConfig, AgentEvent, StopPolicy, TurnOutcome};
12use vak_llm::stream;
13use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
14use vak_llm::{EventStream, LlmError, Provider};
15use vak_session::SessionLog;
16use vak_session::types::{FrozenContract, SessionHeader};
17
18enum ScriptedResponse {
19 Message(AssistantMessage),
20 #[allow(dead_code)]
21 Error(LlmError),
22}
23
24struct Scripted {
25 responses: Mutex<VecDeque<ScriptedResponse>>,
26 requests: Arc<Mutex<Vec<ChatRequest>>>,
27}
28
29#[async_trait::async_trait]
30impl Provider for Scripted {
31 fn name(&self) -> &str {
32 "scripted"
33 }
34
35 async fn stream(
36 &self,
37 request: ChatRequest,
38 _cancel: CancellationToken,
39 ) -> Result<EventStream, LlmError> {
40 self.requests.lock().unwrap().push(request);
41 let next = self.responses.lock().unwrap().pop_front();
42 let (mut sink, rx) = stream::channel(64);
43 match next {
44 Some(ScriptedResponse::Message(m)) => {
45 sink.push(stream::StreamEvent::Start { partial: m.clone() });
46 sink.close_message(m).await;
47 }
48 Some(ScriptedResponse::Error(e)) => sink.close_error(e).await,
49 None => {
50 sink.close_error(LlmError::Parse("script exhausted".into()))
51 .await
52 }
53 }
54 Ok(rx)
55 }
56}
57
58fn text_msg(text: &str) -> AssistantMessage {
59 AssistantMessage {
60 content: vec![ContentBlock::text(text)],
61 stop_reason: StopReason::EndTurn,
62 usage: Usage {
63 input_tokens: 10,
64 output_tokens: 5,
65 ..Default::default()
66 },
67 model: "test-model".into(),
68 response_id: None,
69 }
70}
71
72struct Harness {
73 agent: Option<Agent>,
74 requests: Arc<Mutex<Vec<ChatRequest>>>,
75 events_tx: mpsc::Sender<AgentEvent>,
76 events_rx: mpsc::Receiver<AgentEvent>,
77}
78
79fn harness(policy: Option<StopPolicy>, responses: Vec<ScriptedResponse>) -> Harness {
80 let dir = tempdir().expect("tempdir");
81 let header = SessionHeader {
82 agent: None,
83 session_id: "s-stop".into(),
84 created_at: chrono::Utc::now(),
85 cwd: dir.path().to_path_buf(),
86 parent_session_id: None,
87 contract_id: None,
88 work_item_id: None,
89 conversation: None,
90 contract: FrozenContract {
91 app_version: "0.1.0".into(),
92 provider: "scripted".into(),
93 model: "test-model".into(),
94 route_ladder: Vec::new(),
95 route_objective: String::new(),
96 route_annotations: Vec::new(),
97 system_prompt: "sys".into(),
98 permission_mode: "full-access".into(),
99 capabilities: Vec::new(),
100 prompt_layers: Vec::new(),
101 },
102 };
103 let log = SessionLog::create(dir.path().join("s.jsonl"), header).expect("ledger");
104 let mut cfg = AgentConfig::new("sys");
105 cfg.stop_policy = policy;
106 let requests: Arc<Mutex<Vec<ChatRequest>>> = Arc::new(Mutex::new(Vec::new()));
107 let provider = Arc::new(Scripted {
108 responses: Mutex::new(responses.into_iter().collect()),
109 requests: requests.clone(),
110 });
111 let (tx, rx) = mpsc::channel(4096);
112 std::mem::forget(dir);
113 Harness {
114 agent: Some(Agent::new(provider, log, cfg)),
115 requests,
116 events_tx: tx,
117 events_rx: rx,
118 }
119}
120
121fn drain_events(h: &mut Harness) -> Vec<AgentEvent> {
122 let mut out = Vec::new();
123 while let Ok(ev) = h.events_rx.try_recv() {
124 out.push(ev);
125 }
126 out
127}
128
129// Raw ledger, not the model-visible projection: a stop-guard nudge is
130// mid-turn scaffolding, dropped once the turn actually closes
131// (docs/design/68-context-engine.md §10). This only checks the nudge was
132// RECORDED — being recorded does not by itself mean it reached the model;
133// that depends on the still-open turn projecting verbatim rather than
134// being (incorrectly) treated as already closed. See
135// `TurnIndex::from_log`'s `closed` derivation and
136// `a_draft_followed_by_a_control_nudge_with_no_reply_yet_stays_open` in
137// vak-session/src/turns.rs for the actual delivery guarantee.
138async fn guard_messages(h: &mut Harness) -> Vec<String> {
139 let agent = h.agent.take().expect("guard_messages consumes the agent");
140 let session = agent.into_session().await;
141 session
142 .message_chain()
143 .into_iter()
144 .filter(|(_, m)| m.role == vak_llm::Role::User)
145 .filter_map(|(_, m)| match m.content.first() {
146 Some(ContentBlock::Text { text }) => Some(text.clone()),
147 _ => None,
148 })
149 .collect()
150}
151
152#[tokio::test]
153async fn truncated_plan_gets_one_continuation_then_completes() {
154 let mut h = harness(
155 Some(StopPolicy {
156 marker_gate: true,
157 verify_gate: false,
158 max_blocks: 2,
159 }),
160 vec![
161 ScriptedResponse::Message(text_msg("Fixing both:")),
162 ScriptedResponse::Message(text_msg("All done — both fixed, tests green.")),
163 ],
164 );
165 let outcome = h
166 .agent
167 .as_mut()
168 .expect("agent")
169 .run(
170 "fix the two bugs",
171 &Default::default(),
172 CancellationToken::new(),
173 h.events_tx.clone(),
174 )
175 .await;
176 let text = match outcome {
177 TurnOutcome::Completed { response } => response.text_content(),
178 other => panic!("expected completion, got {other:?}"),
179 };
180 assert_eq!(text, "All done — both fixed, tests green.");
181 assert_eq!(
182 h.requests.lock().unwrap().len(),
183 2,
184 "model must be re-called"
185 );
186
187 let events = drain_events(&mut h);
188 let guards: Vec<&String> = events
189 .iter()
190 .filter_map(|e| match e {
191 AgentEvent::StopHookContinuation { reason } => Some(reason),
192 _ => None,
193 })
194 .collect();
195 assert_eq!(guards.len(), 1);
196 assert!(guards[0].contains("cut off"), "reason: {}", guards[0]);
197
198 let users = guard_messages(&mut h).await;
199 assert!(
200 users.iter().any(|u| u.starts_with("[stop-guard]:")),
201 "ledger must contain the guard nudge: {users:?}"
202 );
203}
204
205/// The regression this file's `guard_messages` alone could not catch: being
206/// recorded in the ledger is not the same as reaching the model. This
207/// inspects the actual second `ChatRequest` the provider received and
208/// checks both halves of the delivery guarantee — the nudge text is
209/// present verbatim, and the request ends on a user-role message (current
210/// Claude models reject a request with no trailing user turn — invariant
211/// covered by `TurnIndex::from_log`'s `closed` derivation in
212/// vak-session/src/turns.rs).
213#[tokio::test]
214async fn the_stop_guard_nudge_reaches_the_actual_next_request() {
215 let mut h = harness(
216 Some(StopPolicy {
217 marker_gate: true,
218 verify_gate: false,
219 max_blocks: 2,
220 }),
221 vec![
222 ScriptedResponse::Message(text_msg("Fixing both:")),
223 ScriptedResponse::Message(text_msg("All done — both fixed, tests green.")),
224 ],
225 );
226 let outcome = h
227 .agent
228 .as_mut()
229 .expect("agent")
230 .run(
231 "fix the two bugs",
232 &Default::default(),
233 CancellationToken::new(),
234 h.events_tx.clone(),
235 )
236 .await;
237 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
238
239 let requests = h.requests.lock().unwrap().clone();
240 assert_eq!(requests.len(), 2, "model must be re-called after the guard");
241 let redo_request = &requests[1];
242
243 let last = redo_request
244 .messages
245 .last()
246 .expect("the redo request must carry at least one message");
247 assert_eq!(
248 last.role,
249 vak_llm::Role::User,
250 "the request must not end on the assistant's own draft: {:?}",
251 redo_request.messages
252 );
253
254 let carries_nudge = redo_request.messages.iter().any(|m| {
255 m.content.iter().any(|b| match b {
256 ContentBlock::Text { text } => text.starts_with("[stop-guard]:"),
257 _ => false,
258 })
259 });
260 assert!(
261 carries_nudge,
262 "the redo request must carry the [stop-guard] nudge verbatim: {:?}",
263 redo_request.messages
264 );
265}
266
267#[tokio::test]
268async fn verify_gate_blocks_when_prompt_demands_and_no_bash_ran() {
269 let mut h = harness(
270 Some(StopPolicy {
271 marker_gate: false,
272 verify_gate: true,
273 max_blocks: 1,
274 }),
275 vec![
276 ScriptedResponse::Message(text_msg("Created fizzbuzz.py.")),
277 ScriptedResponse::Message(text_msg("Ran it — output verified.")),
278 ],
279 );
280 let outcome = h
281 .agent
282 .as_mut()
283 .expect("agent")
284 .run(
285 "Create fizzbuzz.py and run it to prove that it works.",
286 &Default::default(),
287 CancellationToken::new(),
288 h.events_tx.clone(),
289 )
290 .await;
291 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
292 assert_eq!(h.requests.lock().unwrap().len(), 2);
293
294 let guards: Vec<String> = drain_events(&mut h)
295 .into_iter()
296 .filter_map(|e| match e {
297 AgentEvent::StopHookContinuation { reason } => Some(reason),
298 _ => None,
299 })
300 .collect();
301 assert_eq!(guards.len(), 1);
302 assert!(guards[0].contains("verification"), "reason: {}", guards[0]);
303}
304
305#[tokio::test]
306async fn max_blocks_cap_lets_second_bad_response_through() {
307 let mut h = harness(
308 Some(StopPolicy {
309 marker_gate: true,
310 verify_gate: false,
311 max_blocks: 1,
312 }),
313 vec![
314 ScriptedResponse::Message(text_msg("Now I'll fix it")),
315 ScriptedResponse::Message(text_msg("Now I'll really fix it")),
316 ],
317 );
318 let outcome = h
319 .agent
320 .as_mut()
321 .expect("agent")
322 .run(
323 "fix",
324 &Default::default(),
325 CancellationToken::new(),
326 h.events_tx.clone(),
327 )
328 .await;
329 match outcome {
330 TurnOutcome::Completed { response } => {
331 assert_eq!(response.text_content(), "Now I'll really fix it")
332 }
333 other => panic!("expected completion at cap, got {other:?}"),
334 }
335 assert_eq!(h.requests.lock().unwrap().len(), 2, "cap stops the loop");
336 let guards: Vec<_> = guard_messages(&mut h)
337 .await
338 .into_iter()
339 .filter(|u| u.starts_with("[stop-guard]:"))
340 .collect();
341 assert_eq!(guards.len(), 1, "exactly one nudge persisted");
342}
343
344#[tokio::test]
345async fn guard_at_turn_limit_is_not_reported_as_completed() {
346 let mut h = harness(
347 Some(StopPolicy {
348 marker_gate: true,
349 verify_gate: false,
350 max_blocks: 1,
351 }),
352 vec![ScriptedResponse::Message(text_msg("Fixing both:"))],
353 );
354 h.agent.as_mut().expect("agent").config.max_turns = 1;
355 let outcome = h
356 .agent
357 .as_mut()
358 .expect("agent")
359 .run(
360 "fix the two bugs",
361 &Default::default(),
362 CancellationToken::new(),
363 h.events_tx.clone(),
364 )
365 .await;
366 assert!(matches!(outcome, TurnOutcome::MaxTurnsReached));
367}
368
369#[tokio::test]
370async fn disabled_policy_never_blocks() {
371 let mut h = harness(
372 None,
373 vec![ScriptedResponse::Message(text_msg("Fixing both:"))],
374 );
375 let outcome = h
376 .agent
377 .as_mut()
378 .expect("agent")
379 .run(
380 "fix",
381 &Default::default(),
382 CancellationToken::new(),
383 h.events_tx.clone(),
384 )
385 .await;
386 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
387 assert_eq!(h.requests.lock().unwrap().len(), 1, "no continuation");
388 assert!(
389 drain_events(&mut h)
390 .into_iter()
391 .all(|e| !matches!(e, AgentEvent::StopHookContinuation { .. }))
392 );
393 assert!(
394 !guard_messages(&mut h)
395 .await
396 .iter()
397 .any(|u| u.starts_with("[stop-guard]:"))
398 );
399}
400
401#[tokio::test]
402async fn authored_prose_without_a_file_target_completes_without_a_guard() {
403 let mut h = harness(
404 Some(StopPolicy {
405 marker_gate: false,
406 verify_gate: true,
407 max_blocks: 1,
408 }),
409 vec![ScriptedResponse::Message(text_msg(
410 "Here is the component, explained step by step.",
411 ))],
412 );
413
414 let mut reading = vak_intent::Reading::general();
415 reading.act = vak_intent::Act::Author;
416 let spec =
417 vak_intent::OutcomeSpec::from_reading("Build a react visualization component", &reading, 1);
418 h.agent.as_mut().expect("agent").config.outcome = Some(spec);
419
420 let outcome = h
421 .agent
422 .as_mut()
423 .expect("agent")
424 .run(
425 "Build a react visualization component",
426 &Default::default(),
427 CancellationToken::new(),
428 h.events_tx.clone(),
429 )
430 .await;
431 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
432 assert_eq!(
433 h.requests.lock().unwrap().len(),
434 1,
435 "authoring is not an effect"
436 );
437 assert!(
438 drain_events(&mut h)
439 .iter()
440 .all(|e| !matches!(e, AgentEvent::StopHookContinuation { .. }))
441 );
442}
443
444#[tokio::test]
445async fn universal_task_with_verify_completes_without_bash_guard() {
446 let mut h = harness(
447 Some(StopPolicy {
448 marker_gate: true,
449 verify_gate: true,
450 max_blocks: 1,
451 }),
452 vec![ScriptedResponse::Message(text_msg(
453 "Here is the brewing guide: grind size, water-to-coffee ratio, and brew temperature. I have verified that water temperature should be kept between 90°C and 96°C for optimal extraction.",
454 ))],
455 );
456
457 let outcome = h
458 .agent
459 .as_mut()
460 .expect("agent")
461 .run(
462 "Explain the 3 main coffee brewing variables and verify that water temperature recommendations are included.",
463 &Default::default(),
464 CancellationToken::new(),
465 h.events_tx.clone(),
466 )
467 .await;
468
469 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
470 // Must complete in 1 call without being falsely trapped by VerificationMissing demanding bash
471 assert_eq!(h.requests.lock().unwrap().len(), 1);
472 assert!(
473 drain_events(&mut h)
474 .into_iter()
475 .all(|e| !matches!(e, AgentEvent::StopHookContinuation { .. }))
476 );
477}
478

1#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2
3use std::collections::VecDeque;
4use std::sync::{Arc, Mutex};
5
6use tokio::sync::mpsc;
7use tokio_util::sync::CancellationToken;
8
9use tempfile::tempdir;
10
11use vak_agent::{Agent, AgentConfig, AgentEvent, StopPolicy, TurnOutcome};
12use vak_llm::stream;
13use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage};
14use vak_llm::{EventStream, LlmError, Provider};
15use vak_session::SessionLog;
16use vak_session::types::{FrozenContract, SessionHeader};
17
18enum ScriptedResponse {
19 Message(AssistantMessage),
20 #[allow(dead_code)]
21 Error(LlmError),
22}
23
24struct Scripted {
25 responses: Mutex<VecDeque<ScriptedResponse>>,
26 requests: Arc<Mutex<Vec<ChatRequest>>>,
27}
28
29#[async_trait::async_trait]
30impl Provider for Scripted {
31 fn name(&self) -> &str {
32 "scripted"
33 }
34
35 async fn stream(
36 &self,
37 request: ChatRequest,
38 _cancel: CancellationToken,
39 ) -> Result<EventStream, LlmError> {
40 self.requests.lock().unwrap().push(request);
41 let next = self.responses.lock().unwrap().pop_front();
42 let (mut sink, rx) = stream::channel(64);
43 match next {
44 Some(ScriptedResponse::Message(m)) => {
45 sink.push(stream::StreamEvent::Start { partial: m.clone() });
46 sink.close_message(m).await;
47 }
48 Some(ScriptedResponse::Error(e)) => sink.close_error(e).await,
49 None => {
50 sink.close_error(LlmError::Parse("script exhausted".into()))
51 .await
52 }
53 }
54 Ok(rx)
55 }
56}
57
58fn text_msg(text: &str) -> AssistantMessage {
59 AssistantMessage {
60 content: vec![ContentBlock::text(text)],
61 stop_reason: StopReason::EndTurn,
62 usage: Usage {
63 input_tokens: 10,
64 output_tokens: 5,
65 ..Default::default()
66 },
67 model: "test-model".into(),
68 response_id: None,
69 }
70}
71
72struct Harness {
73 agent: Option<Agent>,
74 requests: Arc<Mutex<Vec<ChatRequest>>>,
75 events_tx: mpsc::Sender<AgentEvent>,
76 events_rx: mpsc::Receiver<AgentEvent>,
77}
78
79fn harness(policy: Option<StopPolicy>, responses: Vec<ScriptedResponse>) -> Harness {
80 let dir = tempdir().expect("tempdir");
81 let header = SessionHeader {
82 agent: None,
83 session_id: "s-stop".into(),
84 created_at: chrono::Utc::now(),
85 cwd: dir.path().to_path_buf(),
86 parent_session_id: None,
87 contract_id: None,
88 work_item_id: None,
89 conversation: None,
90 contract: FrozenContract {
91 app_version: "0.1.0".into(),
92 provider: "scripted".into(),
93 model: "test-model".into(),
94 route_ladder: Vec::new(),
95 route_objective: String::new(),
96 route_annotations: Vec::new(),
97 system_prompt: "sys".into(),
98 permission_mode: "full-access".into(),
99 capabilities: Vec::new(),
100 prompt_layers: Vec::new(),
101 },
102 };
103 let log = SessionLog::create(dir.path().join("s.jsonl"), header).expect("ledger");
104 let mut cfg = AgentConfig::new("sys");
105 cfg.stop_policy = policy;
106 let requests: Arc<Mutex<Vec<ChatRequest>>> = Arc::new(Mutex::new(Vec::new()));
107 let provider = Arc::new(Scripted {
108 responses: Mutex::new(responses.into_iter().collect()),
109 requests: requests.clone(),
110 });
111 let (tx, rx) = mpsc::channel(4096);
112 std::mem::forget(dir);
113 Harness {
114 agent: Some(Agent::new(provider, log, cfg)),
115 requests,
116 events_tx: tx,
117 events_rx: rx,
118 }
119}
120
121fn drain_events(h: &mut Harness) -> Vec<AgentEvent> {
122 let mut out = Vec::new();
123 while let Ok(ev) = h.events_rx.try_recv() {
124 out.push(ev);
125 }
126 out
127}
128
129// Raw ledger, not the model-visible projection: a stop-guard nudge is
130// mid-turn scaffolding, dropped once the turn actually closes
131// (docs/design/68-context-engine.md §10). This only checks the nudge was
132// RECORDED — being recorded does not by itself mean it reached the model;
133// that depends on the still-open turn projecting verbatim rather than
134// being (incorrectly) treated as already closed. See
135// `TurnIndex::from_log`'s `closed` derivation and
136// `a_draft_followed_by_a_control_nudge_with_no_reply_yet_stays_open` in
137// vak-session/src/turns.rs for the actual delivery guarantee.
138async fn guard_messages(h: &mut Harness) -> Vec<String> {
139 let agent = h.agent.take().expect("guard_messages consumes the agent");
140 let session = agent.into_session().await;
141 session
142 .message_chain()
143 .into_iter()
144 .filter(|(_, m)| m.role == vak_llm::Role::User)
145 .filter_map(|(_, m)| match m.content.first() {
146 Some(ContentBlock::Text { text }) => Some(text.clone()),
147 _ => None,
148 })
149 .collect()
150}
151
152#[tokio::test]
153async fn truncated_plan_gets_one_continuation_then_completes() {
154 let mut h = harness(
155 Some(StopPolicy {
156 marker_gate: true,
157 verify_gate: false,
158 max_blocks: 2,
159 }),
160 vec![
161 ScriptedResponse::Message(text_msg("Fixing both:")),
162 ScriptedResponse::Message(text_msg("All done — both fixed, tests green.")),
163 ],
164 );
165 let outcome = h
166 .agent
167 .as_mut()
168 .expect("agent")
169 .run(
170 "fix the two bugs",
171 &Default::default(),
172 CancellationToken::new(),
173 h.events_tx.clone(),
174 )
175 .await;
176 let text = match outcome {
177 TurnOutcome::Completed { response } => response.text_content(),
178 other => panic!("expected completion, got {other:?}"),
179 };
180 assert_eq!(text, "All done — both fixed, tests green.");
181 assert_eq!(
182 h.requests.lock().unwrap().len(),
183 2,
184 "model must be re-called"
185 );
186
187 let events = drain_events(&mut h);
188 let guards: Vec<&String> = events
189 .iter()
190 .filter_map(|e| match e {
191 AgentEvent::StopHookContinuation { reason } => Some(reason),
192 _ => None,
193 })
194 .collect();
195 assert_eq!(guards.len(), 1);
196 assert!(guards[0].contains("cut off"), "reason: {}", guards[0]);
197
198 let users = guard_messages(&mut h).await;
199 assert!(
200 users.iter().any(|u| u.starts_with("[stop-guard]:")),
201 "ledger must contain the guard nudge: {users:?}"
202 );
203}
204
205/// The regression this file's `guard_messages` alone could not catch: being
206/// recorded in the ledger is not the same as reaching the model. This
207/// inspects the actual second `ChatRequest` the provider received and
208/// checks both halves of the delivery guarantee — the nudge text is
209/// present verbatim, and the request ends on a user-role message (current
210/// Claude models reject a request with no trailing user turn — invariant
211/// covered by `TurnIndex::from_log`'s `closed` derivation in
212/// vak-session/src/turns.rs).
213#[tokio::test]
214async fn the_stop_guard_nudge_reaches_the_actual_next_request() {
215 let mut h = harness(
216 Some(StopPolicy {
217 marker_gate: true,
218 verify_gate: false,
219 max_blocks: 2,
220 }),
221 vec![
222 ScriptedResponse::Message(text_msg("Fixing both:")),
223 ScriptedResponse::Message(text_msg("All done — both fixed, tests green.")),
224 ],
225 );
226 let outcome = h
227 .agent
228 .as_mut()
229 .expect("agent")
230 .run(
231 "fix the two bugs",
232 &Default::default(),
233 CancellationToken::new(),
234 h.events_tx.clone(),
235 )
236 .await;
237 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
238
239 let requests = h.requests.lock().unwrap().clone();
240 assert_eq!(requests.len(), 2, "model must be re-called after the guard");
241 let redo_request = &requests[1];
242
243 let last = redo_request
244 .messages
245 .last()
246 .expect("the redo request must carry at least one message");
247 assert_eq!(
248 last.role,
249 vak_llm::Role::User,
250 "the request must not end on the assistant's own draft: {:?}",
251 redo_request.messages
252 );
253
254 let carries_nudge = redo_request.messages.iter().any(|m| {
255 m.content.iter().any(|b| match b {
256 ContentBlock::Text { text } => text.starts_with("[stop-guard]:"),
257 _ => false,
258 })
259 });
260 assert!(
261 carries_nudge,
262 "the redo request must carry the [stop-guard] nudge verbatim: {:?}",
263 redo_request.messages
264 );
265}
266
267#[tokio::test]
268async fn verify_gate_blocks_when_prompt_demands_and_no_bash_ran() {
269 let mut h = harness(
270 Some(StopPolicy {
271 marker_gate: false,
272 verify_gate: true,
273 max_blocks: 1,
274 }),
275 vec![
276 ScriptedResponse::Message(text_msg("Created fizzbuzz.py.")),
277 ScriptedResponse::Message(text_msg("Ran it — output verified.")),
278 ],
279 );
280 let outcome = h
281 .agent
282 .as_mut()
283 .expect("agent")
284 .run(
285 "Create fizzbuzz.py and run it to prove that it works.",
286 &Default::default(),
287 CancellationToken::new(),
288 h.events_tx.clone(),
289 )
290 .await;
291 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
292 assert_eq!(h.requests.lock().unwrap().len(), 2);
293
294 let guards: Vec<String> = drain_events(&mut h)
295 .into_iter()
296 .filter_map(|e| match e {
297 AgentEvent::StopHookContinuation { reason } => Some(reason),
298 _ => None,
299 })
300 .collect();
301 assert_eq!(guards.len(), 1);
302 assert!(guards[0].contains("verification"), "reason: {}", guards[0]);
303}
304
305#[tokio::test]
306async fn max_blocks_cap_lets_second_bad_response_through() {
307 let mut h = harness(
308 Some(StopPolicy {
309 marker_gate: true,
310 verify_gate: false,
311 max_blocks: 1,
312 }),
313 vec![
314 ScriptedResponse::Message(text_msg("Now I'll fix it")),
315 ScriptedResponse::Message(text_msg("Now I'll really fix it")),
316 ],
317 );
318 let outcome = h
319 .agent
320 .as_mut()
321 .expect("agent")
322 .run(
323 "fix",
324 &Default::default(),
325 CancellationToken::new(),
326 h.events_tx.clone(),
327 )
328 .await;
329 match outcome {
330 TurnOutcome::Completed { response } => {
331 assert_eq!(response.text_content(), "Now I'll really fix it")
332 }
333 other => panic!("expected completion at cap, got {other:?}"),
334 }
335 assert_eq!(h.requests.lock().unwrap().len(), 2, "cap stops the loop");
336 let guards: Vec<_> = guard_messages(&mut h)
337 .await
338 .into_iter()
339 .filter(|u| u.starts_with("[stop-guard]:"))
340 .collect();
341 assert_eq!(guards.len(), 1, "exactly one nudge persisted");
342}
343
344#[tokio::test]
345async fn guard_at_turn_limit_is_not_reported_as_completed() {
346 let mut h = harness(
347 Some(StopPolicy {
348 marker_gate: true,
349 verify_gate: false,
350 max_blocks: 1,
351 }),
352 vec![ScriptedResponse::Message(text_msg("Fixing both:"))],
353 );
354 h.agent.as_mut().expect("agent").config.max_turns = 1;
355 let outcome = h
356 .agent
357 .as_mut()
358 .expect("agent")
359 .run(
360 "fix the two bugs",
361 &Default::default(),
362 CancellationToken::new(),
363 h.events_tx.clone(),
364 )
365 .await;
366 assert!(matches!(outcome, TurnOutcome::MaxTurnsReached));
367}
368
369#[tokio::test]
370async fn disabled_policy_never_blocks() {
371 let mut h = harness(
372 None,
373 vec![ScriptedResponse::Message(text_msg("Fixing both:"))],
374 );
375 let outcome = h
376 .agent
377 .as_mut()
378 .expect("agent")
379 .run(
380 "fix",
381 &Default::default(),
382 CancellationToken::new(),
383 h.events_tx.clone(),
384 )
385 .await;
386 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
387 assert_eq!(h.requests.lock().unwrap().len(), 1, "no continuation");
388 assert!(
389 drain_events(&mut h)
390 .into_iter()
391 .all(|e| !matches!(e, AgentEvent::StopHookContinuation { .. }))
392 );
393 assert!(
394 !guard_messages(&mut h)
395 .await
396 .iter()
397 .any(|u| u.starts_with("[stop-guard]:"))
398 );
399}
400
401#[tokio::test]
402async fn authored_prose_without_a_file_target_completes_without_a_guard() {
403 let mut h = harness(
404 Some(StopPolicy {
405 marker_gate: false,
406 verify_gate: true,
407 max_blocks: 1,
408 }),
409 vec![ScriptedResponse::Message(text_msg(
410 "Here is the component, explained step by step.",
411 ))],
412 );
413
414 let mut reading = vak_intent::Reading::general();
415 reading.act = vak_intent::Act::Author;
416 let spec =
417 vak_intent::OutcomeSpec::from_reading("Build a react visualization component", &reading, 1);
418 h.agent.as_mut().expect("agent").config.outcome = Some(spec);
419
420 let outcome = h
421 .agent
422 .as_mut()
423 .expect("agent")
424 .run(
425 "Build a react visualization component",
426 &Default::default(),
427 CancellationToken::new(),
428 h.events_tx.clone(),
429 )
430 .await;
431 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
432 assert_eq!(
433 h.requests.lock().unwrap().len(),
434 1,
435 "authoring is not an effect"
436 );
437 assert!(
438 drain_events(&mut h)
439 .iter()
440 .all(|e| !matches!(e, AgentEvent::StopHookContinuation { .. }))
441 );
442}
443
444#[tokio::test]
445async fn universal_task_with_verify_completes_without_bash_guard() {
446 let mut h = harness(
447 Some(StopPolicy {
448 marker_gate: true,
449 verify_gate: true,
450 max_blocks: 1,
451 }),
452 vec![ScriptedResponse::Message(text_msg(
453 "Here is the brewing guide: grind size, water-to-coffee ratio, and brew temperature. I have verified that water temperature should be kept between 90°C and 96°C for optimal extraction.",
454 ))],
455 );
456
457 let outcome = h
458 .agent
459 .as_mut()
460 .expect("agent")
461 .run(
462 "Explain the 3 main coffee brewing variables and verify that water temperature recommendations are included.",
463 &Default::default(),
464 CancellationToken::new(),
465 h.events_tx.clone(),
466 )
467 .await;
468
469 assert!(matches!(outcome, TurnOutcome::Completed { .. }));
470 // Must complete in 1 call without being falsely trapped by VerificationMissing demanding bash
471 assert_eq!(h.requests.lock().unwrap().len(), 1);
472 assert!(
473 drain_events(&mut h)
474 .into_iter()
475 .all(|e| !matches!(e, AgentEvent::StopHookContinuation { .. }))
476 );
477}
478

Stop Guard

Indexing the workspace…

Stop Guard