- 1
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] - 2
- 3
//! A call a tool refuses from its arguments alone is refused before - 4
//! permission is evaluated: nobody is asked to approve a text edit of a - 5
//! Word file, which could only fail, and the model gets the repair hint. - 6
//! And when correctable failures persist, the loop's repair directive is - 7
//! runtime-authored control traffic, never a message from the person. - 8
//! And an answer after a tool delivered a reviewable file (an Office - 9
//! draft) is not sent back to be re-presented as a card; a repeat of the - 10
//! delivering call writes no second draft, and a card previewing the - 11
//! delivered file is not shown. - 12
- 13
use std::collections::VecDeque; - 14
use std::sync::atomic::{AtomicUsize, Ordering}; - 15
use std::sync::{Arc, Mutex}; - 16
- 17
use async_trait::async_trait; - 18
use tokio::sync::mpsc; - 19
use tokio_util::sync::CancellationToken; - 20
- 21
use vak_agent::{Agent, AgentConfig, AgentEvent, TurnOutcome}; - 22
use vak_llm::stream; - 23
use vak_llm::types::{AssistantMessage, ChatRequest, ContentBlock, StopReason, Usage}; - 24
use vak_llm::{EventStream, LlmError, Provider}; - 25
use vak_permission::{Mode, PermissionEngine}; - 26
use vak_session::types::{FrozenContract, SessionHeader}; - 27
use vak_session::{SessionLog, SessionPath}; - 28
- 29
struct Scripted( - 30
Mutex<VecDeque<AssistantMessage>>, - 31
Arc<Mutex<Vec<ChatRequest>>>, - 32
); - 33
- 34
#[async_trait] - 35
impl Provider for Scripted { - 36
fn name(&self) -> &str { - 37
"scripted" - 38
} - 39
- 40
async fn stream( - 41
&self, - 42
request: ChatRequest, - 43
_cancel: CancellationToken, - 44
) -> Result<EventStream, LlmError> { - 45
self.1.lock().unwrap().push(request); - 46
let next = self.0.lock().unwrap().pop_front(); - 47
let (mut sink, rx) = stream::channel(64); - 48
match next { - 49
Some(m) => { - 50
sink.push(stream::StreamEvent::Start { partial: m.clone() }); - 51
sink.close_message(m).await; - 52
} - 53
None => sink.close_error(LlmError::Parse("exhausted".into())).await, - 54
} - 55
Ok(rx) - 56
} - 57
} - 58
- 59
fn msg(content: Vec<ContentBlock>, stop_reason: StopReason) -> AssistantMessage { - 60
AssistantMessage { - 61
content, - 62
stop_reason, - 63
usage: Usage { - 64
input_tokens: 1, - 65
output_tokens: 1, - 66
..Default::default() - 67
}, - 68
model: "test-model".into(), - 69
response_id: None, - 70
} - 71
} - 72
- 73
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] - 74
async fn a_text_edit_of_a_word_file_is_refused_without_asking_anyone() { - 75
let dir = tempfile::tempdir().unwrap(); - 76
let home = dir.path().join("home"); - 77
std::fs::create_dir_all(&home).unwrap(); - 78
std::fs::write( - 79
dir.path().join("q3.docx"), - 80
b"PK\x03\x04 not really a package", - 81
) - 82
.unwrap(); - 83
let header = SessionHeader { - 84
agent: None, - 85
session_id: "refusal".into(), - 86
created_at: chrono::Utc::now(), - 87
cwd: dir.path().to_path_buf(), - 88
parent_session_id: None, - 89
contract_id: None, - 90
work_item_id: None, - 91
conversation: None, - 92
contract: FrozenContract { - 93
app_version: "0".into(), - 94
provider: "scripted".into(), - 95
model: "test-model".into(), - 96
route_ladder: Vec::new(), - 97
route_objective: String::new(), - 98
route_annotations: Vec::new(), - 99
system_prompt: "sys".into(), - 100
permission_mode: "workspace-write".into(), - 101
capabilities: Vec::new(), - 102
prompt_layers: Vec::new(), - 103
}, - 104
}; - 105
let log = SessionLog::create( - 106
SessionPath::new_session_file(&home, dir.path(), "refusal"), - 107
header, - 108
) - 109
.unwrap(); - 110
let mut cfg = AgentConfig::new("sys"); - 111
cfg.model = "test-model".into(); - 112
cfg.mode = Mode::WorkspaceWrite; - 113
cfg.retry_base_backoff_ms = 1; - 114
cfg.run_retry_base_backoff_ms = 1; - 115
// Every edit would ask for approval, so an approval request is what the - 116
// refusal must pre-empt. - 117
cfg.permission = Some(Arc::new( - 118
PermissionEngine::from_rule_strings(&["?edit".to_string()]).unwrap(), - 119
)); - 120
cfg.tools = vec![Arc::new(vak_tools::edit::EditTool)]; - 121
let script = VecDeque::from([msg( - 122
vec![ContentBlock::ToolUse { - 123
id: "c1".into(), - 124
name: "edit".into(), - 125
input: serde_json::json!({ - 126
"path": "q3.docx", - 127
"edits": [{"old_text": "Steady.", "new_text": "Growing."}] - 128
}), - 129
}], - 130
StopReason::ToolUse, - 131
)]); - 132
// Spare answers: a text answer after a failed call may be sent back for - 133
// a redo, and this test is about what happened before it. - 134
let mut script = script; - 135
for _ in 0..6 { - 136
script.push_back(msg( - 137
vec![ContentBlock::text( - 138
"The file is a Word document, so I will use office_apply.", - 139
)], - 140
StopReason::EndTurn, - 141
)); - 142
} - 143
let requests = Arc::new(Mutex::new(Vec::new())); - 144
let mut agent = Agent::new( - 145
Arc::new(Scripted(Mutex::new(script), requests.clone())), - 146
log, - 147
cfg, - 148
); - 149
let (tx, mut rx) = mpsc::channel(256); - 150
let outcome = agent - 151
.run("edit it", &Default::default(), CancellationToken::new(), tx) - 152
.await; - 153
assert!( - 154
matches!(outcome, TurnOutcome::Completed { .. }), - 155
"{outcome:?}" - 156
); - 157
let mut approvals = 0; - 158
while let Ok(event) = rx.try_recv() { - 159
if matches!(event, AgentEvent::ApprovalRequested { .. }) { - 160
approvals += 1; - 161
} - 162
} - 163
assert_eq!( - 164
approvals, 0, - 165
"a call that can only be refused is never put to a person" - 166
); - 167
let requests = requests.lock().unwrap(); - 168
let refusal = requests - 169
.iter() - 170
.flat_map(|request| request.messages.iter()) - 171
.flat_map(|message| message.content.iter()) - 172
.find_map(|block| match block { - 173
ContentBlock::ToolResult { - 174
tool_use_id, - 175
content, - 176
is_error: true, - 177
} if tool_use_id == "c1" => Some(content.clone()), - 178
_ => None, - 179
}) - 180
.expect("the model is told why"); - 181
assert!( - 182
refusal.contains("office_apply") && refusal.contains("doc_read"), - 183
"{refusal}" - 184
); - 185
} - 186
- 187
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] - 188
async fn the_repair_directive_is_recorded_as_control_not_as_the_persons_words() { - 189
let dir = tempfile::tempdir().unwrap(); - 190
let home = dir.path().join("home"); - 191
std::fs::create_dir_all(&home).unwrap(); - 192
std::fs::write(dir.path().join("notes.txt"), "plain").unwrap(); - 193
let header = SessionHeader { - 194
agent: None, - 195
session_id: "directive".into(), - 196
created_at: chrono::Utc::now(), - 197
cwd: dir.path().to_path_buf(), - 198
parent_session_id: None, - 199
contract_id: None, - 200
work_item_id: None, - 201
conversation: None, - 202
contract: FrozenContract { - 203
app_version: "0".into(), - 204
provider: "scripted".into(), - 205
model: "test-model".into(), - 206
route_ladder: Vec::new(), - 207
route_objective: String::new(), - 208
route_annotations: Vec::new(), - 209
system_prompt: "sys".into(), - 210
permission_mode: "workspace-write".into(), - 211
capabilities: Vec::new(), - 212
prompt_layers: Vec::new(), - 213
}, - 214
}; - 215
let ledger = SessionPath::new_session_file(&home, dir.path(), "directive"); - 216
let log = SessionLog::create(ledger.clone(), header).unwrap(); - 217
let mut cfg = AgentConfig::new("sys"); - 218
cfg.model = "test-model".into(); - 219
cfg.mode = Mode::WorkspaceWrite; - 220
cfg.retry_base_backoff_ms = 1; - 221
cfg.run_retry_base_backoff_ms = 1; - 222
cfg.tools = vec![Arc::new(vak_tools::edit::EditTool)]; - 223
// The same malformed call, step after step: `edits` is missing. - 224
let mut script: VecDeque<AssistantMessage> = (0..8) - 225
.map(|n| { - 226
msg( - 227
vec![ContentBlock::ToolUse { - 228
id: format!("c{n}"), - 229
name: "edit".into(), - 230
input: serde_json::json!({"path": "notes.txt"}), - 231
}], - 232
StopReason::ToolUse, - 233
) - 234
}) - 235
.collect(); - 236
for _ in 0..4 { - 237
script.push_back(msg( - 238
vec![ContentBlock::text("Stopped.")], - 239
StopReason::EndTurn, - 240
)); - 241
} - 242
let requests = Arc::new(Mutex::new(Vec::new())); - 243
let mut agent = Agent::new( - 244
Arc::new(Scripted(Mutex::new(script), requests.clone())), - 245
log, - 246
cfg, - 247
); - 248
let (tx, mut rx) = mpsc::channel(512); - 249
tokio::spawn(async move { while rx.recv().await.is_some() {} }); - 250
agent - 251
.run( - 252
"fix the notes", - 253
&Default::default(), - 254
CancellationToken::new(), - 255
tx, - 256
) - 257
.await; - 258
let ledger = std::fs::read_to_string(&ledger).unwrap(); - 259
let directives: Vec<&str> = ledger - 260
.lines() - 261
.filter(|line| line.contains("[repair-directive]")) - 262
.collect(); - 263
assert!(!directives.is_empty(), "the loop issued a repair directive"); - 264
for line in directives { - 265
assert!( - 266
line.contains("\"control\":\"repair_directive\""), - 267
"tagged as runtime control: {line}" - 268
); - 269
} - 270
assert!( - 271
!ledger.contains("[repair directive]"), - 272
"the retired inline marker is not written" - 273
); - 274
} - 275
- 276
#[derive(Default)] - 277
struct Deliverer { - 278
delivers: bool, - 279
runs: AtomicUsize, - 280
} - 281
- 282
#[async_trait] - 283
impl vak_tools::Tool for Deliverer { - 284
fn name(&self) -> &str { - 285
"make_draft" - 286
} - 287
fn description(&self) -> &str { - 288
"test stand-in" - 289
} - 290
fn schema(&self) -> serde_json::Value { - 291
serde_json::json!({"type": "object"}) - 292
} - 293
fn delivered_file(&self, _args: &serde_json::Value) -> Option<String> { - 294
self.delivers.then(|| "deck.pptx".to_string()) - 295
} - 296
async fn execute( - 297
&self, - 298
_args: &serde_json::Value, - 299
_ctx: &vak_tools::context::ToolContext, - 300
) -> vak_tools::ToolOutput { - 301
self.runs.fetch_add(1, Ordering::SeqCst); - 302
vak_tools::ToolOutput::ok("Draft for deck.pptx written; the person reviews it.") - 303
} - 304
} - 305
- 306
#[derive(Default)] - 307
struct Shell { - 308
runs: AtomicUsize, - 309
} - 310
- 311
#[async_trait] - 312
impl vak_tools::Tool for Shell { - 313
fn name(&self) -> &str { - 314
"bash" - 315
} - 316
fn description(&self) -> &str { - 317
"test stand-in" - 318
} - 319
fn schema(&self) -> serde_json::Value { - 320
serde_json::json!({"type": "object"}) - 321
} - 322
async fn execute( - 323
&self, - 324
_args: &serde_json::Value, - 325
_ctx: &vak_tools::context::ToolContext, - 326
) -> vak_tools::ToolOutput { - 327
self.runs.fetch_add(1, Ordering::SeqCst); - 328
vak_tools::ToolOutput::ok("ran") - 329
} - 330
} - 331
- 332
#[derive(Default)] - 333
struct PreviewCard { - 334
runs: AtomicUsize, - 335
} - 336
- 337
#[async_trait] - 338
impl vak_tools::Tool for PreviewCard { - 339
fn name(&self) -> &str { - 340
"emit_ui_preview_card" - 341
} - 342
fn description(&self) -> &str { - 343
"test stand-in" - 344
} - 345
fn schema(&self) -> serde_json::Value { - 346
serde_json::json!({"type": "object"}) - 347
} - 348
fn presents_cards(&self) -> bool { - 349
true - 350
} - 351
async fn execute( - 352
&self, - 353
_args: &serde_json::Value, - 354
_ctx: &vak_tools::context::ToolContext, - 355
) -> vak_tools::ToolOutput { - 356
self.runs.fetch_add(1, Ordering::SeqCst); - 357
vak_tools::ToolOutput::ok(r#"{"ok":true}"#) - 358
} - 359
} - 360
- 361
/// How many model requests one turn takes when the presentation check - 362
/// always wants a card. - 363
async fn requests_after_a_draft(delivers: bool) -> usize { - 364
let dir = tempfile::tempdir().unwrap(); - 365
let home = dir.path().join("home"); - 366
std::fs::create_dir_all(&home).unwrap(); - 367
let header = SessionHeader { - 368
agent: None, - 369
session_id: "deliver".into(), - 370
created_at: chrono::Utc::now(), - 371
cwd: dir.path().to_path_buf(), - 372
parent_session_id: None, - 373
contract_id: None, - 374
work_item_id: None, - 375
conversation: None, - 376
contract: FrozenContract { - 377
app_version: "0".into(), - 378
provider: "scripted".into(), - 379
model: "test-model".into(), - 380
route_ladder: Vec::new(), - 381
route_objective: String::new(), - 382
route_annotations: Vec::new(), - 383
system_prompt: "sys".into(), - 384
permission_mode: "workspace-write".into(), - 385
capabilities: Vec::new(), - 386
prompt_layers: Vec::new(), - 387
}, - 388
}; - 389
let log = SessionLog::create( - 390
SessionPath::new_session_file(&home, dir.path(), "deliver"), - 391
header, - 392
) - 393
.unwrap(); - 394
let mut cfg = AgentConfig::new("sys"); - 395
cfg.model = "test-model".into(); - 396
cfg.mode = Mode::WorkspaceWrite; - 397
cfg.retry_base_backoff_ms = 1; - 398
cfg.run_retry_base_backoff_ms = 1; - 399
cfg.tools = vec![Arc::new(Deliverer { - 400
delivers, - 401
..Default::default() - 402
})]; - 403
cfg.presentation_check = Some(Arc::new(|_text: &str, _offered: &[String]| { - 404
Some(vak_agent::PresentationNudge { - 405
tool: "make_draft".into(), - 406
text: "[presentation-check] show it as a card".into(), - 407
}) - 408
})); - 409
let mut script = VecDeque::from([msg( - 410
vec![ContentBlock::ToolUse { - 411
id: "c1".into(), - 412
name: "make_draft".into(), - 413
input: serde_json::json!({}), - 414
}], - 415
StopReason::ToolUse, - 416
)]); - 417
for _ in 0..3 { - 418
script.push_back(msg( - 419
vec![ContentBlock::text( - 420
"Added the slide; the draft is ready for review.", - 421
)], - 422
StopReason::EndTurn, - 423
)); - 424
} - 425
let requests = Arc::new(Mutex::new(Vec::new())); - 426
let mut agent = Agent::new( - 427
Arc::new(Scripted(Mutex::new(script), requests.clone())), - 428
log, - 429
cfg, - 430
); - 431
let (tx, mut rx) = mpsc::channel(256); - 432
tokio::spawn(async move { while rx.recv().await.is_some() {} }); - 433
let outcome = agent - 434
.run( - 435
"add a slide", - 436
&Default::default(), - 437
CancellationToken::new(), - 438
tx, - 439
) - 440
.await; - 441
assert!( - 442
matches!(outcome, TurnOutcome::Completed { .. }), - 443
"{outcome:?}" - 444
); - 445
requests.lock().unwrap().len() - 446
} - 447
- 448
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] - 449
async fn an_answer_after_a_delivered_draft_is_not_sent_back_to_become_a_card() { - 450
assert_eq!( - 451
requests_after_a_draft(true).await, - 452
2, - 453
"the tool call, then the answer, which ends the turn" - 454
); - 455
assert_eq!( - 456
requests_after_a_draft(false).await, - 457
3, - 458
"without a delivered file, the check still asks once for a card" - 459
); - 460
} - 461
- 462
fn tool_call(id: &str, name: &str, input: serde_json::Value) -> AssistantMessage { - 463
msg( - 464
vec![ContentBlock::ToolUse { - 465
id: id.into(), - 466
name: name.into(), - 467
input, - 468
}], - 469
StopReason::ToolUse, - 470
) - 471
} - 472
- 473
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] - 474
async fn a_repeated_draft_writes_nothing_and_a_preview_of_the_draft_is_not_shown() { - 475
let dir = tempfile::tempdir().unwrap(); - 476
let home = dir.path().join("home"); - 477
std::fs::create_dir_all(&home).unwrap(); - 478
let header = SessionHeader { - 479
agent: None, - 480
session_id: "repeat".into(), - 481
created_at: chrono::Utc::now(), - 482
cwd: dir.path().to_path_buf(), - 483
parent_session_id: None, - 484
contract_id: None, - 485
work_item_id: None, - 486
conversation: None, - 487
contract: FrozenContract { - 488
app_version: "0".into(), - 489
provider: "scripted".into(), - 490
model: "test-model".into(), - 491
route_ladder: Vec::new(), - 492
route_objective: String::new(), - 493
route_annotations: Vec::new(), - 494
system_prompt: "sys".into(), - 495
permission_mode: "workspace-write".into(), - 496
capabilities: Vec::new(), - 497
prompt_layers: Vec::new(), - 498
}, - 499
}; - 500
let log = SessionLog::create( - 501
SessionPath::new_session_file(&home, dir.path(), "repeat"), - 502
header, - 503
) - 504
.unwrap(); - 505
let drafts = Arc::new(Deliverer { - 506
delivers: true, - 507
..Default::default() - 508
}); - 509
let cards = Arc::new(PreviewCard::default()); - 510
let shell = Arc::new(Shell::default()); - 511
let mut cfg = AgentConfig::new("sys"); - 512
cfg.model = "test-model".into(); - 513
cfg.mode = Mode::FullAccess; - 514
cfg.retry_base_backoff_ms = 1; - 515
cfg.run_retry_base_backoff_ms = 1; - 516
cfg.tools = vec![drafts.clone(), cards.clone(), shell.clone()]; - 517
let slide = serde_json::json!({"path": "deck.pptx", "ops": [{"op": "add_slide_from_layout"}]}); - 518
let script = VecDeque::from([ - 519
tool_call("c1", "make_draft", slide.clone()), - 520
tool_call( - 521
"c2", - 522
"emit_ui_preview_card", - 523
serde_json::json!({"payload": {"artifact_path": "./deck.pptx", "html": "<h1>Next steps</h1>"}}), - 524
), - 525
tool_call( - 526
"c4", - 527
"bash", - 528
serde_json::json!({"command": "cp .vak/scratch/vak/c1/deck.pptx ./deck.pptx"}), - 529
), - 530
tool_call( - 531
"c5", - 532
"bash", - 533
serde_json::json!({"command": "ls .vak/scratch"}), - 534
), - 535
tool_call("c3", "make_draft", slide), - 536
msg( - 537
vec![ContentBlock::text("Added the Next steps slide.")], - 538
StopReason::EndTurn, - 539
), - 540
]); - 541
let requests = Arc::new(Mutex::new(Vec::new())); - 542
let mut agent = Agent::new( - 543
Arc::new(Scripted(Mutex::new(script), requests.clone())), - 544
log, - 545
cfg, - 546
); - 547
let (tx, mut rx) = mpsc::channel(256); - 548
tokio::spawn(async move { while rx.recv().await.is_some() {} }); - 549
let outcome = agent - 550
.run( - 551
"add a slide", - 552
&Default::default(), - 553
CancellationToken::new(), - 554
tx, - 555
) - 556
.await; - 557
assert!( - 558
matches!(outcome, TurnOutcome::Completed { .. }), - 559
"{outcome:?}" - 560
); - 561
assert_eq!(drafts.runs.load(Ordering::SeqCst), 1, "one draft written"); - 562
assert_eq!( - 563
cards.runs.load(Ordering::SeqCst), - 564
0, - 565
"the preview never ran" - 566
); - 567
let requests = requests.lock().unwrap(); - 568
let results: Vec<(String, String)> = requests - 569
.last() - 570
.unwrap() - 571
.messages - 572
.iter() - 573
.flat_map(|message| message.content.iter()) - 574
.filter_map(|block| match block { - 575
ContentBlock::ToolResult { - 576
tool_use_id, - 577
content, - 578
.. - 579
} => Some((tool_use_id.clone(), format!("{content:?}"))), - 580
_ => None, - 581
}) - 582
.collect(); - 583
let result = |id: &str| { - 584
results - 585
.iter() - 586
.find(|(call, _)| call == id) - 587
.map(|(_, text)| text.clone()) - 588
.unwrap_or_default() - 589
}; - 590
assert!(result("c2").contains("Not shown: deck.pptx"), "{results:?}"); - 591
assert!( - 592
result("c3").contains("Already drafted") && result("c3").contains("Draft for deck.pptx"), - 593
"{results:?}" - 594
); - 595
assert!( - 596
result("c4").contains("Not run: deck.pptx is a draft waiting for the person's review"), - 597
"copying the draft out of scratch is refused even in FullAccess: {results:?}" - 598
); - 599
assert_eq!( - 600
shell.runs.load(Ordering::SeqCst), - 601
1, - 602
"only the command that does not name the draft ran" - 603
); - 604
} - 605
Indexing the workspace…
Vakyartha documentation is discovering safe artifacts, anchors, and source references.