Vakyartha documentation

73/// Local-model profiles are cheap to re-probe (docs/design/68 §1: "the
74/// probe is free apart from time") and the weights behind a name can change
75/// under an operator's feet (an `ollama pull` swaps them), so the TTL is
76/// short.
77pub const LOCAL_PROFILE_TTL: Duration = Duration::from_secs(24 * 3600);
78
79/// Hosted profiles are slower/costlier to re-probe in full and the
80/// underlying model changes far less often once pinned to a version, so the
81/// TTL is long.
82pub const HOSTED_PROFILE_TTL: Duration = Duration::from_secs(7 * 24 * 3600);
83
84/// Identifies one measured capacity profile. Quantisation is part of the
85/// identity because two quantisations of the same model id can have very
86/// different real horizons despite sharing a declared window; it is `None`
87/// when the provider does not expose it.
88#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
89pub struct ProfileKey {
90 pub provider: String,
91 pub model: String,
92 pub quantisation: Option<String>,
93}
94
95/// Exponentially-weighted moving average, with the sample count needed to
96/// tell "never observed" apart from "observed and stable at zero".
97#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
98pub struct Ewma {
99 pub value: f64,
100 pub alpha: f64,
101 pub samples: u32,
102}
103
104impl Ewma {
105 pub fn new(alpha: f64) -> Self {
106 Ewma {
107 value: 0.0,
108 alpha,
109 samples: 0,
110 }
111 }
112
113 /// Folds one observation in. The first observation replaces the
114 /// (otherwise meaningless) zero seed outright rather than being
115 /// blended into it, so one real sample is fully trusted immediately.
116 pub fn observe(&mut self, sample: f64) {
117 if self.samples == 0 {
118 self.value = sample;
119 } else {
120 self.value = self.alpha * sample + (1.0 - self.alpha) * self.value;
121 }
122 self.samples = self.samples.saturating_add(1);
123 }
124}
125
126/// The largest prompt size at which the model is known to still follow an
127/// explicit instruction (§1). `confidence` and `last_confirmed` make it
128/// possible to tell a freshly probed horizon from an assumed or
129/// feedback-lowered one.
130#[derive(Debug, Clone, Serialize, Deserialize)]
131pub struct Horizon {
132 pub tokens: u64,
133 pub confidence: f64,
134 pub last_confirmed: SystemTime,
135}
136
137/// What is known about a provider's prefix-cache behaviour for this model
138/// (§1 "Cache rung").
139#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
140pub enum CacheBehaviour {
141 Unknown,
142 None,
143 PrefixStable,
144 ProviderReported,
145}
146
147/// One rung of the horizon ladder: a prompt size sent to the provider,
148/// whether the provider accepted the request at all, and — when accepted —
149/// whether the model followed the probe instruction embedded at that size.
150#[derive(Debug, Clone, Serialize, Deserialize)]
151pub struct Rung {
152 pub tokens: u64,
153 pub accepted: bool,
154 pub followed_instruction: Option<bool>,
155 pub prefill_ms: Option<u64>,
156}
157
158/// Audit trail for a `CapacityProfile`: when it was probed, which rungs
159/// were tried, any other signals that fed the decision, and a digest of the
160/// provider metadata it was built from (so a metadata change is detectable
161/// even inside the TTL).
162#[derive(Debug, Clone, Serialize, Deserialize)]
163pub struct ProbeProvenance {
164 pub probed_at: SystemTime,
165 pub rungs: Vec<Rung>,
166 pub signals: Vec<String>,
167 pub metadata_digest: String,
168 /// The quantisation label the profile was keyed under, so every later
169 /// feedback record lands on the same `ProfileKey` as the probe.
170 #[serde(default)]
171 pub quantisation: Option<String>,
172}
173
174/// A ledger-recorded, per-`(provider, model, quantisation)` measurement of
175/// what a model can actually do — the only input to context budgeting
176/// (docs/design/68-context-engine.md §1).
177#[derive(Debug, Clone, Serialize, Deserialize)]
178pub struct CapacityProfile {
179 /// Provider metadata (existing `model_context()`), before any probing.
180 pub declared_window: u64,
181 /// The largest prompt the provider has actually accepted.
182 pub verified_window: Option<u64>,
183 /// The largest prompt at which the model still followed a tool
184 /// instruction — the number budgeting actually uses.
185 pub instruction_horizon: Horizon,
186 /// `usage.prompt_tokens() / chars_sent`, fed by every turn's receipt —
187 /// the whole billed prompt (fresh + both cache tiers) over the whole
188 /// request the assembler actually sent (system + tools + messages), so
189 /// neither side is skewed by which bytes happened to be cache-served.
190 pub tokens_per_char: Ewma,
191 /// Input tokens per second of prefill, cache-miss turns only.
192 pub prefill_tps: Ewma,
193 pub cache: CacheBehaviour,
194 /// Provider max output, or the largest completion observed.
195 pub output_reserve: u64,
196 pub provenance: ProbeProvenance,
197 /// Set when feedback lowered the horizon (§1 "schedules a re-probe");
198 /// the next bind of this key re-runs the ladder instead of trusting the
199 /// lowered estimate indefinitely.
200 #[serde(default)]
201 pub needs_reprobe: bool,
202 /// EWMA of past current-turn sizes (docs/design/68 §4), fed at turn
203 /// close by `Agent::record_capacity_usage_feedback`. The planner uses it
204 /// — floored by the turn-so-far's actually measured size — to reserve
205 /// room for the open turn before budgeting history.
206 #[serde(default = "default_current_turn_reserve")]
207 pub current_turn_reserve: Ewma,
208}
209
210fn default_current_turn_reserve() -> Ewma {
211 Ewma::new(FEEDBACK_EWMA_ALPHA)
212}
213
214impl CapacityProfile {
215 /// A profile built from provider metadata alone, with no probe run —
216 /// the hosted-and-not-opted-in path (§1 "Cost control for hosted
217 /// models"): the horizon starts at the declared window with low
218 /// confidence and is tightened only by feedback or a later full probe.
219 pub fn from_metadata_only(
220 declared_window: u64,
221 output_reserve: u64,
222 metadata_digest: String,
223 probed_at: SystemTime,
224 ) -> Self {
225 CapacityProfile {
226 declared_window,
227 verified_window: None,
228 instruction_horizon: Horizon {
229 tokens: declared_window,
230 confidence: HOSTED_UNPROBED_CONFIDENCE,
231 last_confirmed: probed_at,
232 },
233 tokens_per_char: Ewma::new(FEEDBACK_EWMA_ALPHA),
234 prefill_tps: Ewma::new(FEEDBACK_EWMA_ALPHA),
235 cache: CacheBehaviour::Unknown,
236 output_reserve,
237 provenance: ProbeProvenance {
238 probed_at,
239 rungs: Vec::new(),
240 signals: vec!["metadata-only: hosted probing not opted in".into()],
241 metadata_digest,
242 quantisation: None,
243 },
244 needs_reprobe: false,
245 current_turn_reserve: Ewma::new(FEEDBACK_EWMA_ALPHA),
246 }
247 }
248
249 /// A profile built from a converged horizon ladder (§1 "Horizon
250 /// ladder").
251 pub fn from_probe(
252 declared_window: u64,
253 verified_window: Option<u64>,
254 horizon: Horizon,
255 cache: CacheBehaviour,
256 output_reserve: u64,
257 provenance: ProbeProvenance,
258 ) -> Self {
259 CapacityProfile {
260 declared_window,
261 verified_window,
262 instruction_horizon: horizon,
263 tokens_per_char: Ewma::new(FEEDBACK_EWMA_ALPHA),
264 prefill_tps: Ewma::new(FEEDBACK_EWMA_ALPHA),
265 cache,
266 output_reserve,
267 provenance,
268 needs_reprobe: false,
269 current_turn_reserve: Ewma::new(FEEDBACK_EWMA_ALPHA),
270 }
271 }
272
273 /// Remaining input budget for a request: the horizon minus the
274 /// measured stable prefix, the tail, the output reserve, and any
275 /// reserve already committed by the current turn. Saturates to zero
276 /// rather than underflowing — a caller over budget gets "nothing left",
277 /// never a wrapped huge number.
278 pub fn budget(&self, prefix_tokens: u64, tail_tokens: u64, current_turn_reserve: u64) -> u64 {
279 self.instruction_horizon
280 .tokens
281 .saturating_sub(prefix_tokens)
282 .saturating_sub(tail_tokens)
283 .saturating_sub(self.output_reserve)
284 .saturating_sub(current_turn_reserve)
285 }
286
287 /// Whether `tokens_per_char` reflects a real observation rather than
288 /// the uncalibrated bootstrap value.
289 pub fn is_calibrated(&self) -> bool {
290 self.tokens_per_char.samples > 0
291 }
292
293 /// Estimated token count for `chars`, using the measured tokens/char
294 /// once calibrated and the fixed bootstrap value only before that.
295 pub fn estimate_tokens(&self, chars: u64) -> u64 {
296 let tpc = if self.is_calibrated() {
297 self.tokens_per_char.value
298 } else {
299 UNCALIBRATED_TOKENS_PER_CHAR
300 };
301 (chars as f64 * tpc).round() as u64
302 }
303
304 /// Folds one turn's real usage into the profile (§1 "Feedback").
305 /// `cache_miss` gates the prefill measurement: prefill throughput is
306 /// only meaningful when the provider actually re-evaluated the prefix.
307 ///
308 /// `tokens_per_char` calibrates against `usage.prompt_tokens()` — every
309 /// prompt token the provider billed, cache tiers included — never
310 /// `usage.input_tokens` alone: `chars_sent` (the assembler's char count
311 /// for the WHOLE request) counts every byte sent whether or not it hit
312 /// the cache, so the numerator has to match. Using `input_tokens` alone
313 /// collapses the ratio toward zero as cache hits grow (a full cache hit
314 /// reports `input_tokens == 0` for a request that was not remotely
315 /// empty), which is exactly backwards: the calibration exists to relate
316 /// bytes sent to tokens billed, and prompt caching does not shrink
317 /// either one.
318 pub fn observe_usage(
319 &mut self,
320 chars_sent: u64,
321 usage: &vak_llm::Usage,
322 first_token_latency_ms: Option<u64>,
323 cache_miss: bool,
324 ) {
325 let prompt_tokens = usage.prompt_tokens();
326 if chars_sent > 0 && prompt_tokens > 0 {
327 self.tokens_per_char
328 .observe(prompt_tokens as f64 / chars_sent as f64);
329 }
330 if cache_miss && usage.input_tokens > 0 {
331 // Prefer the provider's own reported prefill latency (Ollama
332 // fills `usage.prefill_ms`) over the caller's wall-clock
333 // estimate, since it excludes model-load and network time.
334 if let Some(ms) = usage.prefill_ms.or(first_token_latency_ms)
335 && ms > 0
336 {
337 self.prefill_tps
338 .observe(usage.input_tokens as f64 / (ms as f64 / 1000.0));
339 }
340 }
341 }
342
343 /// Records that a request of `request_tokens` failed an explicit
344 /// instruction (required card not emitted, required tool not called, a
345 /// stop-policy block) the runtime already classifies. Only requests
346 /// near the current horizon count as evidence about the horizon itself
347 /// (§1 "Horizon tightening"); this never raises the horizon; a re-probe
348 /// is the only path that can.
349 pub fn observe_instruction_failure(&mut self, request_tokens: u64) {
350 let threshold = HORIZON_FAILURE_FRACTION * self.instruction_horizon.tokens as f64;
351 if (request_tokens as f64) < threshold {
352 return;
353 }
354 let lowered = self.instruction_horizon.tokens.min(request_tokens);
355 self.instruction_horizon = Horizon {
356 tokens: lowered,
357 confidence: HORIZON_FEEDBACK_CONFIDENCE,
358 last_confirmed: SystemTime::now(),
359 };
360 self.needs_reprobe = true;
361 }
362
363 /// Folds a provider's over-length rejection (`LlmError::Context`) into
364 /// the profile (docs/design/68-context-engine.md §5's over-length →
365 /// replan path): `verified_window` becomes this exact request size
366 /// (feedback, never a guess), and the horizon shrinks to
367 /// `request_tokens × OVER_LENGTH_SHRINK` with `HORIZON_FEEDBACK_CONFIDENCE`
368 /// — never widened, only ever lowered, like every other horizon
369 /// feedback path.
370 pub fn observe_over_length(&mut self, request_tokens: u64) {
371 self.verified_window = Some(match self.verified_window {
372 Some(previous) => previous.min(request_tokens),
373 None => request_tokens,
374 });
375 let shrunk = (request_tokens as f64 * OVER_LENGTH_SHRINK) as u64;
376 self.instruction_horizon = Horizon {
377 tokens: self.instruction_horizon.tokens.min(shrunk),
378 confidence: HORIZON_FEEDBACK_CONFIDENCE,
379 last_confirmed: SystemTime::now(),
380 };
381 self.needs_reprobe = true;
382 }
383
384 /// Folds one closed turn's total size into `current_turn_reserve` (§4),
385 /// so the planner's reserve for the NEXT open turn reflects how large
386 /// this model's turns actually tend to be.
387 pub fn observe_current_turn_tokens(&mut self, tokens: u64) {
388 self.current_turn_reserve.observe(tokens as f64);
389 }
390
391 /// The reserve the planner sets aside for the still-open turn (§4):
392 /// the EWMA of past current-turn sizes, floored by `measured_so_far` —
393 /// the open turn's own measured size can never be estimated as less
394 /// than what it has already spent.
395 pub fn current_turn_reserve(&self, measured_so_far: u64) -> u64 {
396 if self.current_turn_reserve.samples == 0 {
397 return measured_so_far;
398 }
399 (self.current_turn_reserve.value.round() as u64).max(measured_so_far)
400 }
401
402 /// Whether this profile should be re-probed before being trusted again:
403 /// past its TTL (24h local / 7d hosted), or contradicted metadata.
404 /// `current_metadata_digest` is compared by the caller because only it
405 /// knows the freshly fetched metadata; passing `None` skips that check.
406 pub fn is_stale(&self, now: SystemTime, local: bool) -> bool {
407 let ttl = if local {
408 LOCAL_PROFILE_TTL
409 } else {
410 HOSTED_PROFILE_TTL
411 };
412 match now.duration_since(self.provenance.probed_at) {
413 Ok(elapsed) => elapsed > ttl,
414 // Clock moved backwards since the probe: treat as fresh rather
415 // than fabricating staleness from an impossible duration.
416 Err(_) => false,
417 }
418 }
419}
420
421/// The horizon-ladder state machine (§1 "Horizon ladder"). Pure: the caller
422/// drives it by asking `next_rung()`, sending exactly that request, and
423/// folding the outcome back with `report()`. No network call happens here.
424#[derive(Debug, Clone)]
425pub struct Ladder {
426 rungs: Vec<u64>,
427 next_index: usize,
428 last_pass: Option<u64>,
429 first_fail: Option<u64>,
430 searching: bool,
431 result: Option<Horizon>,
432 verified_window: Option<u64>,
433}
434
435impl Ladder {
436 /// Builds the rung sequence: the named starting sizes filtered to (and
437 /// extended geometrically past, by doubling) `declared_window * 0.9`.
438 pub fn new(declared_window: u64) -> Self {
439 let cap = (declared_window as f64 * LADDER_MAX_FRACTION) as u64;
440 let mut rungs: Vec<u64> = LADDER_START_RUNGS
441 .into_iter()
442 .filter(|&r| r <= cap)
443 .collect();
444 let mut next = LADDER_START_RUNGS
445 .last()
446 .copied()
447 .unwrap_or(4_000)
448 .saturating_mul(2);
449 while next <= cap && next > 0 {
450 rungs.push(next);
451 next = next.saturating_mul(2);
452 }
453 if rungs.is_empty() && cap > 0 {
454 rungs.push(cap);
455 }
456 Ladder {
457 rungs,
458 next_index: 0,
459 last_pass: None,
460 first_fail: None,
461 searching: false,
462 result: None,
463 verified_window: None,
464 }
465 }
466
467 /// The next prompt size to probe, or `None` once the ladder has
468 /// converged on a horizon.
469 pub fn next_rung(&self) -> Option<u64> {
470 if self.result.is_some() {
471 return None;
472 }
473 if self.searching {
474 let lo = self.last_pass.unwrap_or(0);
475 let hi = self.first_fail?;
476 if hi <= lo {
477 return None;
478 }
479 let mid = lo + (hi - lo) / 2;
480 if mid <= lo { None } else { Some(mid) }
481 } else {
482 self.rungs.get(self.next_index).copied()
483 }
484 }
485
486 /// Folds one rung's outcome back in. `accepted` is false for a
487 /// provider-level rejection (400/413 style context error); `followed`
488 /// is meaningless when `accepted` is false.
489 pub fn report(&mut self, tokens: u64, accepted: bool, followed: bool) {
490 if !accepted {
491 self.verified_window = Some(match self.verified_window {
492 Some(v) => v.min(tokens.saturating_sub(1)),
493 None => tokens.saturating_sub(1),
494 });
495 self.first_fail = Some(match self.first_fail {
496 Some(f) => f.min(tokens),
497 None => tokens,
498 });
499 self.searching = true;
500 } else if followed {
501 self.last_pass = Some(match self.last_pass {
502 Some(p) => p.max(tokens),
503 None => tokens,
504 });
505 if !self.searching {
506 self.next_index += 1;
507 }
508 } else {
509 // Accepted but the model did not follow the instruction: this
510 // rung IS the instruction-horizon boundary, same as a rejection
511 // for search purposes.
512 self.first_fail = Some(match self.first_fail {
513 Some(f) => f.min(tokens),
514 None => tokens,
515 });
516 self.searching = true;
517 }
518 self.converge();
519 }
520
521 fn converge(&mut self) {
522 if self.result.is_some() {
523 return;
524 }
525 if let (Some(lo), Some(hi)) = (self.last_pass, self.first_fail) {
526 if hi <= lo {
527 self.result = Some(self.horizon_at(lo));
528 return;
529 }
530 let gap = (hi - lo) as f64 / hi as f64;
531 if gap < LADDER_SEARCH_STOP_FRACTION {
532 self.result = Some(self.horizon_at(lo));
533 }
534 } else if !self.searching && self.next_index >= self.rungs.len() {
535 // Every named rung passed with no failure ever observed: the
536 // horizon is at least the last rung tried.
537 if let Some(lo) = self.last_pass {
538 self.result = Some(self.horizon_at(lo));
539 }
540 }
541 }
542
543 fn horizon_at(&self, tokens: u64) -> Horizon {
544 Horizon {
545 tokens,
546 confidence: PROBED_CONFIDENCE,
547 last_confirmed: SystemTime::now(),
548 }
549 }
550
551 /// The converged horizon, once `next_rung()` has returned `None`.
552 pub fn result(&self) -> Option<Horizon> {
553 self.result.clone()
554 }
555
556 /// The largest prompt size the provider is known to have accepted at
557 /// all (independent of instruction-following), when a rejection was
558 /// observed.
559 pub fn verified_window(&self) -> Option<u64> {
560 self.verified_window
561 }
562}
563
564/// Identical requests sent per ladder rung; the rung's verdict is the
565/// majority. A sampling model answers the same prompt differently run to
566/// run, so a single completion cannot decide whether an instruction was
567/// followed at that size. Three is the smallest odd count with a majority.
568pub const PROBE_SAMPLES_PER_RUNG: u32 = 3;
569
570/// Builds a probe request of roughly `tokens_target` tokens shaped like the
571/// history the model will really see (docs/design/68 §1): each filler turn
572/// is a user question, an assistant `lookup` tool call, a digest-shaped
573/// tool result carrying an `[evidence:…]` tag, and a short assistant
574/// answer — the same call pattern a closed turn projects at `Full`. Inert
575/// prose would measure a horizon the model never reaches on real work.
576/// The content is varied and non-repeating so no adjacent-message
577/// prefix-cache shortcut masks the prefill cost. The request ends with an
578/// instruction to call `probe_ack`. `tokens_per_char` should come from the
579/// profile being probed when calibrated, so rung sizes land close to their
580/// target on real providers.
581pub fn probe_request(
582 tokens_target: u64,
583 tokens_per_char: f64,
584 model: &str,
585) -> vak_llm::ChatRequest {
586 let tpc = if tokens_per_char > 0.0 {
587 tokens_per_char
588 } else {
589 UNCALIBRATED_TOKENS_PER_CHAR
590 };
591 let chars_target = (tokens_target as f64 / tpc) as u64;
592 let mut messages = Vec::new();
593 let mut chars_written: u64 = 0;
594 let mut seed: u64 = 0;
595 while chars_written < chars_target {
596 for message in probe_filler_turn(seed) {
597 chars_written += message.text_content().len() as u64
598 + message
599 .content
600 .iter()
601 .map(|block| match block {
602 vak_llm::ContentBlock::ToolUse { input, .. } => input.to_string().len(),
603 vak_llm::ContentBlock::ToolResult { content, .. } => content.len(),
604 _ => 0,
605 } as u64)
606 .sum::<u64>();
607 messages.push(message);
608 }
609 seed += 1;
610 }
611 messages.push(vak_llm::Message::user_text(
612 "Call the probe_ack tool now with {\"ok\": true} as its only argument. \
613 Do not answer in prose and do not call any other tool.",
614 ));
615 let mut request = vak_llm::ChatRequest::new(model);
616 request.messages = messages;
617 request.tools = vec![probe_lookup_tool(), vak_llm::ToolDefinition::probe_ack()];
618 request.max_tokens = 32;
619 request
620}
621
622/// The stand-in retrieval tool the filler turns "called"; defined on the
623/// request so every provider accepts the replayed pairs.
624fn probe_lookup_tool() -> vak_llm::ToolDefinition {
625 vak_llm::ToolDefinition::new(
626 "lookup",
627 "Look a topic up and return matching records.",
628 serde_json::json!({
629 "type": "object",
630 "properties": { "query": { "type": "string" } },
631 "required": ["query"],
632 }),
633 )
634}
635
636/// One filler turn — four messages in the shape of a projected closed turn.
637/// Seeded by turn index so a probe run is reproducible and every turn
638/// differs.
639fn probe_filler_turn(seed: u64) -> Vec<vak_llm::Message> {
640 let call_id = format!("probe-call-{seed}");
641 let topic = probe_filler_text(seed, 6);
642 let question = format!("What did the {topic} report say in section {}?", seed + 1);
643 let result = format!(
644 "[{{\"title\":\"{}\",\"url\":\"https://example.invalid/{seed}\"}},\
645 {{\"title\":\"{}\",\"url\":\"https://example.invalid/{seed}-b\"}}]\n\
646 [evidence:{call_id} \u{2014} {} chars; call recall to expand]",
647 probe_filler_text(seed.wrapping_add(101), 5),
648 probe_filler_text(seed.wrapping_add(202), 5),
649 900 + seed * 7
650 );
651 let answer = format!(
652 "Section {} of the {topic} report covers {}.",
653 seed + 1,
654 probe_filler_text(seed.wrapping_add(303), 30)
655 );
656 vec![
657 vak_llm::Message::user_text(question),
658 vak_llm::Message::assistant(vec![vak_llm::ContentBlock::ToolUse {
659 id: call_id.clone(),
660 name: "lookup".into(),
661 input: serde_json::json!({ "query": topic }),
662 }]),
663 vak_llm::Message {
664 role: vak_llm::Role::User,
665 content: vec![vak_llm::ContentBlock::tool_result(call_id, result)],
666 },
667 vak_llm::Message::assistant(vec![vak_llm::ContentBlock::text(answer)]),
668 ]
669}
670
671/// Deterministic, non-repeating filler words. Seeded so a probe run is
672/// reproducible, and varied so the provider cannot shortcut prefill via a
673/// repeated-content optimisation.
674fn probe_filler_text(seed: u64, words: u64) -> String {
675 const WORDS: [&str; 16] = [
676 "ridge",
677 "cobalt",
678 "ferry",
679 "lantern",
680 "quartz",
681 "meridian",
682 "otter",
683 "glacier",
684 "ember",
685 "thicket",
686 "vellum",
687 "cinder",
688 "harbor",
689 "tundra",
690 "opal",
691 "switchback",
692 ];
693 let mut s = String::new();
694 for j in 0..words {
695 let idx = ((seed.wrapping_mul(31).wrapping_add(j.wrapping_mul(17))) as usize) % WORDS.len();
696 if j > 0 {
697 s.push(' ');
698 }
699 s.push_str(WORDS[idx]);
700 }
701 s
702}
703
704/// Two identical requests are sent back to back at the same size the
705/// probe already used (§1 "Cache rung"); this classifies what the pair
706/// showed. `ProviderReported` when the second response's usage shows a
707/// cache hit (`cache_read_input_tokens`, which already folds in the OpenAI
708/// `prompt_tokens_details.cached_tokens` shape — see the adapters in
709/// vak-llm). Otherwise `PrefixStable` when the provider reports nothing but
710/// the second request's first-token latency dropped to a fifth or less of
711/// the first's — a local runner's prefix cache shows up only as timing.
712/// `None` (measured, no caching detected) when neither signal fired; the
713/// caller records both raw latencies as provenance signals regardless.
714pub fn classify_cache_rung(
715 first_latency_ms: u64,
716 second_latency_ms: u64,
717 second_usage: &vak_llm::Usage,
718) -> CacheBehaviour {
719 if second_usage.cache_read_input_tokens.unwrap_or(0) > 0 {
720 return CacheBehaviour::ProviderReported;
721 }
722 if first_latency_ms > 0 && second_latency_ms.saturating_mul(5) <= first_latency_ms {
723 return CacheBehaviour::PrefixStable;
724 }
725 CacheBehaviour::None
726}
727
728/// Whether a probe response followed the instruction: a `ToolUse` block
729/// named `probe_ack` anywhere in the response.
730pub fn followed(response: &vak_llm::AssistantMessage) -> bool {
731 response
732 .content
733 .iter()
734 .any(|b| matches!(b, vak_llm::ContentBlock::ToolUse { name, .. } if name == "probe_ack"))
735}
736
737#[cfg(test)]
738#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
739mod tests {
740 use super::*;
741
742 #[test]
743 fn budget_saturates_instead_of_underflowing() {
744 let profile = flat_profile_with_reserve(1_000, 0);
745 assert_eq!(profile.budget(200, 100, 0), 700);
746 // prefix + tail + reserve exceed the horizon: saturates to 0, never wraps.
747 assert_eq!(profile.budget(900, 200, 0), 0);
748 }
749
750 #[test]
751 fn ewma_first_sample_replaces_zero_seed() {
752 let mut ewma = Ewma::new(0.3);
753 ewma.observe(10.0);
754 assert_eq!(ewma.value, 10.0);
755 assert_eq!(ewma.samples, 1);
756 ewma.observe(20.0);
757 assert!((ewma.value - (0.3 * 20.0 + 0.7 * 10.0)).abs() < 1e-9);
758 assert_eq!(ewma.samples, 2);
759 }
760
761 #[test]
762 fn estimate_tokens_uses_bootstrap_until_calibrated() {
763 let profile = flat_profile(10_000);
764 assert_eq!(profile.estimate_tokens(400), 100); // 0.25 tokens/char
765 let mut calibrated = profile;
766 calibrated.tokens_per_char.observe(0.5);
767 assert_eq!(calibrated.estimate_tokens(400), 200);
768 }
769
770 #[test]
771 fn observe_usage_updates_tokens_per_char_and_prefill_on_cache_miss_only() {
772 let mut profile = flat_profile(10_000);
773 let usage = vak_llm::Usage {
774 input_tokens: 1_000,
775 output_tokens: 10,
776 prefill_ms: Some(2_000),
777 ..Default::default()
778 };
779 profile.observe_usage(2_000, &usage, None, true);
780 assert_eq!(profile.tokens_per_char.samples, 1);
781 assert!((profile.tokens_per_char.value - 0.5).abs() < 1e-9);
782 assert_eq!(profile.prefill_tps.samples, 1);
783 assert!((profile.prefill_tps.value - 500.0).abs() < 1e-9); // 1000 tok / 2s
784
785 let mut cache_hit_profile = flat_profile(10_000);
786 cache_hit_profile.observe_usage(2_000, &usage, None, false);
787 assert_eq!(cache_hit_profile.prefill_tps.samples, 0);
788 }
789
790 /// Item 2 fix: on a full cache hit `input_tokens` reports 0 for a
791 /// request that was not remotely empty. Calibrating against
792 /// `input_tokens` alone would either skip the observation (guard fails)
793 /// or, on a partial hit, silently drag `tokens_per_char` toward zero
794 /// over repeated turns. `prompt_tokens()` (fresh + both cache tiers)
795 /// must be what tokens_per_char calibrates against.
796 #[test]
797 fn observe_usage_calibrates_against_prompt_tokens_not_input_tokens_alone() {
798 let mut profile = flat_profile(10_000);
799 let full_cache_hit = vak_llm::Usage {
800 input_tokens: 0,
801 output_tokens: 5,
802 cache_read_input_tokens: Some(8_000),
803 ..Default::default()
804 };
805 // cache_miss=false: this IS a cache hit, but the calibration must
806 // still run — cache_miss only gates prefill_tps, never tokens/char.
807 profile.observe_usage(4_000, &full_cache_hit, None, false);
808 assert_eq!(profile.tokens_per_char.samples, 1);
809 assert!((profile.tokens_per_char.value - 2.0).abs() < 1e-9); // 8000/4000
810 }
811
812 /// A steady stream of partial cache hits must not drag the ratio toward
813 /// zero the way calibrating on `input_tokens` alone would (§1
814 /// "Feedback" bug report: repeated cache hits collapsed the EWMA).
815 #[test]
816 fn repeated_partial_cache_hits_do_not_collapse_tokens_per_char() {
817 let mut profile = flat_profile(10_000);
818 let partial_hit = vak_llm::Usage {
819 input_tokens: 50, // small fresh remainder after the prefix cache
820 output_tokens: 10,
821 cache_read_input_tokens: Some(4_950),
822 ..Default::default()
823 };
824 for _ in 0..5 {
825 profile.observe_usage(5_000, &partial_hit, None, false);
826 }
827 // 5000 prompt tokens / 5000 chars == 1.0 every time; five identical
828 // observations must leave the EWMA at 1.0, not collapse toward the
829 // 50/5000 == 0.01 that `input_tokens` alone would calibrate.
830 assert!((profile.tokens_per_char.value - 1.0).abs() < 1e-9);
831 }
832
833 #[test]
834 fn horizon_never_widens_on_feedback() {
835 let mut profile = flat_profile(10_000);
836 profile.observe_instruction_failure(9_000); // >= 0.8 * 10_000
837 assert_eq!(profile.instruction_horizon.tokens, 9_000);
838 assert!(profile.needs_reprobe);
839 assert_eq!(profile.instruction_horizon.confidence, 0.6);
840
841 // A later "failure" reported at a larger size than the (already
842 // lowered) horizon must not raise it back up.
843 profile.observe_instruction_failure(9_500);
844 assert_eq!(profile.instruction_horizon.tokens, 9_000);
845 }
846
847 #[test]
848 fn observe_instruction_failure_ignores_requests_far_below_horizon() {
849 let mut profile = flat_profile(10_000);
850 profile.observe_instruction_failure(1_000); // well under 0.8 * horizon
851 assert_eq!(profile.instruction_horizon.tokens, 10_000);
852 assert!(!profile.needs_reprobe);
853 }
854
855 #[test]
856 fn over_length_feedback_shrinks_the_horizon_and_never_widens_it() {
857 let mut profile = flat_profile(10_000);
858 profile.observe_over_length(8_000);
859 assert_eq!(profile.verified_window, Some(8_000));
860 assert_eq!(profile.instruction_horizon.tokens, 7_200); // 8_000 * 0.9
861 assert_eq!(profile.instruction_horizon.confidence, 0.6);
862 assert!(profile.needs_reprobe);
863
864 // A later, larger rejected size must not widen the already-lowered
865 // horizon or verified_window back up.
866 profile.observe_over_length(9_000);
867 assert_eq!(profile.verified_window, Some(8_000));
868 assert_eq!(profile.instruction_horizon.tokens, 7_200);
869 }
870
871 #[test]
872 fn current_turn_reserve_floors_at_the_measured_size_before_and_after_samples() {
873 let mut profile = flat_profile(10_000);
874 // No samples yet: the reserve is exactly the measured-so-far floor.
875 assert_eq!(profile.current_turn_reserve(500), 500);
876 profile.observe_current_turn_tokens(200);
877 // One low sample must not undercut a larger turn-so-far measurement.
878 assert_eq!(profile.current_turn_reserve(500), 500);
879 profile.observe_current_turn_tokens(2_000);
880 // EWMA now exceeds a smaller measured-so-far value and wins.
881 assert!(profile.current_turn_reserve(10) > 10);
882 }
883
884 #[test]
885 fn is_stale_respects_local_vs_hosted_ttl() {
886 let profile = flat_profile(10_000);
887 let just_over_a_day = profile.provenance.probed_at + Duration::from_secs(25 * 3600);
888 assert!(profile.is_stale(just_over_a_day, true));
889 assert!(!profile.is_stale(just_over_a_day, false));
890 }
891
892 #[test]
893 fn ladder_converges_via_binary_search_within_tolerance() {
894 // Real horizon is 20_000; every rung <= 20_000 passes, every rung
895 // above fails the instruction.
896 let mut ladder = Ladder::new(200_000);
897 let real_horizon = 20_000u64;
898 let mut iterations = 0;
899 while let Some(rung) = ladder.next_rung() {
900 iterations += 1;
901 assert!(iterations < 100, "ladder did not converge");
902 let followed = rung <= real_horizon;
903 ladder.report(rung, true, followed);
904 }
905 let horizon = ladder.result().expect("ladder should converge");
906 assert!(horizon.tokens <= real_horizon);
907 // within the 25% search-stop tolerance of the true horizon
908 assert!(horizon.tokens as f64 >= real_horizon as f64 * 0.75);
909 }
910
911 #[test]
912 fn ladder_records_verified_window_on_provider_rejection() {
913 let mut ladder = Ladder::new(200_000);
914 // First rung is accepted and followed.
915 let first = ladder.next_rung().expect("first rung");
916 ladder.report(first, true, true);
917 // Next rung is rejected outright by the provider.
918 let second = ladder.next_rung().expect("second rung");
919 ladder.report(second, false, false);
920 assert_eq!(ladder.verified_window(), Some(second - 1));
921 }
922
923 #[test]
924 fn probe_request_is_shaped_like_projected_turns_and_ends_with_probe_ack() {
925 let req = probe_request(1_000, 0.25, "test-model");
926 assert_eq!(req.max_tokens, 32);
927 let names: Vec<&str> = req.tools.iter().map(|t| t.name.as_str()).collect();
928 assert_eq!(names, vec!["lookup", "probe_ack"]);
929 assert!(req.cache.is_none());
930 let last = req
931 .messages
932 .last()
933 .expect("at least the instruction message");
934 assert!(last.text_content().contains("probe_ack"));
935 // Each filler turn is user question → assistant lookup call →
936 // digest-shaped result with an evidence tag → assistant answer.
937 assert_eq!(req.messages[0].role, vak_llm::Role::User);
938 assert!(matches!(
939 &req.messages[1].content[0],
940 vak_llm::ContentBlock::ToolUse { name, .. } if name == "lookup"
941 ));
942 assert!(matches!(
943 &req.messages[2].content[0],
944 vak_llm::ContentBlock::ToolResult { content, .. } if content.contains("[evidence:")
945 ));
946 assert_eq!(req.messages[3].role, vak_llm::Role::Assistant);
947 // Every filler turn differs (no prefix-cache shortcut inside a rung).
948 assert_ne!(
949 req.messages[0].text_content(),
950 req.messages[4].text_content()
951 );
952 // Roughly sized: within a generous tolerance of the token target
953 // translated through the given tokens/char.
954 let total_chars: usize = req
955 .messages
956 .iter()
957 .flat_map(|m| m.content.iter())
958 .map(|b| match b {
959 vak_llm::ContentBlock::Text { text } => text.len(),
960 vak_llm::ContentBlock::ToolUse { input, .. } => input.to_string().len(),
961 vak_llm::ContentBlock::ToolResult { content, .. } => content.len(),
962 _ => 0,
963 })
964 .sum();
965 assert!(total_chars as f64 >= 1_000.0 / 0.25 * 0.5);
966 assert!(total_chars as f64 <= 1_000.0 / 0.25 * 2.0);
967 }
968
969 #[test]
970 fn cache_rung_prefers_provider_reported_cache_hit() {
971 let usage = vak_llm::Usage {
972 cache_read_input_tokens: Some(3_000),
973 ..Default::default()
974 };
975 // Even with no latency improvement at all, a real cache hit wins.
976 assert_eq!(
977 classify_cache_rung(1_000, 1_000, &usage),
978 CacheBehaviour::ProviderReported
979 );
980 }
981
982 #[test]
983 fn cache_rung_falls_back_to_five_x_latency_when_unreported() {
984 let usage = vak_llm::Usage::default();
985 // Exactly 5x faster: still counts (<=), not strictly less-than.
986 assert_eq!(
987 classify_cache_rung(1_000, 200, &usage),
988 CacheBehaviour::PrefixStable
989 );
990 assert_eq!(
991 classify_cache_rung(1_000, 201, &usage),
992 CacheBehaviour::None
993 );
994 }
995
996 #[test]
997 fn cache_rung_is_none_with_no_signal_at_all() {
998 let usage = vak_llm::Usage::default();
999 assert_eq!(classify_cache_rung(500, 480, &usage), CacheBehaviour::None);
1000 // A zero first latency can't establish a ratio; never fabricate a hit.
1001 assert_eq!(classify_cache_rung(0, 0, &usage), CacheBehaviour::None);
1002 }
1003
1004 #[test]
1005 fn followed_detects_probe_ack_tool_use_only() {
1006 let with_ack = vak_llm::AssistantMessage {
1007 content: vec![vak_llm::ContentBlock::ToolUse {
1008 id: "1".into(),
1009 name: "probe_ack".into(),
1010 input: serde_json::json!({"ok": true}),
1011 }],
1012 stop_reason: vak_llm::StopReason::ToolUse,
1013 usage: vak_llm::Usage::default(),
1014 model: "m".into(),
1015 response_id: None,
1016 };
1017 assert!(followed(&with_ack));
1018
1019 let prose_only = vak_llm::AssistantMessage {
1020 content: vec![vak_llm::ContentBlock::text("sure, here you go")],
1021 stop_reason: vak_llm::StopReason::EndTurn,
1022 usage: vak_llm::Usage::default(),
1023 model: "m".into(),
1024 response_id: None,
1025 };
1026 assert!(!followed(&prose_only));
1027 }
1028
1029 fn flat_profile(horizon_tokens: u64) -> CapacityProfile {
1030 flat_profile_with_reserve(horizon_tokens, 1_024)
1031 }
1032
1033 fn flat_profile_with_reserve(horizon_tokens: u64, output_reserve: u64) -> CapacityProfile {
1034 CapacityProfile::from_probe(
1035 horizon_tokens,
1036 None,
1037 Horizon {
1038 tokens: horizon_tokens,
1039 confidence: PROBED_CONFIDENCE,
1040 last_confirmed: SystemTime::now(),
1041 },
1042 CacheBehaviour::Unknown,
1043 output_reserve,
1044 ProbeProvenance {
1045 probed_at: SystemTime::now(),
1046 rungs: Vec::new(),
1047 signals: Vec::new(),
1048 metadata_digest: "digest".into(),
1049 quantisation: None,
1050 },
1051 )
1052 }
1053}
1054

73/// Local-model profiles are cheap to re-probe (docs/design/68 §1: "the
74/// probe is free apart from time") and the weights behind a name can change
75/// under an operator's feet (an `ollama pull` swaps them), so the TTL is
76/// short.
77pub const LOCAL_PROFILE_TTL: Duration = Duration::from_secs(24 * 3600);
78
79/// Hosted profiles are slower/costlier to re-probe in full and the
80/// underlying model changes far less often once pinned to a version, so the
81/// TTL is long.
82pub const HOSTED_PROFILE_TTL: Duration = Duration::from_secs(7 * 24 * 3600);
83
84/// Identifies one measured capacity profile. Quantisation is part of the
85/// identity because two quantisations of the same model id can have very
86/// different real horizons despite sharing a declared window; it is `None`
87/// when the provider does not expose it.
88#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
89pub struct ProfileKey {
90 pub provider: String,
91 pub model: String,
92 pub quantisation: Option<String>,
93}
94
95/// Exponentially-weighted moving average, with the sample count needed to
96/// tell "never observed" apart from "observed and stable at zero".
97#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
98pub struct Ewma {
99 pub value: f64,
100 pub alpha: f64,
101 pub samples: u32,
102}
103
104impl Ewma {
105 pub fn new(alpha: f64) -> Self {
106 Ewma {
107 value: 0.0,
108 alpha,
109 samples: 0,
110 }
111 }
112
113 /// Folds one observation in. The first observation replaces the
114 /// (otherwise meaningless) zero seed outright rather than being
115 /// blended into it, so one real sample is fully trusted immediately.
116 pub fn observe(&mut self, sample: f64) {
117 if self.samples == 0 {
118 self.value = sample;
119 } else {
120 self.value = self.alpha * sample + (1.0 - self.alpha) * self.value;
121 }
122 self.samples = self.samples.saturating_add(1);
123 }
124}
125
126/// The largest prompt size at which the model is known to still follow an
127/// explicit instruction (§1). `confidence` and `last_confirmed` make it
128/// possible to tell a freshly probed horizon from an assumed or
129/// feedback-lowered one.
130#[derive(Debug, Clone, Serialize, Deserialize)]
131pub struct Horizon {
132 pub tokens: u64,
133 pub confidence: f64,
134 pub last_confirmed: SystemTime,
135}
136
137/// What is known about a provider's prefix-cache behaviour for this model
138/// (§1 "Cache rung").
139#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
140pub enum CacheBehaviour {
141 Unknown,
142 None,
143 PrefixStable,
144 ProviderReported,
145}
146
147/// One rung of the horizon ladder: a prompt size sent to the provider,
148/// whether the provider accepted the request at all, and — when accepted —
149/// whether the model followed the probe instruction embedded at that size.
150#[derive(Debug, Clone, Serialize, Deserialize)]
151pub struct Rung {
152 pub tokens: u64,
153 pub accepted: bool,
154 pub followed_instruction: Option<bool>,
155 pub prefill_ms: Option<u64>,
156}
157
158/// Audit trail for a `CapacityProfile`: when it was probed, which rungs
159/// were tried, any other signals that fed the decision, and a digest of the
160/// provider metadata it was built from (so a metadata change is detectable
161/// even inside the TTL).
162#[derive(Debug, Clone, Serialize, Deserialize)]
163pub struct ProbeProvenance {
164 pub probed_at: SystemTime,
165 pub rungs: Vec<Rung>,
166 pub signals: Vec<String>,
167 pub metadata_digest: String,
168 /// The quantisation label the profile was keyed under, so every later
169 /// feedback record lands on the same `ProfileKey` as the probe.
170 #[serde(default)]
171 pub quantisation: Option<String>,
172}
173
174/// A ledger-recorded, per-`(provider, model, quantisation)` measurement of
175/// what a model can actually do — the only input to context budgeting
176/// (docs/design/68-context-engine.md §1).
177#[derive(Debug, Clone, Serialize, Deserialize)]
178pub struct CapacityProfile {
179 /// Provider metadata (existing `model_context()`), before any probing.
180 pub declared_window: u64,
181 /// The largest prompt the provider has actually accepted.
182 pub verified_window: Option<u64>,
183 /// The largest prompt at which the model still followed a tool
184 /// instruction — the number budgeting actually uses.
185 pub instruction_horizon: Horizon,
186 /// `usage.prompt_tokens() / chars_sent`, fed by every turn's receipt —
187 /// the whole billed prompt (fresh + both cache tiers) over the whole
188 /// request the assembler actually sent (system + tools + messages), so
189 /// neither side is skewed by which bytes happened to be cache-served.
190 pub tokens_per_char: Ewma,
191 /// Input tokens per second of prefill, cache-miss turns only.
192 pub prefill_tps: Ewma,
193 pub cache: CacheBehaviour,
194 /// Provider max output, or the largest completion observed.
195 pub output_reserve: u64,
196 pub provenance: ProbeProvenance,
197 /// Set when feedback lowered the horizon (§1 "schedules a re-probe");
198 /// the next bind of this key re-runs the ladder instead of trusting the
199 /// lowered estimate indefinitely.
200 #[serde(default)]
201 pub needs_reprobe: bool,
202 /// EWMA of past current-turn sizes (docs/design/68 §4), fed at turn
203 /// close by `Agent::record_capacity_usage_feedback`. The planner uses it
204 /// — floored by the turn-so-far's actually measured size — to reserve
205 /// room for the open turn before budgeting history.
206 #[serde(default = "default_current_turn_reserve")]
207 pub current_turn_reserve: Ewma,
208}
209
210fn default_current_turn_reserve() -> Ewma {
211 Ewma::new(FEEDBACK_EWMA_ALPHA)
212}
213
214impl CapacityProfile {
215 /// A profile built from provider metadata alone, with no probe run —
216 /// the hosted-and-not-opted-in path (§1 "Cost control for hosted
217 /// models"): the horizon starts at the declared window with low
218 /// confidence and is tightened only by feedback or a later full probe.
219 pub fn from_metadata_only(
220 declared_window: u64,
221 output_reserve: u64,
222 metadata_digest: String,
223 probed_at: SystemTime,
224 ) -> Self {
225 CapacityProfile {
226 declared_window,
227 verified_window: None,
228 instruction_horizon: Horizon {
229 tokens: declared_window,
230 confidence: HOSTED_UNPROBED_CONFIDENCE,
231 last_confirmed: probed_at,
232 },
233 tokens_per_char: Ewma::new(FEEDBACK_EWMA_ALPHA),
234 prefill_tps: Ewma::new(FEEDBACK_EWMA_ALPHA),
235 cache: CacheBehaviour::Unknown,
236 output_reserve,
237 provenance: ProbeProvenance {
238 probed_at,
239 rungs: Vec::new(),
240 signals: vec!["metadata-only: hosted probing not opted in".into()],
241 metadata_digest,
242 quantisation: None,
243 },
244 needs_reprobe: false,
245 current_turn_reserve: Ewma::new(FEEDBACK_EWMA_ALPHA),
246 }
247 }
248
249 /// A profile built from a converged horizon ladder (§1 "Horizon
250 /// ladder").
251 pub fn from_probe(
252 declared_window: u64,
253 verified_window: Option<u64>,
254 horizon: Horizon,
255 cache: CacheBehaviour,
256 output_reserve: u64,
257 provenance: ProbeProvenance,
258 ) -> Self {
259 CapacityProfile {
260 declared_window,
261 verified_window,
262 instruction_horizon: horizon,
263 tokens_per_char: Ewma::new(FEEDBACK_EWMA_ALPHA),
264 prefill_tps: Ewma::new(FEEDBACK_EWMA_ALPHA),
265 cache,
266 output_reserve,
267 provenance,
268 needs_reprobe: false,
269 current_turn_reserve: Ewma::new(FEEDBACK_EWMA_ALPHA),
270 }
271 }
272
273 /// Remaining input budget for a request: the horizon minus the
274 /// measured stable prefix, the tail, the output reserve, and any
275 /// reserve already committed by the current turn. Saturates to zero
276 /// rather than underflowing — a caller over budget gets "nothing left",
277 /// never a wrapped huge number.
278 pub fn budget(&self, prefix_tokens: u64, tail_tokens: u64, current_turn_reserve: u64) -> u64 {
279 self.instruction_horizon
280 .tokens
281 .saturating_sub(prefix_tokens)
282 .saturating_sub(tail_tokens)
283 .saturating_sub(self.output_reserve)
284 .saturating_sub(current_turn_reserve)
285 }
286
287 /// Whether `tokens_per_char` reflects a real observation rather than
288 /// the uncalibrated bootstrap value.
289 pub fn is_calibrated(&self) -> bool {
290 self.tokens_per_char.samples > 0
291 }
292
293 /// Estimated token count for `chars`, using the measured tokens/char
294 /// once calibrated and the fixed bootstrap value only before that.
295 pub fn estimate_tokens(&self, chars: u64) -> u64 {
296 let tpc = if self.is_calibrated() {
297 self.tokens_per_char.value
298 } else {
299 UNCALIBRATED_TOKENS_PER_CHAR
300 };
301 (chars as f64 * tpc).round() as u64
302 }
303
304 /// Folds one turn's real usage into the profile (§1 "Feedback").
305 /// `cache_miss` gates the prefill measurement: prefill throughput is
306 /// only meaningful when the provider actually re-evaluated the prefix.
307 ///
308 /// `tokens_per_char` calibrates against `usage.prompt_tokens()` — every
309 /// prompt token the provider billed, cache tiers included — never
310 /// `usage.input_tokens` alone: `chars_sent` (the assembler's char count
311 /// for the WHOLE request) counts every byte sent whether or not it hit
312 /// the cache, so the numerator has to match. Using `input_tokens` alone
313 /// collapses the ratio toward zero as cache hits grow (a full cache hit
314 /// reports `input_tokens == 0` for a request that was not remotely
315 /// empty), which is exactly backwards: the calibration exists to relate
316 /// bytes sent to tokens billed, and prompt caching does not shrink
317 /// either one.
318 pub fn observe_usage(
319 &mut self,
320 chars_sent: u64,
321 usage: &vak_llm::Usage,
322 first_token_latency_ms: Option<u64>,
323 cache_miss: bool,
324 ) {
325 let prompt_tokens = usage.prompt_tokens();
326 if chars_sent > 0 && prompt_tokens > 0 {
327 self.tokens_per_char
328 .observe(prompt_tokens as f64 / chars_sent as f64);
329 }
330 if cache_miss && usage.input_tokens > 0 {
331 // Prefer the provider's own reported prefill latency (Ollama
332 // fills `usage.prefill_ms`) over the caller's wall-clock
333 // estimate, since it excludes model-load and network time.
334 if let Some(ms) = usage.prefill_ms.or(first_token_latency_ms)
335 && ms > 0
336 {
337 self.prefill_tps
338 .observe(usage.input_tokens as f64 / (ms as f64 / 1000.0));
339 }
340 }
341 }
342
343 /// Records that a request of `request_tokens` failed an explicit
344 /// instruction (required card not emitted, required tool not called, a
345 /// stop-policy block) the runtime already classifies. Only requests
346 /// near the current horizon count as evidence about the horizon itself
347 /// (§1 "Horizon tightening"); this never raises the horizon; a re-probe
348 /// is the only path that can.
349 pub fn observe_instruction_failure(&mut self, request_tokens: u64) {
350 let threshold = HORIZON_FAILURE_FRACTION * self.instruction_horizon.tokens as f64;
351 if (request_tokens as f64) < threshold {
352 return;
353 }
354 let lowered = self.instruction_horizon.tokens.min(request_tokens);
355 self.instruction_horizon = Horizon {
356 tokens: lowered,
357 confidence: HORIZON_FEEDBACK_CONFIDENCE,
358 last_confirmed: SystemTime::now(),
359 };
360 self.needs_reprobe = true;
361 }
362
363 /// Folds a provider's over-length rejection (`LlmError::Context`) into
364 /// the profile (docs/design/68-context-engine.md §5's over-length →
365 /// replan path): `verified_window` becomes this exact request size
366 /// (feedback, never a guess), and the horizon shrinks to
367 /// `request_tokens × OVER_LENGTH_SHRINK` with `HORIZON_FEEDBACK_CONFIDENCE`
368 /// — never widened, only ever lowered, like every other horizon
369 /// feedback path.
370 pub fn observe_over_length(&mut self, request_tokens: u64) {
371 self.verified_window = Some(match self.verified_window {
372 Some(previous) => previous.min(request_tokens),
373 None => request_tokens,
374 });
375 let shrunk = (request_tokens as f64 * OVER_LENGTH_SHRINK) as u64;
376 self.instruction_horizon = Horizon {
377 tokens: self.instruction_horizon.tokens.min(shrunk),
378 confidence: HORIZON_FEEDBACK_CONFIDENCE,
379 last_confirmed: SystemTime::now(),
380 };
381 self.needs_reprobe = true;
382 }
383
384 /// Folds one closed turn's total size into `current_turn_reserve` (§4),
385 /// so the planner's reserve for the NEXT open turn reflects how large
386 /// this model's turns actually tend to be.
387 pub fn observe_current_turn_tokens(&mut self, tokens: u64) {
388 self.current_turn_reserve.observe(tokens as f64);
389 }
390
391 /// The reserve the planner sets aside for the still-open turn (§4):
392 /// the EWMA of past current-turn sizes, floored by `measured_so_far` —
393 /// the open turn's own measured size can never be estimated as less
394 /// than what it has already spent.
395 pub fn current_turn_reserve(&self, measured_so_far: u64) -> u64 {
396 if self.current_turn_reserve.samples == 0 {
397 return measured_so_far;
398 }
399 (self.current_turn_reserve.value.round() as u64).max(measured_so_far)
400 }
401
402 /// Whether this profile should be re-probed before being trusted again:
403 /// past its TTL (24h local / 7d hosted), or contradicted metadata.
404 /// `current_metadata_digest` is compared by the caller because only it
405 /// knows the freshly fetched metadata; passing `None` skips that check.
406 pub fn is_stale(&self, now: SystemTime, local: bool) -> bool {
407 let ttl = if local {
408 LOCAL_PROFILE_TTL
409 } else {
410 HOSTED_PROFILE_TTL
411 };
412 match now.duration_since(self.provenance.probed_at) {
413 Ok(elapsed) => elapsed > ttl,
414 // Clock moved backwards since the probe: treat as fresh rather
415 // than fabricating staleness from an impossible duration.
416 Err(_) => false,
417 }
418 }
419}
420
421/// The horizon-ladder state machine (§1 "Horizon ladder"). Pure: the caller
422/// drives it by asking `next_rung()`, sending exactly that request, and
423/// folding the outcome back with `report()`. No network call happens here.
424#[derive(Debug, Clone)]
425pub struct Ladder {
426 rungs: Vec<u64>,
427 next_index: usize,
428 last_pass: Option<u64>,
429 first_fail: Option<u64>,
430 searching: bool,
431 result: Option<Horizon>,
432 verified_window: Option<u64>,
433}
434
435impl Ladder {
436 /// Builds the rung sequence: the named starting sizes filtered to (and
437 /// extended geometrically past, by doubling) `declared_window * 0.9`.
438 pub fn new(declared_window: u64) -> Self {
439 let cap = (declared_window as f64 * LADDER_MAX_FRACTION) as u64;
440 let mut rungs: Vec<u64> = LADDER_START_RUNGS
441 .into_iter()
442 .filter(|&r| r <= cap)
443 .collect();
444 let mut next = LADDER_START_RUNGS
445 .last()
446 .copied()
447 .unwrap_or(4_000)
448 .saturating_mul(2);
449 while next <= cap && next > 0 {
450 rungs.push(next);
451 next = next.saturating_mul(2);
452 }
453 if rungs.is_empty() && cap > 0 {
454 rungs.push(cap);
455 }
456 Ladder {
457 rungs,
458 next_index: 0,
459 last_pass: None,
460 first_fail: None,
461 searching: false,
462 result: None,
463 verified_window: None,
464 }
465 }
466
467 /// The next prompt size to probe, or `None` once the ladder has
468 /// converged on a horizon.
469 pub fn next_rung(&self) -> Option<u64> {
470 if self.result.is_some() {
471 return None;
472 }
473 if self.searching {
474 let lo = self.last_pass.unwrap_or(0);
475 let hi = self.first_fail?;
476 if hi <= lo {
477 return None;
478 }
479 let mid = lo + (hi - lo) / 2;
480 if mid <= lo { None } else { Some(mid) }
481 } else {
482 self.rungs.get(self.next_index).copied()
483 }
484 }
485
486 /// Folds one rung's outcome back in. `accepted` is false for a
487 /// provider-level rejection (400/413 style context error); `followed`
488 /// is meaningless when `accepted` is false.
489 pub fn report(&mut self, tokens: u64, accepted: bool, followed: bool) {
490 if !accepted {
491 self.verified_window = Some(match self.verified_window {
492 Some(v) => v.min(tokens.saturating_sub(1)),
493 None => tokens.saturating_sub(1),
494 });
495 self.first_fail = Some(match self.first_fail {
496 Some(f) => f.min(tokens),
497 None => tokens,
498 });
499 self.searching = true;
500 } else if followed {
501 self.last_pass = Some(match self.last_pass {
502 Some(p) => p.max(tokens),
503 None => tokens,
504 });
505 if !self.searching {
506 self.next_index += 1;
507 }
508 } else {
509 // Accepted but the model did not follow the instruction: this
510 // rung IS the instruction-horizon boundary, same as a rejection
511 // for search purposes.
512 self.first_fail = Some(match self.first_fail {
513 Some(f) => f.min(tokens),
514 None => tokens,
515 });
516 self.searching = true;
517 }
518 self.converge();
519 }
520
521 fn converge(&mut self) {
522 if self.result.is_some() {
523 return;
524 }
525 if let (Some(lo), Some(hi)) = (self.last_pass, self.first_fail) {
526 if hi <= lo {
527 self.result = Some(self.horizon_at(lo));
528 return;
529 }
530 let gap = (hi - lo) as f64 / hi as f64;
531 if gap < LADDER_SEARCH_STOP_FRACTION {
532 self.result = Some(self.horizon_at(lo));
533 }
534 } else if !self.searching && self.next_index >= self.rungs.len() {
535 // Every named rung passed with no failure ever observed: the
536 // horizon is at least the last rung tried.
537 if let Some(lo) = self.last_pass {
538 self.result = Some(self.horizon_at(lo));
539 }
540 }
541 }
542
543 fn horizon_at(&self, tokens: u64) -> Horizon {
544 Horizon {
545 tokens,
546 confidence: PROBED_CONFIDENCE,
547 last_confirmed: SystemTime::now(),
548 }
549 }
550
551 /// The converged horizon, once `next_rung()` has returned `None`.
552 pub fn result(&self) -> Option<Horizon> {
553 self.result.clone()
554 }
555
556 /// The largest prompt size the provider is known to have accepted at
557 /// all (independent of instruction-following), when a rejection was
558 /// observed.
559 pub fn verified_window(&self) -> Option<u64> {
560 self.verified_window
561 }
562}
563
564/// Identical requests sent per ladder rung; the rung's verdict is the
565/// majority. A sampling model answers the same prompt differently run to
566/// run, so a single completion cannot decide whether an instruction was
567/// followed at that size. Three is the smallest odd count with a majority.
568pub const PROBE_SAMPLES_PER_RUNG: u32 = 3;
569
570/// Builds a probe request of roughly `tokens_target` tokens shaped like the
571/// history the model will really see (docs/design/68 §1): each filler turn
572/// is a user question, an assistant `lookup` tool call, a digest-shaped
573/// tool result carrying an `[evidence:…]` tag, and a short assistant
574/// answer — the same call pattern a closed turn projects at `Full`. Inert
575/// prose would measure a horizon the model never reaches on real work.
576/// The content is varied and non-repeating so no adjacent-message
577/// prefix-cache shortcut masks the prefill cost. The request ends with an
578/// instruction to call `probe_ack`. `tokens_per_char` should come from the
579/// profile being probed when calibrated, so rung sizes land close to their
580/// target on real providers.
581pub fn probe_request(
582 tokens_target: u64,
583 tokens_per_char: f64,
584 model: &str,
585) -> vak_llm::ChatRequest {
586 let tpc = if tokens_per_char > 0.0 {
587 tokens_per_char
588 } else {
589 UNCALIBRATED_TOKENS_PER_CHAR
590 };
591 let chars_target = (tokens_target as f64 / tpc) as u64;
592 let mut messages = Vec::new();
593 let mut chars_written: u64 = 0;
594 let mut seed: u64 = 0;
595 while chars_written < chars_target {
596 for message in probe_filler_turn(seed) {
597 chars_written += message.text_content().len() as u64
598 + message
599 .content
600 .iter()
601 .map(|block| match block {
602 vak_llm::ContentBlock::ToolUse { input, .. } => input.to_string().len(),
603 vak_llm::ContentBlock::ToolResult { content, .. } => content.len(),
604 _ => 0,
605 } as u64)
606 .sum::<u64>();
607 messages.push(message);
608 }
609 seed += 1;
610 }
611 messages.push(vak_llm::Message::user_text(
612 "Call the probe_ack tool now with {\"ok\": true} as its only argument. \
613 Do not answer in prose and do not call any other tool.",
614 ));
615 let mut request = vak_llm::ChatRequest::new(model);
616 request.messages = messages;
617 request.tools = vec![probe_lookup_tool(), vak_llm::ToolDefinition::probe_ack()];
618 request.max_tokens = 32;
619 request
620}
621
622/// The stand-in retrieval tool the filler turns "called"; defined on the
623/// request so every provider accepts the replayed pairs.
624fn probe_lookup_tool() -> vak_llm::ToolDefinition {
625 vak_llm::ToolDefinition::new(
626 "lookup",
627 "Look a topic up and return matching records.",
628 serde_json::json!({
629 "type": "object",
630 "properties": { "query": { "type": "string" } },
631 "required": ["query"],
632 }),
633 )
634}
635
636/// One filler turn — four messages in the shape of a projected closed turn.
637/// Seeded by turn index so a probe run is reproducible and every turn
638/// differs.
639fn probe_filler_turn(seed: u64) -> Vec<vak_llm::Message> {
640 let call_id = format!("probe-call-{seed}");
641 let topic = probe_filler_text(seed, 6);
642 let question = format!("What did the {topic} report say in section {}?", seed + 1);
643 let result = format!(
644 "[{{\"title\":\"{}\",\"url\":\"https://example.invalid/{seed}\"}},\
645 {{\"title\":\"{}\",\"url\":\"https://example.invalid/{seed}-b\"}}]\n\
646 [evidence:{call_id} \u{2014} {} chars; call recall to expand]",
647 probe_filler_text(seed.wrapping_add(101), 5),
648 probe_filler_text(seed.wrapping_add(202), 5),
649 900 + seed * 7
650 );
651 let answer = format!(
652 "Section {} of the {topic} report covers {}.",
653 seed + 1,
654 probe_filler_text(seed.wrapping_add(303), 30)
655 );
656 vec![
657 vak_llm::Message::user_text(question),
658 vak_llm::Message::assistant(vec![vak_llm::ContentBlock::ToolUse {
659 id: call_id.clone(),
660 name: "lookup".into(),
661 input: serde_json::json!({ "query": topic }),
662 }]),
663 vak_llm::Message {
664 role: vak_llm::Role::User,
665 content: vec![vak_llm::ContentBlock::tool_result(call_id, result)],
666 },
667 vak_llm::Message::assistant(vec![vak_llm::ContentBlock::text(answer)]),
668 ]
669}
670
671/// Deterministic, non-repeating filler words. Seeded so a probe run is
672/// reproducible, and varied so the provider cannot shortcut prefill via a
673/// repeated-content optimisation.
674fn probe_filler_text(seed: u64, words: u64) -> String {
675 const WORDS: [&str; 16] = [
676 "ridge",
677 "cobalt",
678 "ferry",
679 "lantern",
680 "quartz",
681 "meridian",
682 "otter",
683 "glacier",
684 "ember",
685 "thicket",
686 "vellum",
687 "cinder",
688 "harbor",
689 "tundra",
690 "opal",
691 "switchback",
692 ];
693 let mut s = String::new();
694 for j in 0..words {
695 let idx = ((seed.wrapping_mul(31).wrapping_add(j.wrapping_mul(17))) as usize) % WORDS.len();
696 if j > 0 {
697 s.push(' ');
698 }
699 s.push_str(WORDS[idx]);
700 }
701 s
702}
703
704/// Two identical requests are sent back to back at the same size the
705/// probe already used (§1 "Cache rung"); this classifies what the pair
706/// showed. `ProviderReported` when the second response's usage shows a
707/// cache hit (`cache_read_input_tokens`, which already folds in the OpenAI
708/// `prompt_tokens_details.cached_tokens` shape — see the adapters in
709/// vak-llm). Otherwise `PrefixStable` when the provider reports nothing but
710/// the second request's first-token latency dropped to a fifth or less of
711/// the first's — a local runner's prefix cache shows up only as timing.
712/// `None` (measured, no caching detected) when neither signal fired; the
713/// caller records both raw latencies as provenance signals regardless.
714pub fn classify_cache_rung(
715 first_latency_ms: u64,
716 second_latency_ms: u64,
717 second_usage: &vak_llm::Usage,
718) -> CacheBehaviour {
719 if second_usage.cache_read_input_tokens.unwrap_or(0) > 0 {
720 return CacheBehaviour::ProviderReported;
721 }
722 if first_latency_ms > 0 && second_latency_ms.saturating_mul(5) <= first_latency_ms {
723 return CacheBehaviour::PrefixStable;
724 }
725 CacheBehaviour::None
726}
727
728/// Whether a probe response followed the instruction: a `ToolUse` block
729/// named `probe_ack` anywhere in the response.
730pub fn followed(response: &vak_llm::AssistantMessage) -> bool {
731 response
732 .content
733 .iter()
734 .any(|b| matches!(b, vak_llm::ContentBlock::ToolUse { name, .. } if name == "probe_ack"))
735}
736
737#[cfg(test)]
738#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
739mod tests {
740 use super::*;
741
742 #[test]
743 fn budget_saturates_instead_of_underflowing() {
744 let profile = flat_profile_with_reserve(1_000, 0);
745 assert_eq!(profile.budget(200, 100, 0), 700);
746 // prefix + tail + reserve exceed the horizon: saturates to 0, never wraps.
747 assert_eq!(profile.budget(900, 200, 0), 0);
748 }
749
750 #[test]
751 fn ewma_first_sample_replaces_zero_seed() {
752 let mut ewma = Ewma::new(0.3);
753 ewma.observe(10.0);
754 assert_eq!(ewma.value, 10.0);
755 assert_eq!(ewma.samples, 1);
756 ewma.observe(20.0);
757 assert!((ewma.value - (0.3 * 20.0 + 0.7 * 10.0)).abs() < 1e-9);
758 assert_eq!(ewma.samples, 2);
759 }
760
761 #[test]
762 fn estimate_tokens_uses_bootstrap_until_calibrated() {
763 let profile = flat_profile(10_000);
764 assert_eq!(profile.estimate_tokens(400), 100); // 0.25 tokens/char
765 let mut calibrated = profile;
766 calibrated.tokens_per_char.observe(0.5);
767 assert_eq!(calibrated.estimate_tokens(400), 200);
768 }
769
770 #[test]
771 fn observe_usage_updates_tokens_per_char_and_prefill_on_cache_miss_only() {
772 let mut profile = flat_profile(10_000);
773 let usage = vak_llm::Usage {
774 input_tokens: 1_000,
775 output_tokens: 10,
776 prefill_ms: Some(2_000),
777 ..Default::default()
778 };
779 profile.observe_usage(2_000, &usage, None, true);
780 assert_eq!(profile.tokens_per_char.samples, 1);
781 assert!((profile.tokens_per_char.value - 0.5).abs() < 1e-9);
782 assert_eq!(profile.prefill_tps.samples, 1);
783 assert!((profile.prefill_tps.value - 500.0).abs() < 1e-9); // 1000 tok / 2s
784
785 let mut cache_hit_profile = flat_profile(10_000);
786 cache_hit_profile.observe_usage(2_000, &usage, None, false);
787 assert_eq!(cache_hit_profile.prefill_tps.samples, 0);
788 }
789
790 /// Item 2 fix: on a full cache hit `input_tokens` reports 0 for a
791 /// request that was not remotely empty. Calibrating against
792 /// `input_tokens` alone would either skip the observation (guard fails)
793 /// or, on a partial hit, silently drag `tokens_per_char` toward zero
794 /// over repeated turns. `prompt_tokens()` (fresh + both cache tiers)
795 /// must be what tokens_per_char calibrates against.
796 #[test]
797 fn observe_usage_calibrates_against_prompt_tokens_not_input_tokens_alone() {
798 let mut profile = flat_profile(10_000);
799 let full_cache_hit = vak_llm::Usage {
800 input_tokens: 0,
801 output_tokens: 5,
802 cache_read_input_tokens: Some(8_000),
803 ..Default::default()
804 };
805 // cache_miss=false: this IS a cache hit, but the calibration must
806 // still run — cache_miss only gates prefill_tps, never tokens/char.
807 profile.observe_usage(4_000, &full_cache_hit, None, false);
808 assert_eq!(profile.tokens_per_char.samples, 1);
809 assert!((profile.tokens_per_char.value - 2.0).abs() < 1e-9); // 8000/4000
810 }
811
812 /// A steady stream of partial cache hits must not drag the ratio toward
813 /// zero the way calibrating on `input_tokens` alone would (§1
814 /// "Feedback" bug report: repeated cache hits collapsed the EWMA).
815 #[test]
816 fn repeated_partial_cache_hits_do_not_collapse_tokens_per_char() {
817 let mut profile = flat_profile(10_000);
818 let partial_hit = vak_llm::Usage {
819 input_tokens: 50, // small fresh remainder after the prefix cache
820 output_tokens: 10,
821 cache_read_input_tokens: Some(4_950),
822 ..Default::default()
823 };
824 for _ in 0..5 {
825 profile.observe_usage(5_000, &partial_hit, None, false);
826 }
827 // 5000 prompt tokens / 5000 chars == 1.0 every time; five identical
828 // observations must leave the EWMA at 1.0, not collapse toward the
829 // 50/5000 == 0.01 that `input_tokens` alone would calibrate.
830 assert!((profile.tokens_per_char.value - 1.0).abs() < 1e-9);
831 }
832
833 #[test]
834 fn horizon_never_widens_on_feedback() {
835 let mut profile = flat_profile(10_000);
836 profile.observe_instruction_failure(9_000); // >= 0.8 * 10_000
837 assert_eq!(profile.instruction_horizon.tokens, 9_000);
838 assert!(profile.needs_reprobe);
839 assert_eq!(profile.instruction_horizon.confidence, 0.6);
840
841 // A later "failure" reported at a larger size than the (already
842 // lowered) horizon must not raise it back up.
843 profile.observe_instruction_failure(9_500);
844 assert_eq!(profile.instruction_horizon.tokens, 9_000);
845 }
846
847 #[test]
848 fn observe_instruction_failure_ignores_requests_far_below_horizon() {
849 let mut profile = flat_profile(10_000);
850 profile.observe_instruction_failure(1_000); // well under 0.8 * horizon
851 assert_eq!(profile.instruction_horizon.tokens, 10_000);
852 assert!(!profile.needs_reprobe);
853 }
854
855 #[test]
856 fn over_length_feedback_shrinks_the_horizon_and_never_widens_it() {
857 let mut profile = flat_profile(10_000);
858 profile.observe_over_length(8_000);
859 assert_eq!(profile.verified_window, Some(8_000));
860 assert_eq!(profile.instruction_horizon.tokens, 7_200); // 8_000 * 0.9
861 assert_eq!(profile.instruction_horizon.confidence, 0.6);
862 assert!(profile.needs_reprobe);
863
864 // A later, larger rejected size must not widen the already-lowered
865 // horizon or verified_window back up.
866 profile.observe_over_length(9_000);
867 assert_eq!(profile.verified_window, Some(8_000));
868 assert_eq!(profile.instruction_horizon.tokens, 7_200);
869 }
870
871 #[test]
872 fn current_turn_reserve_floors_at_the_measured_size_before_and_after_samples() {
873 let mut profile = flat_profile(10_000);
874 // No samples yet: the reserve is exactly the measured-so-far floor.
875 assert_eq!(profile.current_turn_reserve(500), 500);
876 profile.observe_current_turn_tokens(200);
877 // One low sample must not undercut a larger turn-so-far measurement.
878 assert_eq!(profile.current_turn_reserve(500), 500);
879 profile.observe_current_turn_tokens(2_000);
880 // EWMA now exceeds a smaller measured-so-far value and wins.
881 assert!(profile.current_turn_reserve(10) > 10);
882 }
883
884 #[test]
885 fn is_stale_respects_local_vs_hosted_ttl() {
886 let profile = flat_profile(10_000);
887 let just_over_a_day = profile.provenance.probed_at + Duration::from_secs(25 * 3600);
888 assert!(profile.is_stale(just_over_a_day, true));
889 assert!(!profile.is_stale(just_over_a_day, false));
890 }
891
892 #[test]
893 fn ladder_converges_via_binary_search_within_tolerance() {
894 // Real horizon is 20_000; every rung <= 20_000 passes, every rung
895 // above fails the instruction.
896 let mut ladder = Ladder::new(200_000);
897 let real_horizon = 20_000u64;
898 let mut iterations = 0;
899 while let Some(rung) = ladder.next_rung() {
900 iterations += 1;
901 assert!(iterations < 100, "ladder did not converge");
902 let followed = rung <= real_horizon;
903 ladder.report(rung, true, followed);
904 }
905 let horizon = ladder.result().expect("ladder should converge");
906 assert!(horizon.tokens <= real_horizon);
907 // within the 25% search-stop tolerance of the true horizon
908 assert!(horizon.tokens as f64 >= real_horizon as f64 * 0.75);
909 }
910
911 #[test]
912 fn ladder_records_verified_window_on_provider_rejection() {
913 let mut ladder = Ladder::new(200_000);
914 // First rung is accepted and followed.
915 let first = ladder.next_rung().expect("first rung");
916 ladder.report(first, true, true);
917 // Next rung is rejected outright by the provider.
918 let second = ladder.next_rung().expect("second rung");
919 ladder.report(second, false, false);
920 assert_eq!(ladder.verified_window(), Some(second - 1));
921 }
922
923 #[test]
924 fn probe_request_is_shaped_like_projected_turns_and_ends_with_probe_ack() {
925 let req = probe_request(1_000, 0.25, "test-model");
926 assert_eq!(req.max_tokens, 32);
927 let names: Vec<&str> = req.tools.iter().map(|t| t.name.as_str()).collect();
928 assert_eq!(names, vec!["lookup", "probe_ack"]);
929 assert!(req.cache.is_none());
930 let last = req
931 .messages
932 .last()
933 .expect("at least the instruction message");
934 assert!(last.text_content().contains("probe_ack"));
935 // Each filler turn is user question → assistant lookup call →
936 // digest-shaped result with an evidence tag → assistant answer.
937 assert_eq!(req.messages[0].role, vak_llm::Role::User);
938 assert!(matches!(
939 &req.messages[1].content[0],
940 vak_llm::ContentBlock::ToolUse { name, .. } if name == "lookup"
941 ));
942 assert!(matches!(
943 &req.messages[2].content[0],
944 vak_llm::ContentBlock::ToolResult { content, .. } if content.contains("[evidence:")
945 ));
946 assert_eq!(req.messages[3].role, vak_llm::Role::Assistant);
947 // Every filler turn differs (no prefix-cache shortcut inside a rung).
948 assert_ne!(
949 req.messages[0].text_content(),
950 req.messages[4].text_content()
951 );
952 // Roughly sized: within a generous tolerance of the token target
953 // translated through the given tokens/char.
954 let total_chars: usize = req
955 .messages
956 .iter()
957 .flat_map(|m| m.content.iter())
958 .map(|b| match b {
959 vak_llm::ContentBlock::Text { text } => text.len(),
960 vak_llm::ContentBlock::ToolUse { input, .. } => input.to_string().len(),
961 vak_llm::ContentBlock::ToolResult { content, .. } => content.len(),
962 _ => 0,
963 })
964 .sum();
965 assert!(total_chars as f64 >= 1_000.0 / 0.25 * 0.5);
966 assert!(total_chars as f64 <= 1_000.0 / 0.25 * 2.0);
967 }
968
969 #[test]
970 fn cache_rung_prefers_provider_reported_cache_hit() {
971 let usage = vak_llm::Usage {
972 cache_read_input_tokens: Some(3_000),
973 ..Default::default()
974 };
975 // Even with no latency improvement at all, a real cache hit wins.
976 assert_eq!(
977 classify_cache_rung(1_000, 1_000, &usage),
978 CacheBehaviour::ProviderReported
979 );
980 }
981
982 #[test]
983 fn cache_rung_falls_back_to_five_x_latency_when_unreported() {
984 let usage = vak_llm::Usage::default();
985 // Exactly 5x faster: still counts (<=), not strictly less-than.
986 assert_eq!(
987 classify_cache_rung(1_000, 200, &usage),
988 CacheBehaviour::PrefixStable
989 );
990 assert_eq!(
991 classify_cache_rung(1_000, 201, &usage),
992 CacheBehaviour::None
993 );
994 }
995
996 #[test]
997 fn cache_rung_is_none_with_no_signal_at_all() {
998 let usage = vak_llm::Usage::default();
999 assert_eq!(classify_cache_rung(500, 480, &usage), CacheBehaviour::None);
1000 // A zero first latency can't establish a ratio; never fabricate a hit.
1001 assert_eq!(classify_cache_rung(0, 0, &usage), CacheBehaviour::None);
1002 }
1003
1004 #[test]
1005 fn followed_detects_probe_ack_tool_use_only() {
1006 let with_ack = vak_llm::AssistantMessage {
1007 content: vec![vak_llm::ContentBlock::ToolUse {
1008 id: "1".into(),
1009 name: "probe_ack".into(),
1010 input: serde_json::json!({"ok": true}),
1011 }],
1012 stop_reason: vak_llm::StopReason::ToolUse,
1013 usage: vak_llm::Usage::default(),
1014 model: "m".into(),
1015 response_id: None,
1016 };
1017 assert!(followed(&with_ack));
1018
1019 let prose_only = vak_llm::AssistantMessage {
1020 content: vec![vak_llm::ContentBlock::text("sure, here you go")],
1021 stop_reason: vak_llm::StopReason::EndTurn,
1022 usage: vak_llm::Usage::default(),
1023 model: "m".into(),
1024 response_id: None,
1025 };
1026 assert!(!followed(&prose_only));
1027 }
1028
1029 fn flat_profile(horizon_tokens: u64) -> CapacityProfile {
1030 flat_profile_with_reserve(horizon_tokens, 1_024)
1031 }
1032
1033 fn flat_profile_with_reserve(horizon_tokens: u64, output_reserve: u64) -> CapacityProfile {
1034 CapacityProfile::from_probe(
1035 horizon_tokens,
1036 None,
1037 Horizon {
1038 tokens: horizon_tokens,
1039 confidence: PROBED_CONFIDENCE,
1040 last_confirmed: SystemTime::now(),
1041 },
1042 CacheBehaviour::Unknown,
1043 output_reserve,
1044 ProbeProvenance {
1045 probed_at: SystemTime::now(),
1046 rungs: Vec::new(),
1047 signals: Vec::new(),
1048 metadata_digest: "digest".into(),
1049 quantisation: None,
1050 },
1051 )
1052 }
1053}
1054

Indexing the workspace…