Vakyartha documentation

1//! Did we read the request right? (docs/design/47-commitment-kernel.md, I8)
2//!
3//! Every other feedback loop in this codebase closes on *observed outcomes*
4//! rather than on self-assessment: the routing ladder learns from settlements,
5//! not from asking a model whether the answer was good. Intent resolution gets
6//! the same treatment.
7//!
8//! The strongest signal is a gift from progressive disclosure. When a reading
9//! leaves a tool deferred and the model then loads and uses that exact tool,
10//! the reading was **measurably** wrong — not suspected wrong. Deferring does
11//! not merely save context; it turns misclassification into an observable
12//! event, which is the only reason a feedback loop is possible at all.
13//!
14//! Epistemics match the routing ledger deliberately: success / failure /
15//! **unknown**, TTL-filtered, and absence is a neutral prior rather than a
16//! zero. A reading nobody corrected is not thereby proven right.
17
18use std::collections::{BTreeSet, HashMap};
19use std::io::Write;
20use std::path::{Path, PathBuf};
21
22use serde::{Deserialize, Serialize};
23
24use vak_intent::{Act, Tier};
25
26/// How long an observation stays relevant. Matches the routing ledger's TTL:
27/// a lexicon that changed six weeks ago should not be judged on what it did
28/// before the change.
29const TTL_DAYS: i64 = 30;
30
31/// What we observed about a reading after the fact.
32#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
33#[serde(rename_all = "kebab-case")]
34pub enum Outcome {
35 /// The turn proceeded and nothing contradicted the reading.
36 Held,
37 /// The engagement withheld a capability the turn then asked for. The one
38 /// signal here that is measured rather than inferred.
39 Escalated,
40 /// A human overrode the reading before the run.
41 Overridden,
42 /// The user immediately restated the same request, which usually means
43 /// the first attempt did the wrong thing.
44 Restated,
45 /// The turn was abandoned. Says something, but not what — it could be
46 /// the reading, the answer, or the user changing their mind.
47 Abandoned,
48}
49
50impl Outcome {
51 pub fn as_str(self) -> &'static str {
52 match self {
53 Outcome::Held => "held",
54 Outcome::Escalated => "escalated",
55 Outcome::Overridden => "overridden",
56 Outcome::Restated => "restated",
57 Outcome::Abandoned => "abandoned",
58 }
59 }
60
61 /// Whether this counts against the reading.
62 ///
63 /// `Abandoned` deliberately does not: a user who walked away tells us the
64 /// turn ended, not that it was misread, and counting it as a failure would
65 /// train the resolver on noise.
66 pub fn contradicts(self) -> bool {
67 matches!(
68 self,
69 Outcome::Escalated | Outcome::Overridden | Outcome::Restated
70 )
71 }
72}
73
74#[derive(Debug, Clone, Serialize, Deserialize)]
75pub struct MisreadRow {
76 pub ts: chrono::DateTime<chrono::Utc>,
77 /// The act that was read, so accuracy can be reported per cell.
78 pub act: String,
79 pub stakes: String,
80 /// Which tier produced the reading; a lexicon miss and a classifier miss
81 /// call for different fixes.
82 pub tier: String,
83 pub resolver_version: u32,
84 pub outcome: String,
85 /// The tool the model used after the reading left it deferred. Present
86 /// only on `Escalated`, and the most actionable field in the row: it names
87 /// the exact lexicon or slice entry to change.
88 #[serde(default, skip_serializing_if = "Option::is_none")]
89 pub wanted: Option<String>,
90 /// Whether the reading was confident enough to decide what was loaded.
91 /// A weak reading loads only the orientation floor and expects the model
92 /// to discover the rest, so a deferred tool used after one is the
93 /// system working as designed, not the reading being wrong.
94 #[serde(default)]
95 pub sliced: bool,
96}
97
98/// Append-only observations about intent readings.
99///
100/// Sits beside `routing-evidence.jsonl` and `commitments.jsonl` for the same
101/// reason: it is evidence about the runtime's own behaviour, not conversation
102/// content, and it must outlive any session.
103pub struct MisreadLedger {
104 path: PathBuf,
105}
106
107/// Accuracy for one `(resolver version, act, stakes)` cell.
108#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
109pub struct CellAccuracy {
110 /// The lexicon that produced these readings. A lexicon is only judged
111 /// on its own readings: rows written before a change say nothing about
112 /// the reader that replaced it.
113 pub resolver_version: u32,
114 pub act: String,
115 pub stakes: String,
116 pub held: u64,
117 pub contradicted: u64,
118 /// Observations that say nothing either way.
119 pub unknown: u64,
120 /// Deferred tools the model then used, most
121 /// frequent first. This is the actionable output of the whole loop.
122 pub wanted: Vec<(String, u64)>,
123}
124
125impl CellAccuracy {
126 /// Laplace-shrunk accuracy in [0,1].
127 ///
128 /// Identical treatment to `routing::reliability`: unknowns shrink
129 /// confidence without punishing direction, and an absent record is a
130 /// neutral prior rather than a zero. Three observations must not read as
131 /// certainty in either direction.
132 pub fn accuracy(&self) -> f64 {
133 let trials = self.held + self.contradicted + self.unknown;
134 (self.held as f64 + 0.5 * self.unknown as f64 + 1.0) / (trials as f64 + 2.0)
135 }
136
137 pub fn observations(&self) -> u64 {
138 self.held + self.contradicted + self.unknown
139 }
140}
141
142impl MisreadLedger {
143 pub fn new(sessions_home: &Path) -> Self {
144 MisreadLedger {
145 path: sessions_home.join("intent-evidence.jsonl"),
146 }
147 }
148
149 pub fn path(&self) -> &Path {
150 &self.path
151 }
152
153 pub fn record(
154 &self,
155 reading: &vak_intent::Reading,
156 tier: Tier,
157 resolver_version: u32,
158 outcome: Outcome,
159 wanted: Option<String>,
160 sliced: bool,
161 ) {
162 let row = MisreadRow {
163 ts: chrono::Utc::now(),
164 act: reading.act.as_str().to_string(),
165 stakes: reading.stakes.as_str().to_string(),
166 tier: tier.as_str().to_string(),
167 resolver_version,
168 outcome: outcome.as_str().to_string(),
169 wanted,
170 sliced,
171 };
172 let Ok(line) = serde_json::to_string(&row) else {
173 return;
174 };
175 if let Some(parent) = self.path.parent() {
176 let _ = std::fs::create_dir_all(parent);
177 }
178 // Best-effort: losing a telemetry row must never cost the user a turn.
179 if let Ok(mut file) = std::fs::OpenOptions::new()
180 .create(true)
181 .append(true)
182 .open(&self.path)
183 {
184 let _ = writeln!(file, "{line}");
185 }
186 }
187
188 /// TTL-filtered rows. A corrupt line — torn, not UTF-8, or not a row —
189 /// is skipped rather than trusted, and never ends the read.
190 pub fn rows(&self) -> Vec<MisreadRow> {
191 let cutoff = chrono::Utc::now() - chrono::Duration::days(TTL_DAYS);
192 let Ok(bytes) = std::fs::read(&self.path) else {
193 return Vec::new();
194 };
195 bytes
196 .split(|byte| *byte == b'\n')
197 .filter_map(|line| serde_json::from_slice::<MisreadRow>(line).ok())
198 .filter(|row| row.ts >= cutoff)
199 .collect()
200 }
201
202 /// Accuracy per `(resolver version, act, stakes)` cell, weakest first.
203 ///
204 /// Ordering by weakest is the point: this report exists to say where the
205 /// lexicon needs work, not to produce a flattering aggregate. A deferred
206 /// tool used after a reading that decided nothing — too weak to slice —
207 /// counts as unknown: the reading never claimed to know.
208 pub fn accuracy(&self) -> Vec<CellAccuracy> {
209 type Key = (u32, String, String);
210 let mut cells: HashMap<Key, CellAccuracy> = HashMap::new();
211 let mut wanted: HashMap<Key, HashMap<String, u64>> = HashMap::new();
212 for row in self.rows() {
213 let key = (row.resolver_version, row.act.clone(), row.stakes.clone());
214 let cell = cells.entry(key.clone()).or_insert_with(|| CellAccuracy {
215 resolver_version: row.resolver_version,
216 act: row.act.clone(),
217 stakes: row.stakes.clone(),
218 ..CellAccuracy::default()
219 });
220 match row.outcome.as_str() {
221 "held" => cell.held += 1,
222 "escalated" if !row.sliced => cell.unknown += 1,
223 "escalated" | "overridden" | "restated" => cell.contradicted += 1,
224 _ => cell.unknown += 1,
225 }
226 if let Some(name) = row.wanted {
227 *wanted.entry(key).or_default().entry(name).or_insert(0) += 1;
228 }
229 }
230 let mut out: Vec<CellAccuracy> = cells
231 .into_iter()
232 .map(|(key, mut cell)| {
233 if let Some(counts) = wanted.remove(&key) {
234 let mut ranked: Vec<(String, u64)> = counts.into_iter().collect();
235 ranked.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
236 cell.wanted = ranked;
237 }
238 cell
239 })
240 .collect();
241 out.sort_by(|a, b| {
242 a.accuracy()
243 .partial_cmp(&b.accuracy())
244 .unwrap_or(std::cmp::Ordering::Equal)
245 .then_with(|| b.resolver_version.cmp(&a.resolver_version))
246 .then_with(|| a.act.cmp(&b.act))
247 .then_with(|| a.stakes.cmp(&b.stakes))
248 });
249 out
250 }
251}
252
253/// Detect the measured misread: a tool the reading left deferred, that the
254/// model then used anyway.
255///
256/// `unpredicted` is `ToolSurface::unpredicted` (crates/vak-core/src/capability/
257/// surface.rs) — admitted tools the reading did not load, card tools excluded.
258/// `attempted` is every tool name the model invoked this turn.
259pub fn escalated_capability(
260 unpredicted: &BTreeSet<String>,
261 attempted: &[String],
262) -> Option<String> {
263 attempted
264 .iter()
265 .find(|name| unpredicted.contains(name.as_str()))
266 .cloned()
267}
268
269/// Acts whose readings, by the lexicon running now, are contradicted often
270/// enough to be worth a look.
271///
272/// Requires a real sample: a cell with two observations says nothing, and
273/// reporting it as a problem would send someone to rewrite a lexicon entry on
274/// the strength of a coin flip. A cell nothing contradicted is not weak,
275/// however many unknowns shrink its score. Cells from an earlier lexicon are
276/// left out: they describe a reader that no longer runs.
277pub fn weak_cells(ledger: &MisreadLedger, min_observations: u64) -> Vec<CellAccuracy> {
278 ledger
279 .accuracy()
280 .into_iter()
281 .filter(|cell| cell.resolver_version == vak_intent::RESOLVER_VERSION)
282 .filter(|cell| {
283 cell.contradicted > 0
284 && cell.observations() >= min_observations
285 && cell.accuracy() < 0.75
286 })
287 .collect()
288}
289
290/// Parse an act name back, for callers reporting per-cell coverage.
291pub fn act_of(cell: &CellAccuracy) -> Option<Act> {
292 Act::parse(&cell.act)
293}
294
295#[cfg(test)]
296#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
297mod tests {
298 use super::*;
299 use vak_intent::{Reading, Stakes};
300
301 fn reading(act: Act, stakes: Stakes) -> Reading {
302 Reading {
303 act,
304 stakes,
305 ..Reading::general()
306 }
307 }
308
309 #[test]
310 fn a_deferred_tool_the_model_then_used_is_the_measured_misread() {
311 let excluded = BTreeSet::from(["bash".to_string()]);
312 assert_eq!(
313 escalated_capability(&excluded, &["read".into(), "bash".into()]),
314 Some("bash".into())
315 );
316 // Nothing wanted that was not excluded.
317 assert_eq!(escalated_capability(&excluded, &["read".into()]), None);
318 }
319
320 /// A turn where nothing was deferred cannot have escalated past
321 /// an exclusion that never happened.
322 #[test]
323 fn no_exclusions_reports_no_escalation() {
324 let excluded = BTreeSet::new();
325 assert_eq!(
326 escalated_capability(&excluded, &["bash".into(), "anything".into()]),
327 None
328 );
329 }
330
331 /// Abandonment says the turn ended, not that it was misread. Counting it
332 /// against the reading would train the resolver on noise.
333 #[test]
334 fn abandonment_does_not_count_against_a_reading() {
335 assert!(!Outcome::Abandoned.contradicts());
336 assert!(!Outcome::Held.contradicts());
337 for outcome in [Outcome::Escalated, Outcome::Overridden, Outcome::Restated] {
338 assert!(outcome.contradicts());
339 }
340 }
341
342 /// A row from the running lexicon, on a reading confident enough to
343 /// have decided what was loaded.
344 fn record(
345 ledger: &MisreadLedger,
346 act: Act,
347 stakes: Stakes,
348 outcome: Outcome,
349 wanted: Option<&str>,
350 ) {
351 ledger.record(
352 &reading(act, stakes),
353 Tier::Signals,
354 vak_intent::RESOLVER_VERSION,
355 outcome,
356 wanted.map(str::to_string),
357 true,
358 );
359 }
360
361 #[test]
362 fn accuracy_is_shrunk_so_a_tiny_sample_is_not_certainty() {
363 let dir = tempfile::tempdir().unwrap();
364 let ledger = MisreadLedger::new(dir.path());
365 record(
366 &ledger,
367 Act::Modify,
368 Stakes::Reversible,
369 Outcome::Escalated,
370 Some("bash"),
371 );
372 let cells = ledger.accuracy();
373 assert_eq!(cells.len(), 1);
374 // One contradiction out of one observation is 0.0 raw; shrinkage keeps
375 // it well off the floor because one sample is not proof.
376 assert!(cells[0].accuracy() > 0.2, "{}", cells[0].accuracy());
377 assert!(cells[0].accuracy() < 0.5);
378 }
379
380 #[test]
381 fn the_report_names_the_capability_to_put_back() {
382 let dir = tempfile::tempdir().unwrap();
383 let ledger = MisreadLedger::new(dir.path());
384 for _ in 0..3 {
385 record(
386 &ledger,
387 Act::Answer,
388 Stakes::Inert,
389 Outcome::Escalated,
390 Some("webfetch"),
391 );
392 }
393 record(
394 &ledger,
395 Act::Answer,
396 Stakes::Inert,
397 Outcome::Escalated,
398 Some("bash"),
399 );
400 let cells = ledger.accuracy();
401 assert_eq!(cells[0].wanted.first().unwrap().0, "webfetch");
402 assert_eq!(cells[0].wanted.first().unwrap().1, 3);
403 }
404
405 #[test]
406 fn weak_cells_need_a_real_sample_before_they_are_reported() {
407 let dir = tempfile::tempdir().unwrap();
408 let ledger = MisreadLedger::new(dir.path());
409 record(
410 &ledger,
411 Act::Operate,
412 Stakes::Irreversible,
413 Outcome::Escalated,
414 None,
415 );
416 // One observation is not evidence of a weak cell.
417 assert!(weak_cells(&ledger, 5).is_empty());
418 for _ in 0..6 {
419 record(
420 &ledger,
421 Act::Operate,
422 Stakes::Irreversible,
423 Outcome::Escalated,
424 None,
425 );
426 }
427 assert_eq!(weak_cells(&ledger, 5).len(), 1);
428 }
429
430 #[test]
431 fn a_healthy_cell_is_not_reported_as_weak() {
432 let dir = tempfile::tempdir().unwrap();
433 let ledger = MisreadLedger::new(dir.path());
434 for _ in 0..20 {
435 record(&ledger, Act::Answer, Stakes::Inert, Outcome::Held, None);
436 }
437 assert!(weak_cells(&ledger, 5).is_empty());
438 assert!(ledger.accuracy()[0].accuracy() > 0.9);
439 }
440
441 /// A weak reading loads only the orientation floor and leaves the rest to
442 /// discovery, so a deferred tool used after one is not a misread.
443 #[test]
444 fn discovery_after_a_weak_reading_does_not_count_against_it() {
445 let dir = tempfile::tempdir().unwrap();
446 let ledger = MisreadLedger::new(dir.path());
447 for _ in 0..8 {
448 ledger.record(
449 &reading(Act::Answer, Stakes::Inert),
450 Tier::General,
451 vak_intent::RESOLVER_VERSION,
452 Outcome::Escalated,
453 Some("bash".into()),
454 false,
455 );
456 }
457 let cells = ledger.accuracy();
458 assert_eq!(cells[0].contradicted, 0);
459 assert_eq!(cells[0].unknown, 8);
460 assert!(weak_cells(&ledger, 5).is_empty());
461 }
462
463 /// A lexicon is judged only on its own readings: rows an earlier reader
464 /// wrote say nothing about the one running now.
465 #[test]
466 fn an_earlier_lexicon_does_not_score_the_current_one() {
467 let dir = tempfile::tempdir().unwrap();
468 let ledger = MisreadLedger::new(dir.path());
469 for _ in 0..8 {
470 ledger.record(
471 &reading(Act::Modify, Stakes::Reversible),
472 Tier::Signals,
473 vak_intent::RESOLVER_VERSION - 1,
474 Outcome::Escalated,
475 Some("bash".into()),
476 true,
477 );
478 }
479 assert_eq!(ledger.accuracy().len(), 1);
480 assert!(weak_cells(&ledger, 5).is_empty());
481 }
482
483 #[test]
484 fn corrupt_rows_are_skipped_rather_than_trusted() {
485 let dir = tempfile::tempdir().unwrap();
486 let ledger = MisreadLedger::new(dir.path());
487 record(
488 &ledger,
489 Act::Modify,
490 Stakes::Reversible,
491 Outcome::Held,
492 None,
493 );
494 let mut file = std::fs::OpenOptions::new()
495 .append(true)
496 .open(ledger.path())
497 .unwrap();
498 writeln!(file, "not json at all").unwrap();
499 file.write_all(b"{\"torn\": \xff\n").unwrap();
500 drop(file);
501 // A bad line never ends the read: the row after it still counts.
502 record(
503 &ledger,
504 Act::Modify,
505 Stakes::Reversible,
506 Outcome::Held,
507 None,
508 );
509 assert_eq!(ledger.rows().len(), 2);
510 }
511}
512

1//! Did we read the request right? (docs/design/47-commitment-kernel.md, I8)
2//!
3//! Every other feedback loop in this codebase closes on *observed outcomes*
4//! rather than on self-assessment: the routing ladder learns from settlements,
5//! not from asking a model whether the answer was good. Intent resolution gets
6//! the same treatment.
7//!
8//! The strongest signal is a gift from progressive disclosure. When a reading
9//! leaves a tool deferred and the model then loads and uses that exact tool,
10//! the reading was **measurably** wrong — not suspected wrong. Deferring does
11//! not merely save context; it turns misclassification into an observable
12//! event, which is the only reason a feedback loop is possible at all.
13//!
14//! Epistemics match the routing ledger deliberately: success / failure /
15//! **unknown**, TTL-filtered, and absence is a neutral prior rather than a
16//! zero. A reading nobody corrected is not thereby proven right.
17
18use std::collections::{BTreeSet, HashMap};
19use std::io::Write;
20use std::path::{Path, PathBuf};
21
22use serde::{Deserialize, Serialize};
23
24use vak_intent::{Act, Tier};
25
26/// How long an observation stays relevant. Matches the routing ledger's TTL:
27/// a lexicon that changed six weeks ago should not be judged on what it did
28/// before the change.
29const TTL_DAYS: i64 = 30;
30
31/// What we observed about a reading after the fact.
32#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
33#[serde(rename_all = "kebab-case")]
34pub enum Outcome {
35 /// The turn proceeded and nothing contradicted the reading.
36 Held,
37 /// The engagement withheld a capability the turn then asked for. The one
38 /// signal here that is measured rather than inferred.
39 Escalated,
40 /// A human overrode the reading before the run.
41 Overridden,
42 /// The user immediately restated the same request, which usually means
43 /// the first attempt did the wrong thing.
44 Restated,
45 /// The turn was abandoned. Says something, but not what — it could be
46 /// the reading, the answer, or the user changing their mind.
47 Abandoned,
48}
49
50impl Outcome {
51 pub fn as_str(self) -> &'static str {
52 match self {
53 Outcome::Held => "held",
54 Outcome::Escalated => "escalated",
55 Outcome::Overridden => "overridden",
56 Outcome::Restated => "restated",
57 Outcome::Abandoned => "abandoned",
58 }
59 }
60
61 /// Whether this counts against the reading.
62 ///
63 /// `Abandoned` deliberately does not: a user who walked away tells us the
64 /// turn ended, not that it was misread, and counting it as a failure would
65 /// train the resolver on noise.
66 pub fn contradicts(self) -> bool {
67 matches!(
68 self,
69 Outcome::Escalated | Outcome::Overridden | Outcome::Restated
70 )
71 }
72}
73
74#[derive(Debug, Clone, Serialize, Deserialize)]
75pub struct MisreadRow {
76 pub ts: chrono::DateTime<chrono::Utc>,
77 /// The act that was read, so accuracy can be reported per cell.
78 pub act: String,
79 pub stakes: String,
80 /// Which tier produced the reading; a lexicon miss and a classifier miss
81 /// call for different fixes.
82 pub tier: String,
83 pub resolver_version: u32,
84 pub outcome: String,
85 /// The tool the model used after the reading left it deferred. Present
86 /// only on `Escalated`, and the most actionable field in the row: it names
87 /// the exact lexicon or slice entry to change.
88 #[serde(default, skip_serializing_if = "Option::is_none")]
89 pub wanted: Option<String>,
90 /// Whether the reading was confident enough to decide what was loaded.
91 /// A weak reading loads only the orientation floor and expects the model
92 /// to discover the rest, so a deferred tool used after one is the
93 /// system working as designed, not the reading being wrong.
94 #[serde(default)]
95 pub sliced: bool,
96}
97
98/// Append-only observations about intent readings.
99///
100/// Sits beside `routing-evidence.jsonl` and `commitments.jsonl` for the same
101/// reason: it is evidence about the runtime's own behaviour, not conversation
102/// content, and it must outlive any session.
103pub struct MisreadLedger {
104 path: PathBuf,
105}
106
107/// Accuracy for one `(resolver version, act, stakes)` cell.
108#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
109pub struct CellAccuracy {
110 /// The lexicon that produced these readings. A lexicon is only judged
111 /// on its own readings: rows written before a change say nothing about
112 /// the reader that replaced it.
113 pub resolver_version: u32,
114 pub act: String,
115 pub stakes: String,
116 pub held: u64,
117 pub contradicted: u64,
118 /// Observations that say nothing either way.
119 pub unknown: u64,
120 /// Deferred tools the model then used, most
121 /// frequent first. This is the actionable output of the whole loop.
122 pub wanted: Vec<(String, u64)>,
123}
124
125impl CellAccuracy {
126 /// Laplace-shrunk accuracy in [0,1].
127 ///
128 /// Identical treatment to `routing::reliability`: unknowns shrink
129 /// confidence without punishing direction, and an absent record is a
130 /// neutral prior rather than a zero. Three observations must not read as
131 /// certainty in either direction.
132 pub fn accuracy(&self) -> f64 {
133 let trials = self.held + self.contradicted + self.unknown;
134 (self.held as f64 + 0.5 * self.unknown as f64 + 1.0) / (trials as f64 + 2.0)
135 }
136
137 pub fn observations(&self) -> u64 {
138 self.held + self.contradicted + self.unknown
139 }
140}
141
142impl MisreadLedger {
143 pub fn new(sessions_home: &Path) -> Self {
144 MisreadLedger {
145 path: sessions_home.join("intent-evidence.jsonl"),
146 }
147 }
148
149 pub fn path(&self) -> &Path {
150 &self.path
151 }
152
153 pub fn record(
154 &self,
155 reading: &vak_intent::Reading,
156 tier: Tier,
157 resolver_version: u32,
158 outcome: Outcome,
159 wanted: Option<String>,
160 sliced: bool,
161 ) {
162 let row = MisreadRow {
163 ts: chrono::Utc::now(),
164 act: reading.act.as_str().to_string(),
165 stakes: reading.stakes.as_str().to_string(),
166 tier: tier.as_str().to_string(),
167 resolver_version,
168 outcome: outcome.as_str().to_string(),
169 wanted,
170 sliced,
171 };
172 let Ok(line) = serde_json::to_string(&row) else {
173 return;
174 };
175 if let Some(parent) = self.path.parent() {
176 let _ = std::fs::create_dir_all(parent);
177 }
178 // Best-effort: losing a telemetry row must never cost the user a turn.
179 if let Ok(mut file) = std::fs::OpenOptions::new()
180 .create(true)
181 .append(true)
182 .open(&self.path)
183 {
184 let _ = writeln!(file, "{line}");
185 }
186 }
187
188 /// TTL-filtered rows. A corrupt line — torn, not UTF-8, or not a row —
189 /// is skipped rather than trusted, and never ends the read.
190 pub fn rows(&self) -> Vec<MisreadRow> {
191 let cutoff = chrono::Utc::now() - chrono::Duration::days(TTL_DAYS);
192 let Ok(bytes) = std::fs::read(&self.path) else {
193 return Vec::new();
194 };
195 bytes
196 .split(|byte| *byte == b'\n')
197 .filter_map(|line| serde_json::from_slice::<MisreadRow>(line).ok())
198 .filter(|row| row.ts >= cutoff)
199 .collect()
200 }
201
202 /// Accuracy per `(resolver version, act, stakes)` cell, weakest first.
203 ///
204 /// Ordering by weakest is the point: this report exists to say where the
205 /// lexicon needs work, not to produce a flattering aggregate. A deferred
206 /// tool used after a reading that decided nothing — too weak to slice —
207 /// counts as unknown: the reading never claimed to know.
208 pub fn accuracy(&self) -> Vec<CellAccuracy> {
209 type Key = (u32, String, String);
210 let mut cells: HashMap<Key, CellAccuracy> = HashMap::new();
211 let mut wanted: HashMap<Key, HashMap<String, u64>> = HashMap::new();
212 for row in self.rows() {
213 let key = (row.resolver_version, row.act.clone(), row.stakes.clone());
214 let cell = cells.entry(key.clone()).or_insert_with(|| CellAccuracy {
215 resolver_version: row.resolver_version,
216 act: row.act.clone(),
217 stakes: row.stakes.clone(),
218 ..CellAccuracy::default()
219 });
220 match row.outcome.as_str() {
221 "held" => cell.held += 1,
222 "escalated" if !row.sliced => cell.unknown += 1,
223 "escalated" | "overridden" | "restated" => cell.contradicted += 1,
224 _ => cell.unknown += 1,
225 }
226 if let Some(name) = row.wanted {
227 *wanted.entry(key).or_default().entry(name).or_insert(0) += 1;
228 }
229 }
230 let mut out: Vec<CellAccuracy> = cells
231 .into_iter()
232 .map(|(key, mut cell)| {
233 if let Some(counts) = wanted.remove(&key) {
234 let mut ranked: Vec<(String, u64)> = counts.into_iter().collect();
235 ranked.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
236 cell.wanted = ranked;
237 }
238 cell
239 })
240 .collect();
241 out.sort_by(|a, b| {
242 a.accuracy()
243 .partial_cmp(&b.accuracy())
244 .unwrap_or(std::cmp::Ordering::Equal)
245 .then_with(|| b.resolver_version.cmp(&a.resolver_version))
246 .then_with(|| a.act.cmp(&b.act))
247 .then_with(|| a.stakes.cmp(&b.stakes))
248 });
249 out
250 }
251}
252
253/// Detect the measured misread: a tool the reading left deferred, that the
254/// model then used anyway.
255///
256/// `unpredicted` is `ToolSurface::unpredicted` (crates/vak-core/src/capability/
257/// surface.rs) — admitted tools the reading did not load, card tools excluded.
258/// `attempted` is every tool name the model invoked this turn.
259pub fn escalated_capability(
260 unpredicted: &BTreeSet<String>,
261 attempted: &[String],
262) -> Option<String> {
263 attempted
264 .iter()
265 .find(|name| unpredicted.contains(name.as_str()))
266 .cloned()
267}
268
269/// Acts whose readings, by the lexicon running now, are contradicted often
270/// enough to be worth a look.
271///
272/// Requires a real sample: a cell with two observations says nothing, and
273/// reporting it as a problem would send someone to rewrite a lexicon entry on
274/// the strength of a coin flip. A cell nothing contradicted is not weak,
275/// however many unknowns shrink its score. Cells from an earlier lexicon are
276/// left out: they describe a reader that no longer runs.
277pub fn weak_cells(ledger: &MisreadLedger, min_observations: u64) -> Vec<CellAccuracy> {
278 ledger
279 .accuracy()
280 .into_iter()
281 .filter(|cell| cell.resolver_version == vak_intent::RESOLVER_VERSION)
282 .filter(|cell| {
283 cell.contradicted > 0
284 && cell.observations() >= min_observations
285 && cell.accuracy() < 0.75
286 })
287 .collect()
288}
289
290/// Parse an act name back, for callers reporting per-cell coverage.
291pub fn act_of(cell: &CellAccuracy) -> Option<Act> {
292 Act::parse(&cell.act)
293}
294
295#[cfg(test)]
296#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
297mod tests {
298 use super::*;
299 use vak_intent::{Reading, Stakes};
300
301 fn reading(act: Act, stakes: Stakes) -> Reading {
302 Reading {
303 act,
304 stakes,
305 ..Reading::general()
306 }
307 }
308
309 #[test]
310 fn a_deferred_tool_the_model_then_used_is_the_measured_misread() {
311 let excluded = BTreeSet::from(["bash".to_string()]);
312 assert_eq!(
313 escalated_capability(&excluded, &["read".into(), "bash".into()]),
314 Some("bash".into())
315 );
316 // Nothing wanted that was not excluded.
317 assert_eq!(escalated_capability(&excluded, &["read".into()]), None);
318 }
319
320 /// A turn where nothing was deferred cannot have escalated past
321 /// an exclusion that never happened.
322 #[test]
323 fn no_exclusions_reports_no_escalation() {
324 let excluded = BTreeSet::new();
325 assert_eq!(
326 escalated_capability(&excluded, &["bash".into(), "anything".into()]),
327 None
328 );
329 }
330
331 /// Abandonment says the turn ended, not that it was misread. Counting it
332 /// against the reading would train the resolver on noise.
333 #[test]
334 fn abandonment_does_not_count_against_a_reading() {
335 assert!(!Outcome::Abandoned.contradicts());
336 assert!(!Outcome::Held.contradicts());
337 for outcome in [Outcome::Escalated, Outcome::Overridden, Outcome::Restated] {
338 assert!(outcome.contradicts());
339 }
340 }
341
342 /// A row from the running lexicon, on a reading confident enough to
343 /// have decided what was loaded.
344 fn record(
345 ledger: &MisreadLedger,
346 act: Act,
347 stakes: Stakes,
348 outcome: Outcome,
349 wanted: Option<&str>,
350 ) {
351 ledger.record(
352 &reading(act, stakes),
353 Tier::Signals,
354 vak_intent::RESOLVER_VERSION,
355 outcome,
356 wanted.map(str::to_string),
357 true,
358 );
359 }
360
361 #[test]
362 fn accuracy_is_shrunk_so_a_tiny_sample_is_not_certainty() {
363 let dir = tempfile::tempdir().unwrap();
364 let ledger = MisreadLedger::new(dir.path());
365 record(
366 &ledger,
367 Act::Modify,
368 Stakes::Reversible,
369 Outcome::Escalated,
370 Some("bash"),
371 );
372 let cells = ledger.accuracy();
373 assert_eq!(cells.len(), 1);
374 // One contradiction out of one observation is 0.0 raw; shrinkage keeps
375 // it well off the floor because one sample is not proof.
376 assert!(cells[0].accuracy() > 0.2, "{}", cells[0].accuracy());
377 assert!(cells[0].accuracy() < 0.5);
378 }
379
380 #[test]
381 fn the_report_names_the_capability_to_put_back() {
382 let dir = tempfile::tempdir().unwrap();
383 let ledger = MisreadLedger::new(dir.path());
384 for _ in 0..3 {
385 record(
386 &ledger,
387 Act::Answer,
388 Stakes::Inert,
389 Outcome::Escalated,
390 Some("webfetch"),
391 );
392 }
393 record(
394 &ledger,
395 Act::Answer,
396 Stakes::Inert,
397 Outcome::Escalated,
398 Some("bash"),
399 );
400 let cells = ledger.accuracy();
401 assert_eq!(cells[0].wanted.first().unwrap().0, "webfetch");
402 assert_eq!(cells[0].wanted.first().unwrap().1, 3);
403 }
404
405 #[test]
406 fn weak_cells_need_a_real_sample_before_they_are_reported() {
407 let dir = tempfile::tempdir().unwrap();
408 let ledger = MisreadLedger::new(dir.path());
409 record(
410 &ledger,
411 Act::Operate,
412 Stakes::Irreversible,
413 Outcome::Escalated,
414 None,
415 );
416 // One observation is not evidence of a weak cell.
417 assert!(weak_cells(&ledger, 5).is_empty());
418 for _ in 0..6 {
419 record(
420 &ledger,
421 Act::Operate,
422 Stakes::Irreversible,
423 Outcome::Escalated,
424 None,
425 );
426 }
427 assert_eq!(weak_cells(&ledger, 5).len(), 1);
428 }
429
430 #[test]
431 fn a_healthy_cell_is_not_reported_as_weak() {
432 let dir = tempfile::tempdir().unwrap();
433 let ledger = MisreadLedger::new(dir.path());
434 for _ in 0..20 {
435 record(&ledger, Act::Answer, Stakes::Inert, Outcome::Held, None);
436 }
437 assert!(weak_cells(&ledger, 5).is_empty());
438 assert!(ledger.accuracy()[0].accuracy() > 0.9);
439 }
440
441 /// A weak reading loads only the orientation floor and leaves the rest to
442 /// discovery, so a deferred tool used after one is not a misread.
443 #[test]
444 fn discovery_after_a_weak_reading_does_not_count_against_it() {
445 let dir = tempfile::tempdir().unwrap();
446 let ledger = MisreadLedger::new(dir.path());
447 for _ in 0..8 {
448 ledger.record(
449 &reading(Act::Answer, Stakes::Inert),
450 Tier::General,
451 vak_intent::RESOLVER_VERSION,
452 Outcome::Escalated,
453 Some("bash".into()),
454 false,
455 );
456 }
457 let cells = ledger.accuracy();
458 assert_eq!(cells[0].contradicted, 0);
459 assert_eq!(cells[0].unknown, 8);
460 assert!(weak_cells(&ledger, 5).is_empty());
461 }
462
463 /// A lexicon is judged only on its own readings: rows an earlier reader
464 /// wrote say nothing about the one running now.
465 #[test]
466 fn an_earlier_lexicon_does_not_score_the_current_one() {
467 let dir = tempfile::tempdir().unwrap();
468 let ledger = MisreadLedger::new(dir.path());
469 for _ in 0..8 {
470 ledger.record(
471 &reading(Act::Modify, Stakes::Reversible),
472 Tier::Signals,
473 vak_intent::RESOLVER_VERSION - 1,
474 Outcome::Escalated,
475 Some("bash".into()),
476 true,
477 );
478 }
479 assert_eq!(ledger.accuracy().len(), 1);
480 assert!(weak_cells(&ledger, 5).is_empty());
481 }
482
483 #[test]
484 fn corrupt_rows_are_skipped_rather_than_trusted() {
485 let dir = tempfile::tempdir().unwrap();
486 let ledger = MisreadLedger::new(dir.path());
487 record(
488 &ledger,
489 Act::Modify,
490 Stakes::Reversible,
491 Outcome::Held,
492 None,
493 );
494 let mut file = std::fs::OpenOptions::new()
495 .append(true)
496 .open(ledger.path())
497 .unwrap();
498 writeln!(file, "not json at all").unwrap();
499 file.write_all(b"{\"torn\": \xff\n").unwrap();
500 drop(file);
501 // A bad line never ends the read: the row after it still counts.
502 record(
503 &ledger,
504 Act::Modify,
505 Stakes::Reversible,
506 Outcome::Held,
507 None,
508 );
509 assert_eq!(ledger.rows().len(), 2);
510 }
511}
512

Indexing the workspace…