1. 1//! Finding and reading the `vak` fences in an assistant's answer.
2. 2//!
3. 3//! A `vak` fence is the model writing a card envelope inline. These helpers
4. 4//! find them the way a Markdown renderer does (so an indented fence, a `~~~`
5. 5//! fence, and a `vak` block quoted inside a longer fence are all handled
6. 6//! correctly) and read `semantic_type` as a JSON field, not as a substring of
7. 7//! the text.
8. 8
9. 9use pulldown_cmark::{CodeBlockKind, Event, Parser, Tag, TagEnd};
10. 10
11. 11/// The body of every fenced block whose info string is `vak`, in order. An
12. 12/// unclosed fence runs to the end of the text, exactly as a renderer would
13. 13/// treat it — which is what an answer cut off mid-card looks like.
14. 14pub(crate) fn vak_fence_bodies(text: &str) -> Vec<String> {
15. 15 let mut bodies = Vec::new();
16. 16 let mut current: Option<String> = None;
17. 17 for event in Parser::new(text) {
18. 18 match event {
19. 19 Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(info)))
20. 20 if info.split_whitespace().next() == Some("vak") =>
21. 21 {
22. 22 current = Some(String::new());
23. 23 }
24. 24 Event::Text(chunk) => {
25. 25 if let Some(body) = current.as_mut() {
26. 26 body.push_str(&chunk);
27. 27 }
28. 28 }
29. 29 Event::End(TagEnd::CodeBlock) => {
30. 30 if let Some(body) = current.take() {
31. 31 bodies.push(body);
32. 32 }
33. 33 }
34. 34 _ => {}
35. 35 }
36. 36 }
37. 37 bodies
38. 38}
39. 39
40. 40/// The parse error of the first `vak` fence whose body is not valid JSON. A
41. 41/// fence tagged `vak` is by definition an attempt at an envelope, so no
42. 42/// further guess is made about whether it "looks like" one.
43. 43pub(crate) fn find_malformed_vak_fence(text: &str) -> Option<String> {
44. 44 vak_fence_bodies(text).into_iter().find_map(|body| {
45. 45 serde_json::from_str::<serde_json::Value>(body.trim())
46. 46 .err()
47. 47 .map(|error| error.to_string())
48. 48 })
49. 49}
50. 50
51. 51/// The `semantic_type` of the first `vak` fence that repeats a card already
52. 52/// emitted through a tool call this run, if any. Compares the parsed
53. 53/// `semantic_type` field; a card whose payload merely mentions another type
54. 54/// in a string does not count.
55. 55pub(crate) fn find_duplicate_card_fence(text: &str, emitted_types: &[String]) -> Option<String> {
56. 56 vak_fence_bodies(text).into_iter().find_map(|body| {
57. 57 let value: serde_json::Value = serde_json::from_str(body.trim()).ok()?;
58. 58 let declared = value.get("semantic_type")?.as_str()?;
59. 59 emitted_types
60. 60 .iter()
61. 61 .find(|emitted| emitted.as_str() == declared)
62. 62 .cloned()
63. 63 })
64. 64}
65. 65
66. 66#[cfg(test)]
67. 67#[allow(clippy::unwrap_used)]
68. 68mod tests {
69. 69 use super::*;
70. 70
71. 71 const CHART: &str = r#"{"semantic_type":"chart","payload":{"series":[]}}"#;
72. 72
73. 73 fn fenced(body: &str) -> String {
74. 74 format!("Here you go.\n\n```vak\n{body}\n```\n")
75. 75 }
76. 76
77. 77 #[test]
78. 78 fn a_valid_fence_is_found_and_not_malformed() {
79. 79 let text = fenced(CHART);
80. 80 assert_eq!(vak_fence_bodies(&text).len(), 1);
81. 81 assert_eq!(find_malformed_vak_fence(&text), None);
82. 82 }
83. 83
84. 84 #[test]
85. 85 fn an_invalid_body_is_reported_with_its_parse_error() {
86. 86 let text = fenced(r#"{"semantic_type":"chart","payload":{"series":[}}"#);
87. 87 assert!(find_malformed_vak_fence(&text).is_some());
88. 88 }
89. 89
90. 90 #[test]
91. 91 fn an_answer_cut_off_mid_card_is_malformed() {
92. 92 let text = "Result:\n\n```vak\n{\"semantic_type\":\"chart\",\"payload\":{\"ser";
93. 93 assert!(
94. 94 find_malformed_vak_fence(text).is_some(),
95. 95 "unclosed fence is a truncated card"
96. 96 );
97. 97 }
98. 98
99. 99 #[test]
100. 100 fn a_vak_block_quoted_inside_a_longer_fence_is_not_a_fence() {
101. 101 let text = "Example of the format:\n\n````markdown\n```vak\n{not json\n```\n````\n";
102. 102 assert!(vak_fence_bodies(text).is_empty());
103. 103 assert_eq!(find_malformed_vak_fence(text), None);
104. 104 }
105. 105
106. 106 #[test]
107. 107 fn tilde_and_indented_fences_are_found_like_a_renderer_would() {
108. 108 let tilde = format!("~~~vak\n{CHART}\n~~~\n");
109. 109 assert_eq!(vak_fence_bodies(&tilde).len(), 1);
110. 110 let indented = format!("- item\n\n ```vak\n {CHART}\n ```\n");
111. 111 assert_eq!(vak_fence_bodies(&indented).len(), 1);
112. 112 }
113. 113
114. 114 #[test]
115. 115 fn other_languages_are_not_vak_fences() {
116. 116 let text = "```json\n{not json\n```\n";
117. 117 assert!(vak_fence_bodies(text).is_empty());
118. 118 }
119. 119
120. 120 #[test]
121. 121 fn a_duplicate_is_decided_by_the_parsed_type_not_by_a_substring() {
122. 122 let emitted = vec!["chart".to_string()];
123. 123 assert_eq!(
124. 124 find_duplicate_card_fence(&fenced(CHART), &emitted),
125. 125 Some("chart".into())
126. 126 );
127. 127 // A different card whose *text* mentions the chart envelope is not one.
128. 128 let metric = r#"{"semantic_type":"metric","payload":{"label":"say \"semantic_type\":\"chart\" here","value":1}}"#;
129. 129 assert_eq!(find_duplicate_card_fence(&fenced(metric), &emitted), None);
130. 130 assert_eq!(find_duplicate_card_fence("no fence at all", &emitted), None);
131. 131 }
132. 132}
133. 133