Vakyartha documentation

1//! Product-shaped sandbox exercises. These deliberately use the real BashTool
2//! contract so a passing unit test cannot hide a broken workbench workflow:
3//! a command works in the workspace, and what it creates is reported.
4
5#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
6
7use vak_tools::{Tool, ToolContext, bash::BashTool, sandbox_events::SandboxEventSink};
8
9async fn run(
10 workspace: &std::path::Path,
11 id: &str,
12 command: &str,
13) -> (vak_tools::ToolOutput, Vec<vak_tools::SandboxEvent>) {
14 let (sink, mut rx) = SandboxEventSink::new_with_id(id.into());
15 let ctx = ToolContext::new(workspace.to_path_buf()).with_sandbox_sink(sink);
16 let output = BashTool
17 .execute(&serde_json::json!({"command": command}), &ctx)
18 .await;
19 let mut events = Vec::new();
20 while let Ok(event) = rx.try_recv() {
21 events.push(event);
22 }
23 (output, events)
24}
25
26#[tokio::test]
27async fn full_stack_build_and_smoke_test_runs_in_the_workspace() {
28 let workspace = tempfile::tempdir().unwrap();
29 let command = r#"set -eu
30mkdir -p app
31cat > app/index.html <<'EOF'
32<!doctype html><html><body><h1>vak app</h1></body></html>
33EOF
34cat > app/server.py <<'EOF'
35from http.server import HTTPServer, SimpleHTTPRequestHandler
36HTTPServer(('127.0.0.1', 8765), SimpleHTTPRequestHandler).serve_forever()
37EOF
38python3 -m py_compile app/server.py
39python3 -m http.server 8765 --directory app >/tmp/vak-stack.log 2>&1 &
40server_pid=$!
41trap 'kill "$server_pid" 2>/dev/null || true' EXIT
42sleep 0.3
43python3 - <<'PY'
44from urllib.request import urlopen
45from pathlib import Path
46assert '<h1>vak app</h1>' in Path('app/index.html').read_text()
47assert Path('app/server.py').exists()
48assert b'<h1>vak app</h1>' in urlopen('http://127.0.0.1:8765', timeout=2).read()
49print('full-stack build and smoke test passed')
50PY
51"#;
52 let (output, events) = run(workspace.path(), "workflow-stack", command).await;
53 assert!(!output.is_error, "{}", output.content);
54 assert!(
55 output
56 .content
57 .contains("full-stack build and smoke test passed")
58 );
59 assert!(events.iter().any(|e| matches!(e, vak_tools::SandboxEvent::ArtifactGenerated { path, .. } if path.ends_with("app/index.html"))));
60 assert!(workspace.path().join("app/index.html").is_file());
61}
62
63#[tokio::test]
64async fn research_bundle_validates_sources_and_citations() {
65 let workspace = tempfile::tempdir().unwrap();
66 let command = r#"set -eu
67mkdir -p research
68cat > research/sources.json <<'EOF'
69[{"title":"Rust Book","url":"https://doc.rust-lang.org/book/"},{"title":"HTTP Semantics","url":"https://httpwg.org/specs/"}]
70EOF
71cat > research/synthesis.md <<'EOF'
72# Findings
73
74Rust provides memory-safe systems programming [1]. HTTP semantics define interoperable requests [2].
75
76## Sources
771. https://doc.rust-lang.org/book/
782. https://httpwg.org/specs/
79EOF
80python3 - <<'PY'
81import json
82from pathlib import Path
83sources=json.loads(Path('research/sources.json').read_text())
84text=Path('research/synthesis.md').read_text()
85assert len(sources)==2
86assert all(item['url'].startswith('https://') for item in sources)
87assert '[1]' in text and '[2]' in text
88print('research evidence and citations verified')
89PY
90"#;
91 let (output, events) = run(workspace.path(), "workflow-research", command).await;
92 assert!(!output.is_error, "{}", output.content);
93 assert!(
94 output
95 .content
96 .contains("research evidence and citations verified")
97 );
98 assert!(events.iter().any(|e| matches!(e, vak_tools::SandboxEvent::ArtifactGenerated { path, .. } if path.ends_with("research/synthesis.md"))));
99 assert!(workspace.path().join("research/synthesis.md").is_file());
100}
101
102#[tokio::test]
103async fn document_and_data_outputs_are_reported_as_artifacts() {
104 let workspace = tempfile::tempdir().unwrap();
105 let command = r#"set -eu
106mkdir -p deliverable
107cat > deliverable/report.md <<'EOF'
108# Quarterly report
109
110Generated from the verified task environment.
111EOF
112cat > deliverable/metrics.csv <<'EOF'
113metric,value
114latency_ms,42
115success_rate,0.99
116EOF
117python3 - <<'PY'
118from pathlib import Path
119assert Path('deliverable/report.md').read_text().startswith('# Quarterly report')
120rows=Path('deliverable/metrics.csv').read_text().splitlines()
121assert rows[0]=='metric,value' and len(rows)==3
122print('document and data outputs validated')
123PY
124"#;
125 let (output, events) = run(workspace.path(), "workflow-docs", command).await;
126 assert!(!output.is_error, "{}", output.content);
127 assert!(
128 output
129 .content
130 .contains("document and data outputs validated")
131 );
132 let artifacts: Vec<_> = events
133 .iter()
134 .filter_map(|event| match event {
135 vak_tools::SandboxEvent::ArtifactGenerated { path, .. } => Some(path.as_str()),
136 _ => None,
137 })
138 .collect();
139 assert!(
140 artifacts
141 .iter()
142 .any(|path| path.ends_with("deliverable/report.md"))
143 );
144 assert!(
145 artifacts
146 .iter()
147 .any(|path| path.ends_with("deliverable/metrics.csv"))
148 );
149 assert!(workspace.path().join("deliverable/report.md").is_file());
150}
151

1//! Product-shaped sandbox exercises. These deliberately use the real BashTool
2//! contract so a passing unit test cannot hide a broken workbench workflow:
3//! a command works in the workspace, and what it creates is reported.
4
5#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
6
7use vak_tools::{Tool, ToolContext, bash::BashTool, sandbox_events::SandboxEventSink};
8
9async fn run(
10 workspace: &std::path::Path,
11 id: &str,
12 command: &str,
13) -> (vak_tools::ToolOutput, Vec<vak_tools::SandboxEvent>) {
14 let (sink, mut rx) = SandboxEventSink::new_with_id(id.into());
15 let ctx = ToolContext::new(workspace.to_path_buf()).with_sandbox_sink(sink);
16 let output = BashTool
17 .execute(&serde_json::json!({"command": command}), &ctx)
18 .await;
19 let mut events = Vec::new();
20 while let Ok(event) = rx.try_recv() {
21 events.push(event);
22 }
23 (output, events)
24}
25
26#[tokio::test]
27async fn full_stack_build_and_smoke_test_runs_in_the_workspace() {
28 let workspace = tempfile::tempdir().unwrap();
29 let command = r#"set -eu
30mkdir -p app
31cat > app/index.html <<'EOF'
32<!doctype html><html><body><h1>vak app</h1></body></html>
33EOF
34cat > app/server.py <<'EOF'
35from http.server import HTTPServer, SimpleHTTPRequestHandler
36HTTPServer(('127.0.0.1', 8765), SimpleHTTPRequestHandler).serve_forever()
37EOF
38python3 -m py_compile app/server.py
39python3 -m http.server 8765 --directory app >/tmp/vak-stack.log 2>&1 &
40server_pid=$!
41trap 'kill "$server_pid" 2>/dev/null || true' EXIT
42sleep 0.3
43python3 - <<'PY'
44from urllib.request import urlopen
45from pathlib import Path
46assert '<h1>vak app</h1>' in Path('app/index.html').read_text()
47assert Path('app/server.py').exists()
48assert b'<h1>vak app</h1>' in urlopen('http://127.0.0.1:8765', timeout=2).read()
49print('full-stack build and smoke test passed')
50PY
51"#;
52 let (output, events) = run(workspace.path(), "workflow-stack", command).await;
53 assert!(!output.is_error, "{}", output.content);
54 assert!(
55 output
56 .content
57 .contains("full-stack build and smoke test passed")
58 );
59 assert!(events.iter().any(|e| matches!(e, vak_tools::SandboxEvent::ArtifactGenerated { path, .. } if path.ends_with("app/index.html"))));
60 assert!(workspace.path().join("app/index.html").is_file());
61}
62
63#[tokio::test]
64async fn research_bundle_validates_sources_and_citations() {
65 let workspace = tempfile::tempdir().unwrap();
66 let command = r#"set -eu
67mkdir -p research
68cat > research/sources.json <<'EOF'
69[{"title":"Rust Book","url":"https://doc.rust-lang.org/book/"},{"title":"HTTP Semantics","url":"https://httpwg.org/specs/"}]
70EOF
71cat > research/synthesis.md <<'EOF'
72# Findings
73
74Rust provides memory-safe systems programming [1]. HTTP semantics define interoperable requests [2].
75
76## Sources
771. https://doc.rust-lang.org/book/
782. https://httpwg.org/specs/
79EOF
80python3 - <<'PY'
81import json
82from pathlib import Path
83sources=json.loads(Path('research/sources.json').read_text())
84text=Path('research/synthesis.md').read_text()
85assert len(sources)==2
86assert all(item['url'].startswith('https://') for item in sources)
87assert '[1]' in text and '[2]' in text
88print('research evidence and citations verified')
89PY
90"#;
91 let (output, events) = run(workspace.path(), "workflow-research", command).await;
92 assert!(!output.is_error, "{}", output.content);
93 assert!(
94 output
95 .content
96 .contains("research evidence and citations verified")
97 );
98 assert!(events.iter().any(|e| matches!(e, vak_tools::SandboxEvent::ArtifactGenerated { path, .. } if path.ends_with("research/synthesis.md"))));
99 assert!(workspace.path().join("research/synthesis.md").is_file());
100}
101
102#[tokio::test]
103async fn document_and_data_outputs_are_reported_as_artifacts() {
104 let workspace = tempfile::tempdir().unwrap();
105 let command = r#"set -eu
106mkdir -p deliverable
107cat > deliverable/report.md <<'EOF'
108# Quarterly report
109
110Generated from the verified task environment.
111EOF
112cat > deliverable/metrics.csv <<'EOF'
113metric,value
114latency_ms,42
115success_rate,0.99
116EOF
117python3 - <<'PY'
118from pathlib import Path
119assert Path('deliverable/report.md').read_text().startswith('# Quarterly report')
120rows=Path('deliverable/metrics.csv').read_text().splitlines()
121assert rows[0]=='metric,value' and len(rows)==3
122print('document and data outputs validated')
123PY
124"#;
125 let (output, events) = run(workspace.path(), "workflow-docs", command).await;
126 assert!(!output.is_error, "{}", output.content);
127 assert!(
128 output
129 .content
130 .contains("document and data outputs validated")
131 );
132 let artifacts: Vec<_> = events
133 .iter()
134 .filter_map(|event| match event {
135 vak_tools::SandboxEvent::ArtifactGenerated { path, .. } => Some(path.as_str()),
136 _ => None,
137 })
138 .collect();
139 assert!(
140 artifacts
141 .iter()
142 .any(|path| path.ends_with("deliverable/report.md"))
143 );
144 assert!(
145 artifacts
146 .iter()
147 .any(|path| path.ends_with("deliverable/metrics.csv"))
148 );
149 assert!(workspace.path().join("deliverable/report.md").is_file());
150}
151

Indexing the workspace…