Vakyartha documentation

1//! Gemini Live API (`BidiGenerateContent`) session wrapper for
2//! text-to-speech voice synthesis. Mirrors `google.rs`'s conventions
3//! (config shape, `x-goog-api-key` auth, `map_status_error`-style error
4//! mapping, `tokio::select!` cancellation) but speaks raw WebSocket frames
5//! instead of SSE, since the Live API is bidirectional.
6//!
7//! Wire quirks handled here: the Live API takes a JSON `setup` message
8//! first, then JSON `clientContent` turns, and streams back JSON
9//! `serverContent` messages carrying base64 PCM audio in
10//! `modelTurn.parts[].inlineData.data` until `turnComplete`. Audio comes
11//! back as raw 24kHz/16-bit/mono PCM, which we wrap in a WAV container
12//! before returning it — nothing downstream should have to know the wire
13//! format.
14
15use std::time::Duration;
16
17use futures::{SinkExt, StreamExt};
18use serde_json::{Value, json};
19use tokio_tungstenite::tungstenite::Message;
20use tokio_util::sync::CancellationToken;
21
22use crate::error::LlmError;
23
24const LIVE_WS_HOST: &str = "generativelanguage.googleapis.com";
25const LIVE_WS_PATH: &str =
26 "/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
27
28/// Overall wall-clock budget for one `speak()` call — connect, setup
29/// round-trip, and audio collection combined. Nothing upstream of this
30/// module enforces a deadline (the `CancellationToken` passed in is only
31/// ever caller-triggered, never time-based), so without this a stalled
32/// socket — one that neither errors nor closes — would block the request
33/// task forever. Generous enough for a one-sentence TTS turn over a slow
34/// connection; short enough that a hung request can't tie up a task
35/// indefinitely.
36const LIVE_SESSION_TIMEOUT: Duration = Duration::from_secs(25);
37/// Some native-audio model revisions emit the complete audio turn but omit
38/// `turnComplete`. Once audio has arrived, a short quiet window is therefore
39/// sufficient to finalize the response while still allowing trailing chunks.
40const AUDIO_IDLE_TIMEOUT: Duration = Duration::from_secs(3);
41
42/// Hard cap on synthesis input length. This is a paid, per-call API; an
43/// unbounded `text` field lets any bearer-authenticated caller run up
44/// billing (or hang the session far longer than `LIVE_SESSION_TIMEOUT`
45/// allows for) with a single oversized request. Real callers — short bot
46/// replies, one-line narration cues — sit nowhere near this.
47pub const MAX_SPEAK_TEXT_CHARS: usize = 2_000;
48
49/// Batch speech-to-text through Gemini's multimodal generateContent API.
50/// The audio bytes are caller-bounded; the provider returns plain transcript
51/// text so the voice session can feed it into the governed turn path.
52pub async fn transcribe(
53 config: &GoogleLiveConfig,
54 audio: &[u8],
55 mime: &str,
56 cancel: &CancellationToken,
57) -> Result<String, LlmError> {
58 if audio.is_empty() || mime.trim().is_empty() {
59 return Err(LlmError::InvalidRequest(
60 "audio and mime are required".into(),
61 ));
62 }
63 if cancel.is_cancelled() {
64 return Err(LlmError::Aborted { partial: None });
65 }
66 let body = build_transcribe_request(audio, mime);
67 if config.model.trim().is_empty() {
68 return Err(LlmError::InvalidRequest(
69 "a discovered Gemini model is required".into(),
70 ));
71 }
72 let url = format!(
73 "https://{LIVE_WS_HOST}/v1beta/models/{}:generateContent",
74 config.model
75 );
76 let response = tokio::select! {
77 _ = cancel.cancelled() => return Err(LlmError::Aborted { partial: None }),
78 result = reqwest::Client::new().post(url).query(&[("key", &config.api_key)]).json(&body).send() => result.map_err(|e| LlmError::Network(e.to_string()))?,
79 };
80 let status = response.status().as_u16();
81 let value: Value = response
82 .json()
83 .await
84 .map_err(|e| LlmError::Parse(e.to_string()))?;
85 if status >= 400 {
86 return Err(map_status_error(status, &value.to_string()));
87 }
88 let text = value
89 .pointer("/candidates/0/content/parts/0/text")
90 .and_then(Value::as_str)
91 .unwrap_or("")
92 .trim()
93 .to_string();
94 if text.is_empty() {
95 return Err(LlmError::InvalidRequest(
96 "provider returned an empty transcript".into(),
97 ));
98 }
99 Ok(text)
100}
101
102fn build_transcribe_request(audio: &[u8], mime: &str) -> Value {
103 use base64::Engine as _;
104 json!({"contents":[{"parts":[
105 {"inline_data":{"mime_type":mime,"data":base64::engine::general_purpose::STANDARD.encode(audio)}},
106 {"text":"Transcribe this audio exactly. Return only the spoken words, without commentary."}
107 ]}]})
108}
109
110/// Live API output is always 24kHz, 16-bit, mono PCM (scratchpad-validated
111/// against the real API — see test_gemini_live.py's `SAMPLE_RATE_OUT`).
112const OUTPUT_SAMPLE_RATE_HZ: u32 = 24_000;
113
114#[derive(Debug, Clone)]
115pub struct GoogleLiveConfig {
116 pub api_key: String,
117 pub model: String,
118}
119
120impl GoogleLiveConfig {
121 pub fn new(api_key: impl Into<String>, model: impl Into<String>) -> Self {
122 GoogleLiveConfig {
123 api_key: api_key.into(),
124 model: model.into(),
125 }
126 }
127}
128
129fn map_status_error(status: u16, body: &str) -> LlmError {
130 let message = serde_json::from_str::<Value>(body)
131 .ok()
132 .and_then(|v| {
133 v.pointer("/error/message")
134 .and_then(|m| m.as_str().map(String::from))
135 })
136 .unwrap_or_else(|| body.chars().take(500).collect());
137 match status {
138 401 | 403 => LlmError::Auth(message),
139 400 | 404 | 413 | 422 => LlmError::InvalidRequest(message),
140 429 => LlmError::RateLimit {
141 message,
142 retry_after_secs: None,
143 },
144 503 | 529 => LlmError::Overloaded(message),
145 _ => LlmError::Api { status, message },
146 }
147}
148
149/// Build the `setup` message's `generationConfig`/`speechConfig` payload.
150/// Pure and unit-testable, mirroring `google::build_body`. `persona`
151/// becomes the session's `systemInstruction`; `voice_name` selects the
152/// provider-discovered prebuilt Live voice. When absent, the provider chooses
153/// its configured default; the harness never invents a voice identifier.
154pub fn build_live_config(persona: Option<&str>, voice_name: Option<&str>) -> Value {
155 let mut generation_config = json!({"responseModalities": ["AUDIO"]});
156 if let Some(voice) = voice_name.filter(|v| !v.trim().is_empty()) {
157 generation_config["speechConfig"] = json!({"voiceConfig": {"prebuiltVoiceConfig": {
158 "voiceName": voice,
159 }}});
160 }
161 let mut setup = json!({ "generationConfig": generation_config });
162 if let Some(persona) = persona.filter(|p| !p.trim().is_empty()) {
163 setup["systemInstruction"] = json!({
164 "parts": [{ "text": persona }]
165 });
166 }
167 setup
168}
169
170fn build_setup_message(model: &str, persona: Option<&str>, voice_name: Option<&str>) -> Value {
171 let mut config = build_live_config(persona, voice_name);
172 config["model"] = json!(format!("models/{model}"));
173 json!({ "setup": config })
174}
175
176fn build_client_content(text: &str) -> Value {
177 json!({
178 "clientContent": {
179 "turns": [{ "role": "user", "parts": [{ "text": text }] }],
180 "turnComplete": true,
181 }
182 })
183}
184
185/// Wrap raw 24kHz/16-bit/mono PCM samples in a minimal WAV container by
186/// hand (no extra crate needed for a 44-byte canonical header).
187fn wrap_wav(pcm: &[u8]) -> Vec<u8> {
188 let channels: u16 = 1;
189 let bits_per_sample: u16 = 16;
190 let byte_rate = OUTPUT_SAMPLE_RATE_HZ * u32::from(channels) * u32::from(bits_per_sample) / 8;
191 let block_align = channels * bits_per_sample / 8;
192 let data_len = pcm.len() as u32;
193 let riff_len = 36 + data_len;
194
195 let mut out = Vec::with_capacity(44 + pcm.len());
196 out.extend_from_slice(b"RIFF");
197 out.extend_from_slice(&riff_len.to_le_bytes());
198 out.extend_from_slice(b"WAVE");
199 out.extend_from_slice(b"fmt ");
200 out.extend_from_slice(&16u32.to_le_bytes()); // fmt chunk size (PCM)
201 out.extend_from_slice(&1u16.to_le_bytes()); // audio format: PCM
202 out.extend_from_slice(&channels.to_le_bytes());
203 out.extend_from_slice(&OUTPUT_SAMPLE_RATE_HZ.to_le_bytes());
204 out.extend_from_slice(&byte_rate.to_le_bytes());
205 out.extend_from_slice(&block_align.to_le_bytes());
206 out.extend_from_slice(&bits_per_sample.to_le_bytes());
207 out.extend_from_slice(b"data");
208 out.extend_from_slice(&data_len.to_le_bytes());
209 out.extend_from_slice(pcm);
210 out
211}
212
213/// Open a Live session, send one text turn, collect the synthesized audio,
214/// and return it as WAV bytes. Races cancellation the same way
215/// `google::GoogleProvider::stream` does, and additionally enforces
216/// `LIVE_SESSION_TIMEOUT` as a wall-clock backstop — see that constant's
217/// doc comment for why a cancellation token alone isn't enough here.
218pub async fn speak(
219 config: &GoogleLiveConfig,
220 text: &str,
221 persona: Option<&str>,
222 voice_name: Option<&str>,
223 cancel: &CancellationToken,
224) -> Result<Vec<u8>, LlmError> {
225 if text.chars().count() > MAX_SPEAK_TEXT_CHARS {
226 return Err(LlmError::InvalidRequest(format!(
227 "text too long for voice synthesis: {} chars (max {MAX_SPEAK_TEXT_CHARS})",
228 text.chars().count()
229 )));
230 }
231 if config.model.ends_with("-tts") {
232 return speak_batch(config, text, persona, voice_name, cancel).await;
233 }
234 match tokio::time::timeout(
235 LIVE_SESSION_TIMEOUT,
236 speak_inner(config, text, persona, voice_name, cancel),
237 )
238 .await
239 {
240 Ok(result) => result,
241 Err(_) => Err(LlmError::Network(format!(
242 "live session timed out after {}s",
243 LIVE_SESSION_TIMEOUT.as_secs()
244 ))),
245 }
246}
247
248/// Generate speech with a discovered Gemini TTS model through generateContent.
249/// The Live socket path is reserved for models that actually support bidi turns.
250async fn speak_batch(
251 config: &GoogleLiveConfig,
252 text: &str,
253 persona: Option<&str>,
254 voice_name: Option<&str>,
255 cancel: &CancellationToken,
256) -> Result<Vec<u8>, LlmError> {
257 use base64::Engine as _;
258 let instruction = persona.filter(|p| !p.trim().is_empty());
259 let prompt = match instruction {
260 Some(persona) => format!("{persona}\n\nSay: {text}"),
261 None => text.to_string(),
262 };
263 let voice = voice_name
264 .filter(|v| !v.trim().is_empty())
265 .unwrap_or("Kore");
266 let body = json!({
267 "contents": [{"parts": [{"text": prompt}]}],
268 "generationConfig": {
269 "responseModalities": ["AUDIO"],
270 "speechConfig": {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": voice}}}
271 }
272 });
273 let url = format!(
274 "https://{LIVE_WS_HOST}/v1beta/models/{}:generateContent",
275 config.model
276 );
277 let response = tokio::select! {
278 _ = cancel.cancelled() => return Err(LlmError::Aborted { partial: None }),
279 result = reqwest::Client::new().post(url).header("x-goog-api-key", &config.api_key).json(&body).send() => result.map_err(|e| LlmError::Network(e.to_string()))?,
280 };
281 let status = response.status().as_u16();
282 let value: Value = tokio::select! {
283 _ = cancel.cancelled() => return Err(LlmError::Aborted { partial: None }),
284 result = response.json() => result.map_err(|e| LlmError::Parse(e.to_string()))?,
285 };
286 if status >= 400 {
287 return Err(map_status_error(status, &value.to_string()));
288 }
289 let encoded = value
290 .pointer("/candidates/0/content/parts/0/inlineData/data")
291 .and_then(Value::as_str)
292 .ok_or_else(|| LlmError::Parse("TTS response contained no audio".into()))?;
293 let pcm = base64::engine::general_purpose::STANDARD
294 .decode(encoded)
295 .map_err(|e| LlmError::Parse(format!("invalid TTS audio: {e}")))?;
296 if pcm.is_empty() {
297 return Err(LlmError::Parse("TTS response contained empty audio".into()));
298 }
299 Ok(wrap_wav(&pcm))
300}
301
302async fn speak_inner(
303 config: &GoogleLiveConfig,
304 text: &str,
305 persona: Option<&str>,
306 voice_name: Option<&str>,
307 cancel: &CancellationToken,
308) -> Result<Vec<u8>, LlmError> {
309 let _ =
310 rustls::crypto::CryptoProvider::install_default(rustls::crypto::ring::default_provider());
311 let url = format!("wss://{LIVE_WS_HOST}{LIVE_WS_PATH}?key={}", config.api_key);
312
313 let connect_fut = tokio_tungstenite::connect_async(&url);
314 let (mut ws, _resp) = tokio::select! {
315 _ = cancel.cancelled() => return Err(LlmError::Aborted { partial: None }),
316 r = connect_fut => match r {
317 Ok(pair) => pair,
318 Err(tokio_tungstenite::tungstenite::Error::Http(resp)) => {
319 let status = resp.status().as_u16();
320 let body = resp
321 .body()
322 .as_ref()
323 .map(|b| String::from_utf8_lossy(b).to_string())
324 .unwrap_or_default();
325 return Err(map_status_error(status, &body));
326 }
327 Err(e) => return Err(LlmError::Network(e.to_string())),
328 },
329 };
330
331 let setup = build_setup_message(&config.model, persona, voice_name);
332 let send_setup = ws.send(Message::Text(setup.to_string()));
333 tokio::select! {
334 _ = cancel.cancelled() => return Err(LlmError::Aborted { partial: None }),
335 r = send_setup => r.map_err(|e| LlmError::Network(e.to_string()))?,
336 };
337
338 // Wait for the server's `setupComplete` acknowledgement before sending
339 // the turn — the Live API requires setup to round-trip first.
340 loop {
341 let next = ws.next();
342 let msg = tokio::select! {
343 _ = cancel.cancelled() => return Err(LlmError::Aborted { partial: None }),
344 m = next => m,
345 };
346 match msg {
347 Some(Ok(Message::Text(t))) => {
348 let v: Value = serde_json::from_str(&t)
349 .map_err(|e| LlmError::Parse(format!("bad setup response: {e}")))?;
350 if v.get("setupComplete").is_some() {
351 break;
352 }
353 if let Some(err) = v.get("error") {
354 return Err(LlmError::Api {
355 status: 0,
356 message: err.to_string(),
357 });
358 }
359 }
360 Some(Ok(Message::Binary(bytes))) => {
361 let v: Value = serde_json::from_slice(&bytes)
362 .map_err(|e| LlmError::Parse(format!("bad setup response: {e}")))?;
363 if v.get("setupComplete").is_some() {
364 break;
365 }
366 if let Some(err) = v.get("error") {
367 return Err(LlmError::Api {
368 status: 0,
369 message: err.to_string(),
370 });
371 }
372 }
373 Some(Ok(Message::Close(frame))) => {
374 let reason = frame.map(|f| f.reason.to_string()).unwrap_or_default();
375 return Err(LlmError::Network(format!(
376 "live session closed during setup: {reason}"
377 )));
378 }
379 Some(Ok(_)) => continue,
380 Some(Err(e)) => return Err(LlmError::Network(e.to_string())),
381 None => {
382 return Err(LlmError::Network(
383 "live session closed before setupComplete".into(),
384 ));
385 }
386 }
387 }
388
389 let turn = build_client_content(text);
390 let send_turn = ws.send(Message::Text(turn.to_string()));
391 tokio::select! {
392 _ = cancel.cancelled() => return Err(LlmError::Aborted { partial: None }),
393 r = send_turn => r.map_err(|e| LlmError::Network(e.to_string()))?,
394 };
395
396 let mut pcm = Vec::new();
397 loop {
398 let next = ws.next();
399 let msg = tokio::select! {
400 _ = cancel.cancelled() => {
401 let partial = (!pcm.is_empty())
402 .then(|| Box::new(crate::types::AssistantMessage::empty(&config.model)));
403 return Err(LlmError::Aborted { partial });
404 }
405 m = async {
406 if pcm.is_empty() {
407 next.await
408 } else {
409 tokio::time::timeout(AUDIO_IDLE_TIMEOUT, next)
410 .await
411 .unwrap_or_default()
412 }
413 } => m,
414 };
415 match msg {
416 Some(Ok(Message::Text(t))) => {
417 let v: Value = serde_json::from_str(&t)
418 .map_err(|e| LlmError::Parse(format!("bad server message: {e}")))?;
419 if let Some(err) = v.get("error") {
420 return Err(LlmError::Api {
421 status: 0,
422 message: err.to_string(),
423 });
424 }
425 let Some(sc) = v.get("serverContent") else {
426 continue;
427 };
428 if let Some(parts) = sc.pointer("/modelTurn/parts").and_then(|p| p.as_array()) {
429 for part in parts {
430 if let Some(data) =
431 part.pointer("/inlineData/data").and_then(|d| d.as_str())
432 {
433 use base64::Engine as _;
434 match base64::engine::general_purpose::STANDARD.decode(data) {
435 Ok(bytes) => pcm.extend_from_slice(&bytes),
436 Err(e) => {
437 return Err(LlmError::Parse(format!(
438 "bad base64 audio chunk: {e}"
439 )));
440 }
441 }
442 }
443 }
444 }
445 if sc
446 .get("turnComplete")
447 .and_then(|b| b.as_bool())
448 .unwrap_or(false)
449 {
450 break;
451 }
452 }
453 Some(Ok(Message::Binary(bytes))) => {
454 let v: Value = serde_json::from_slice(&bytes)
455 .map_err(|e| LlmError::Parse(format!("bad server message: {e}")))?;
456 if let Some(err) = v.get("error") {
457 return Err(LlmError::Api {
458 status: 0,
459 message: err.to_string(),
460 });
461 }
462 if let Some(sc) = v.get("serverContent") {
463 if let Some(parts) = sc.pointer("/modelTurn/parts").and_then(|p| p.as_array()) {
464 for part in parts {
465 if let Some(data) =
466 part.pointer("/inlineData/data").and_then(|d| d.as_str())
467 {
468 use base64::Engine as _;
469 let bytes = base64::engine::general_purpose::STANDARD
470 .decode(data)
471 .map_err(|e| {
472 LlmError::Parse(format!("bad base64 audio chunk: {e}"))
473 })?;
474 pcm.extend_from_slice(&bytes);
475 }
476 }
477 }
478 if sc
479 .get("turnComplete")
480 .and_then(Value::as_bool)
481 .unwrap_or(false)
482 {
483 break;
484 }
485 }
486 }
487 Some(Ok(Message::Close(_))) => break,
488 Some(Ok(_)) => continue,
489 Some(Err(e)) => return Err(LlmError::Network(e.to_string())),
490 None => break,
491 }
492 }
493
494 let _ = ws.close(None).await;
495
496 if pcm.is_empty() {
497 return Err(LlmError::Parse("no audio returned by live session".into()));
498 }
499
500 Ok(wrap_wav(&pcm))
501}
502
503#[cfg(test)]
504mod tests {
505 #![allow(clippy::unwrap_used, clippy::expect_used)]
506 use super::*;
507
508 #[test]
509 fn build_live_config_omits_unspecified_voice_without_persona() {
510 let cfg = build_live_config(None, None);
511 assert!(cfg.pointer("/generationConfig/speechConfig").is_none());
512 assert_eq!(
513 cfg.pointer("/generationConfig/responseModalities/0")
514 .and_then(|v| v.as_str()),
515 Some("AUDIO")
516 );
517 assert!(cfg.get("systemInstruction").is_none());
518 }
519
520 #[test]
521 fn build_live_config_honors_voice_name_and_persona() {
522 let cfg = build_live_config(Some("warm and upbeat"), Some("Puck"));
523 assert_eq!(
524 cfg.pointer("/generationConfig/speechConfig/voiceConfig/prebuiltVoiceConfig/voiceName")
525 .and_then(|v| v.as_str()),
526 Some("Puck")
527 );
528 assert_eq!(
529 cfg.pointer("/systemInstruction/parts/0/text")
530 .and_then(|v| v.as_str()),
531 Some("warm and upbeat")
532 );
533 }
534
535 #[test]
536 fn build_live_config_ignores_blank_persona_and_voice() {
537 let cfg = build_live_config(Some(" "), Some(""));
538 assert!(cfg.get("systemInstruction").is_none());
539 assert!(cfg.pointer("/generationConfig/speechConfig").is_none());
540 }
541
542 #[tokio::test]
543 async fn speak_rejects_text_over_the_length_cap() {
544 let config = GoogleLiveConfig::new("test-key-not-used", "discovered-model");
545 let text: String = "a".repeat(MAX_SPEAK_TEXT_CHARS + 1);
546 let cancel = CancellationToken::new();
547 let err = speak(&config, &text, None, None, &cancel)
548 .await
549 .unwrap_err();
550 assert!(matches!(err, LlmError::InvalidRequest(_)));
551 }
552
553 #[tokio::test]
554 async fn transcribe_rejects_cancelled_request_without_network() {
555 let config = GoogleLiveConfig::new("test-key-not-used", "discovered-model");
556 let cancel = CancellationToken::new();
557 cancel.cancel();
558 let err = transcribe(&config, b"audio", "audio/pcm", &cancel)
559 .await
560 .unwrap_err();
561 assert!(matches!(err, LlmError::Aborted { .. }));
562 }
563
564 #[test]
565 fn provider_status_errors_preserve_classification_and_message() {
566 assert!(
567 matches!(map_status_error(401, r#"{"error":{"message":"bad key"}}"#), LlmError::Auth(message) if message == "bad key")
568 );
569 assert!(
570 matches!(map_status_error(429, r#"{"error":{"message":"slow down"}}"#), LlmError::RateLimit { message, .. } if message == "slow down")
571 );
572 assert!(
573 matches!(map_status_error(503, r#"{"error":{"message":"busy"}}"#), LlmError::Overloaded(message) if message == "busy")
574 );
575 }
576
577 #[test]
578 fn transcription_request_contains_audio_and_strict_instruction() {
579 let request = build_transcribe_request(&[0, 1, 2], "audio/pcm");
580 assert_eq!(
581 request
582 .pointer("/contents/0/parts/0/inline_data/mime_type")
583 .and_then(Value::as_str),
584 Some("audio/pcm")
585 );
586 assert!(
587 request
588 .pointer("/contents/0/parts/0/inline_data/data")
589 .and_then(Value::as_str)
590 .is_some()
591 );
592 assert!(
593 request
594 .pointer("/contents/0/parts/1/text")
595 .and_then(Value::as_str)
596 .unwrap_or_default()
597 .contains("only the spoken words")
598 );
599 }
600
601 /// Opt-in live smoke test. The credential is read only from the process
602 /// environment and is never printed or persisted.
603 #[tokio::test]
604 #[ignore = "requires GEMINI_API_KEY and network access"]
605 async fn live_smoke_synthesizes_audio() {
606 let Ok(key) = smoke_key() else {
607 return;
608 };
609 let Ok(model) = std::env::var("VAK_GEMINI_LIVE_MODEL") else {
610 return;
611 };
612 let config = GoogleLiveConfig::new(key, model);
613 let bytes = speak(
614 &config,
615 "Say hello briefly.",
616 None,
617 None,
618 &CancellationToken::new(),
619 )
620 .await
621 .expect("configured live provider should synthesize");
622 assert!(bytes.starts_with(b"RIFF"));
623 assert!(bytes.len() > 44);
624 }
625
626 #[tokio::test]
627 #[ignore = "requires GEMINI_API_KEY and network access"]
628 async fn live_smoke_transcribes_audio() {
629 let Ok(key) = smoke_key() else {
630 return;
631 };
632 let Ok(model) = std::env::var("VAK_GEMINI_TRANSCRIBE_MODEL") else {
633 return;
634 };
635 let config = GoogleLiveConfig::new(key, model);
636 let audio = vec![0u8; 3200];
637 let text = transcribe(&config, &audio, "audio/pcm", &CancellationToken::new())
638 .await
639 .expect("configured live provider should transcribe");
640 assert!(!text.trim().is_empty());
641 }
642
643 fn smoke_key() -> Result<String, std::env::VarError> {
644 std::env::var("GEMINI_API_KEY").or_else(|_| std::env::var("GOOGLE_API_KEY"))
645 }
646
647 #[test]
648 fn wrap_wav_produces_valid_riff_header() {
649 let pcm = vec![0u8, 1, 2, 3, 4, 5, 6, 7];
650 let wav = wrap_wav(&pcm);
651 assert_eq!(&wav[0..4], b"RIFF");
652 assert_eq!(&wav[8..12], b"WAVE");
653 assert_eq!(&wav[36..40], b"data");
654 assert_eq!(wav.len(), 44 + pcm.len());
655 }
656}
657

1//! Gemini Live API (`BidiGenerateContent`) session wrapper for
2//! text-to-speech voice synthesis. Mirrors `google.rs`'s conventions
3//! (config shape, `x-goog-api-key` auth, `map_status_error`-style error
4//! mapping, `tokio::select!` cancellation) but speaks raw WebSocket frames
5//! instead of SSE, since the Live API is bidirectional.
6//!
7//! Wire quirks handled here: the Live API takes a JSON `setup` message
8//! first, then JSON `clientContent` turns, and streams back JSON
9//! `serverContent` messages carrying base64 PCM audio in
10//! `modelTurn.parts[].inlineData.data` until `turnComplete`. Audio comes
11//! back as raw 24kHz/16-bit/mono PCM, which we wrap in a WAV container
12//! before returning it — nothing downstream should have to know the wire
13//! format.
14
15use std::time::Duration;
16
17use futures::{SinkExt, StreamExt};
18use serde_json::{Value, json};
19use tokio_tungstenite::tungstenite::Message;
20use tokio_util::sync::CancellationToken;
21
22use crate::error::LlmError;
23
24const LIVE_WS_HOST: &str = "generativelanguage.googleapis.com";
25const LIVE_WS_PATH: &str =
26 "/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
27
28/// Overall wall-clock budget for one `speak()` call — connect, setup
29/// round-trip, and audio collection combined. Nothing upstream of this
30/// module enforces a deadline (the `CancellationToken` passed in is only
31/// ever caller-triggered, never time-based), so without this a stalled
32/// socket — one that neither errors nor closes — would block the request
33/// task forever. Generous enough for a one-sentence TTS turn over a slow
34/// connection; short enough that a hung request can't tie up a task
35/// indefinitely.
36const LIVE_SESSION_TIMEOUT: Duration = Duration::from_secs(25);
37/// Some native-audio model revisions emit the complete audio turn but omit
38/// `turnComplete`. Once audio has arrived, a short quiet window is therefore
39/// sufficient to finalize the response while still allowing trailing chunks.
40const AUDIO_IDLE_TIMEOUT: Duration = Duration::from_secs(3);
41
42/// Hard cap on synthesis input length. This is a paid, per-call API; an
43/// unbounded `text` field lets any bearer-authenticated caller run up
44/// billing (or hang the session far longer than `LIVE_SESSION_TIMEOUT`
45/// allows for) with a single oversized request. Real callers — short bot
46/// replies, one-line narration cues — sit nowhere near this.
47pub const MAX_SPEAK_TEXT_CHARS: usize = 2_000;
48
49/// Batch speech-to-text through Gemini's multimodal generateContent API.
50/// The audio bytes are caller-bounded; the provider returns plain transcript
51/// text so the voice session can feed it into the governed turn path.
52pub async fn transcribe(
53 config: &GoogleLiveConfig,
54 audio: &[u8],
55 mime: &str,
56 cancel: &CancellationToken,
57) -> Result<String, LlmError> {
58 if audio.is_empty() || mime.trim().is_empty() {
59 return Err(LlmError::InvalidRequest(
60 "audio and mime are required".into(),
61 ));
62 }
63 if cancel.is_cancelled() {
64 return Err(LlmError::Aborted { partial: None });
65 }
66 let body = build_transcribe_request(audio, mime);
67 if config.model.trim().is_empty() {
68 return Err(LlmError::InvalidRequest(
69 "a discovered Gemini model is required".into(),
70 ));
71 }
72 let url = format!(
73 "https://{LIVE_WS_HOST}/v1beta/models/{}:generateContent",
74 config.model
75 );
76 let response = tokio::select! {
77 _ = cancel.cancelled() => return Err(LlmError::Aborted { partial: None }),
78 result = reqwest::Client::new().post(url).query(&[("key", &config.api_key)]).json(&body).send() => result.map_err(|e| LlmError::Network(e.to_string()))?,
79 };
80 let status = response.status().as_u16();
81 let value: Value = response
82 .json()
83 .await
84 .map_err(|e| LlmError::Parse(e.to_string()))?;
85 if status >= 400 {
86 return Err(map_status_error(status, &value.to_string()));
87 }
88 let text = value
89 .pointer("/candidates/0/content/parts/0/text")
90 .and_then(Value::as_str)
91 .unwrap_or("")
92 .trim()
93 .to_string();
94 if text.is_empty() {
95 return Err(LlmError::InvalidRequest(
96 "provider returned an empty transcript".into(),
97 ));
98 }
99 Ok(text)
100}
101
102fn build_transcribe_request(audio: &[u8], mime: &str) -> Value {
103 use base64::Engine as _;
104 json!({"contents":[{"parts":[
105 {"inline_data":{"mime_type":mime,"data":base64::engine::general_purpose::STANDARD.encode(audio)}},
106 {"text":"Transcribe this audio exactly. Return only the spoken words, without commentary."}
107 ]}]})
108}
109
110/// Live API output is always 24kHz, 16-bit, mono PCM (scratchpad-validated
111/// against the real API — see test_gemini_live.py's `SAMPLE_RATE_OUT`).
112const OUTPUT_SAMPLE_RATE_HZ: u32 = 24_000;
113
114#[derive(Debug, Clone)]
115pub struct GoogleLiveConfig {
116 pub api_key: String,
117 pub model: String,
118}
119
120impl GoogleLiveConfig {
121 pub fn new(api_key: impl Into<String>, model: impl Into<String>) -> Self {
122 GoogleLiveConfig {
123 api_key: api_key.into(),
124 model: model.into(),
125 }
126 }
127}
128
129fn map_status_error(status: u16, body: &str) -> LlmError {
130 let message = serde_json::from_str::<Value>(body)
131 .ok()
132 .and_then(|v| {
133 v.pointer("/error/message")
134 .and_then(|m| m.as_str().map(String::from))
135 })
136 .unwrap_or_else(|| body.chars().take(500).collect());
137 match status {
138 401 | 403 => LlmError::Auth(message),
139 400 | 404 | 413 | 422 => LlmError::InvalidRequest(message),
140 429 => LlmError::RateLimit {
141 message,
142 retry_after_secs: None,
143 },
144 503 | 529 => LlmError::Overloaded(message),
145 _ => LlmError::Api { status, message },
146 }
147}
148
149/// Build the `setup` message's `generationConfig`/`speechConfig` payload.
150/// Pure and unit-testable, mirroring `google::build_body`. `persona`
151/// becomes the session's `systemInstruction`; `voice_name` selects the
152/// provider-discovered prebuilt Live voice. When absent, the provider chooses
153/// its configured default; the harness never invents a voice identifier.
154pub fn build_live_config(persona: Option<&str>, voice_name: Option<&str>) -> Value {
155 let mut generation_config = json!({"responseModalities": ["AUDIO"]});
156 if let Some(voice) = voice_name.filter(|v| !v.trim().is_empty()) {
157 generation_config["speechConfig"] = json!({"voiceConfig": {"prebuiltVoiceConfig": {
158 "voiceName": voice,
159 }}});
160 }
161 let mut setup = json!({ "generationConfig": generation_config });
162 if let Some(persona) = persona.filter(|p| !p.trim().is_empty()) {
163 setup["systemInstruction"] = json!({
164 "parts": [{ "text": persona }]
165 });
166 }
167 setup
168}
169
170fn build_setup_message(model: &str, persona: Option<&str>, voice_name: Option<&str>) -> Value {
171 let mut config = build_live_config(persona, voice_name);
172 config["model"] = json!(format!("models/{model}"));
173 json!({ "setup": config })
174}
175
176fn build_client_content(text: &str) -> Value {
177 json!({
178 "clientContent": {
179 "turns": [{ "role": "user", "parts": [{ "text": text }] }],
180 "turnComplete": true,
181 }
182 })
183}
184
185/// Wrap raw 24kHz/16-bit/mono PCM samples in a minimal WAV container by
186/// hand (no extra crate needed for a 44-byte canonical header).
187fn wrap_wav(pcm: &[u8]) -> Vec<u8> {
188 let channels: u16 = 1;
189 let bits_per_sample: u16 = 16;
190 let byte_rate = OUTPUT_SAMPLE_RATE_HZ * u32::from(channels) * u32::from(bits_per_sample) / 8;
191 let block_align = channels * bits_per_sample / 8;
192 let data_len = pcm.len() as u32;
193 let riff_len = 36 + data_len;
194
195 let mut out = Vec::with_capacity(44 + pcm.len());
196 out.extend_from_slice(b"RIFF");
197 out.extend_from_slice(&riff_len.to_le_bytes());
198 out.extend_from_slice(b"WAVE");
199 out.extend_from_slice(b"fmt ");
200 out.extend_from_slice(&16u32.to_le_bytes()); // fmt chunk size (PCM)
201 out.extend_from_slice(&1u16.to_le_bytes()); // audio format: PCM
202 out.extend_from_slice(&channels.to_le_bytes());
203 out.extend_from_slice(&OUTPUT_SAMPLE_RATE_HZ.to_le_bytes());
204 out.extend_from_slice(&byte_rate.to_le_bytes());
205 out.extend_from_slice(&block_align.to_le_bytes());
206 out.extend_from_slice(&bits_per_sample.to_le_bytes());
207 out.extend_from_slice(b"data");
208 out.extend_from_slice(&data_len.to_le_bytes());
209 out.extend_from_slice(pcm);
210 out
211}
212
213/// Open a Live session, send one text turn, collect the synthesized audio,
214/// and return it as WAV bytes. Races cancellation the same way
215/// `google::GoogleProvider::stream` does, and additionally enforces
216/// `LIVE_SESSION_TIMEOUT` as a wall-clock backstop — see that constant's
217/// doc comment for why a cancellation token alone isn't enough here.
218pub async fn speak(
219 config: &GoogleLiveConfig,
220 text: &str,
221 persona: Option<&str>,
222 voice_name: Option<&str>,
223 cancel: &CancellationToken,
224) -> Result<Vec<u8>, LlmError> {
225 if text.chars().count() > MAX_SPEAK_TEXT_CHARS {
226 return Err(LlmError::InvalidRequest(format!(
227 "text too long for voice synthesis: {} chars (max {MAX_SPEAK_TEXT_CHARS})",
228 text.chars().count()
229 )));
230 }
231 if config.model.ends_with("-tts") {
232 return speak_batch(config, text, persona, voice_name, cancel).await;
233 }
234 match tokio::time::timeout(
235 LIVE_SESSION_TIMEOUT,
236 speak_inner(config, text, persona, voice_name, cancel),
237 )
238 .await
239 {
240 Ok(result) => result,
241 Err(_) => Err(LlmError::Network(format!(
242 "live session timed out after {}s",
243 LIVE_SESSION_TIMEOUT.as_secs()
244 ))),
245 }
246}
247
248/// Generate speech with a discovered Gemini TTS model through generateContent.
249/// The Live socket path is reserved for models that actually support bidi turns.
250async fn speak_batch(
251 config: &GoogleLiveConfig,
252 text: &str,
253 persona: Option<&str>,
254 voice_name: Option<&str>,
255 cancel: &CancellationToken,
256) -> Result<Vec<u8>, LlmError> {
257 use base64::Engine as _;
258 let instruction = persona.filter(|p| !p.trim().is_empty());
259 let prompt = match instruction {
260 Some(persona) => format!("{persona}\n\nSay: {text}"),
261 None => text.to_string(),
262 };
263 let voice = voice_name
264 .filter(|v| !v.trim().is_empty())
265 .unwrap_or("Kore");
266 let body = json!({
267 "contents": [{"parts": [{"text": prompt}]}],
268 "generationConfig": {
269 "responseModalities": ["AUDIO"],
270 "speechConfig": {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": voice}}}
271 }
272 });
273 let url = format!(
274 "https://{LIVE_WS_HOST}/v1beta/models/{}:generateContent",
275 config.model
276 );
277 let response = tokio::select! {
278 _ = cancel.cancelled() => return Err(LlmError::Aborted { partial: None }),
279 result = reqwest::Client::new().post(url).header("x-goog-api-key", &config.api_key).json(&body).send() => result.map_err(|e| LlmError::Network(e.to_string()))?,
280 };
281 let status = response.status().as_u16();
282 let value: Value = tokio::select! {
283 _ = cancel.cancelled() => return Err(LlmError::Aborted { partial: None }),
284 result = response.json() => result.map_err(|e| LlmError::Parse(e.to_string()))?,
285 };
286 if status >= 400 {
287 return Err(map_status_error(status, &value.to_string()));
288 }
289 let encoded = value
290 .pointer("/candidates/0/content/parts/0/inlineData/data")
291 .and_then(Value::as_str)
292 .ok_or_else(|| LlmError::Parse("TTS response contained no audio".into()))?;
293 let pcm = base64::engine::general_purpose::STANDARD
294 .decode(encoded)
295 .map_err(|e| LlmError::Parse(format!("invalid TTS audio: {e}")))?;
296 if pcm.is_empty() {
297 return Err(LlmError::Parse("TTS response contained empty audio".into()));
298 }
299 Ok(wrap_wav(&pcm))
300}
301
302async fn speak_inner(
303 config: &GoogleLiveConfig,
304 text: &str,
305 persona: Option<&str>,
306 voice_name: Option<&str>,
307 cancel: &CancellationToken,
308) -> Result<Vec<u8>, LlmError> {
309 let _ =
310 rustls::crypto::CryptoProvider::install_default(rustls::crypto::ring::default_provider());
311 let url = format!("wss://{LIVE_WS_HOST}{LIVE_WS_PATH}?key={}", config.api_key);
312
313 let connect_fut = tokio_tungstenite::connect_async(&url);
314 let (mut ws, _resp) = tokio::select! {
315 _ = cancel.cancelled() => return Err(LlmError::Aborted { partial: None }),
316 r = connect_fut => match r {
317 Ok(pair) => pair,
318 Err(tokio_tungstenite::tungstenite::Error::Http(resp)) => {
319 let status = resp.status().as_u16();
320 let body = resp
321 .body()
322 .as_ref()
323 .map(|b| String::from_utf8_lossy(b).to_string())
324 .unwrap_or_default();
325 return Err(map_status_error(status, &body));
326 }
327 Err(e) => return Err(LlmError::Network(e.to_string())),
328 },
329 };
330
331 let setup = build_setup_message(&config.model, persona, voice_name);
332 let send_setup = ws.send(Message::Text(setup.to_string()));
333 tokio::select! {
334 _ = cancel.cancelled() => return Err(LlmError::Aborted { partial: None }),
335 r = send_setup => r.map_err(|e| LlmError::Network(e.to_string()))?,
336 };
337
338 // Wait for the server's `setupComplete` acknowledgement before sending
339 // the turn — the Live API requires setup to round-trip first.
340 loop {
341 let next = ws.next();
342 let msg = tokio::select! {
343 _ = cancel.cancelled() => return Err(LlmError::Aborted { partial: None }),
344 m = next => m,
345 };
346 match msg {
347 Some(Ok(Message::Text(t))) => {
348 let v: Value = serde_json::from_str(&t)
349 .map_err(|e| LlmError::Parse(format!("bad setup response: {e}")))?;
350 if v.get("setupComplete").is_some() {
351 break;
352 }
353 if let Some(err) = v.get("error") {
354 return Err(LlmError::Api {
355 status: 0,
356 message: err.to_string(),
357 });
358 }
359 }
360 Some(Ok(Message::Binary(bytes))) => {
361 let v: Value = serde_json::from_slice(&bytes)
362 .map_err(|e| LlmError::Parse(format!("bad setup response: {e}")))?;
363 if v.get("setupComplete").is_some() {
364 break;
365 }
366 if let Some(err) = v.get("error") {
367 return Err(LlmError::Api {
368 status: 0,
369 message: err.to_string(),
370 });
371 }
372 }
373 Some(Ok(Message::Close(frame))) => {
374 let reason = frame.map(|f| f.reason.to_string()).unwrap_or_default();
375 return Err(LlmError::Network(format!(
376 "live session closed during setup: {reason}"
377 )));
378 }
379 Some(Ok(_)) => continue,
380 Some(Err(e)) => return Err(LlmError::Network(e.to_string())),
381 None => {
382 return Err(LlmError::Network(
383 "live session closed before setupComplete".into(),
384 ));
385 }
386 }
387 }
388
389 let turn = build_client_content(text);
390 let send_turn = ws.send(Message::Text(turn.to_string()));
391 tokio::select! {
392 _ = cancel.cancelled() => return Err(LlmError::Aborted { partial: None }),
393 r = send_turn => r.map_err(|e| LlmError::Network(e.to_string()))?,
394 };
395
396 let mut pcm = Vec::new();
397 loop {
398 let next = ws.next();
399 let msg = tokio::select! {
400 _ = cancel.cancelled() => {
401 let partial = (!pcm.is_empty())
402 .then(|| Box::new(crate::types::AssistantMessage::empty(&config.model)));
403 return Err(LlmError::Aborted { partial });
404 }
405 m = async {
406 if pcm.is_empty() {
407 next.await
408 } else {
409 tokio::time::timeout(AUDIO_IDLE_TIMEOUT, next)
410 .await
411 .unwrap_or_default()
412 }
413 } => m,
414 };
415 match msg {
416 Some(Ok(Message::Text(t))) => {
417 let v: Value = serde_json::from_str(&t)
418 .map_err(|e| LlmError::Parse(format!("bad server message: {e}")))?;
419 if let Some(err) = v.get("error") {
420 return Err(LlmError::Api {
421 status: 0,
422 message: err.to_string(),
423 });
424 }
425 let Some(sc) = v.get("serverContent") else {
426 continue;
427 };
428 if let Some(parts) = sc.pointer("/modelTurn/parts").and_then(|p| p.as_array()) {
429 for part in parts {
430 if let Some(data) =
431 part.pointer("/inlineData/data").and_then(|d| d.as_str())
432 {
433 use base64::Engine as _;
434 match base64::engine::general_purpose::STANDARD.decode(data) {
435 Ok(bytes) => pcm.extend_from_slice(&bytes),
436 Err(e) => {
437 return Err(LlmError::Parse(format!(
438 "bad base64 audio chunk: {e}"
439 )));
440 }
441 }
442 }
443 }
444 }
445 if sc
446 .get("turnComplete")
447 .and_then(|b| b.as_bool())
448 .unwrap_or(false)
449 {
450 break;
451 }
452 }
453 Some(Ok(Message::Binary(bytes))) => {
454 let v: Value = serde_json::from_slice(&bytes)
455 .map_err(|e| LlmError::Parse(format!("bad server message: {e}")))?;
456 if let Some(err) = v.get("error") {
457 return Err(LlmError::Api {
458 status: 0,
459 message: err.to_string(),
460 });
461 }
462 if let Some(sc) = v.get("serverContent") {
463 if let Some(parts) = sc.pointer("/modelTurn/parts").and_then(|p| p.as_array()) {
464 for part in parts {
465 if let Some(data) =
466 part.pointer("/inlineData/data").and_then(|d| d.as_str())
467 {
468 use base64::Engine as _;
469 let bytes = base64::engine::general_purpose::STANDARD
470 .decode(data)
471 .map_err(|e| {
472 LlmError::Parse(format!("bad base64 audio chunk: {e}"))
473 })?;
474 pcm.extend_from_slice(&bytes);
475 }
476 }
477 }
478 if sc
479 .get("turnComplete")
480 .and_then(Value::as_bool)
481 .unwrap_or(false)
482 {
483 break;
484 }
485 }
486 }
487 Some(Ok(Message::Close(_))) => break,
488 Some(Ok(_)) => continue,
489 Some(Err(e)) => return Err(LlmError::Network(e.to_string())),
490 None => break,
491 }
492 }
493
494 let _ = ws.close(None).await;
495
496 if pcm.is_empty() {
497 return Err(LlmError::Parse("no audio returned by live session".into()));
498 }
499
500 Ok(wrap_wav(&pcm))
501}
502
503#[cfg(test)]
504mod tests {
505 #![allow(clippy::unwrap_used, clippy::expect_used)]
506 use super::*;
507
508 #[test]
509 fn build_live_config_omits_unspecified_voice_without_persona() {
510 let cfg = build_live_config(None, None);
511 assert!(cfg.pointer("/generationConfig/speechConfig").is_none());
512 assert_eq!(
513 cfg.pointer("/generationConfig/responseModalities/0")
514 .and_then(|v| v.as_str()),
515 Some("AUDIO")
516 );
517 assert!(cfg.get("systemInstruction").is_none());
518 }
519
520 #[test]
521 fn build_live_config_honors_voice_name_and_persona() {
522 let cfg = build_live_config(Some("warm and upbeat"), Some("Puck"));
523 assert_eq!(
524 cfg.pointer("/generationConfig/speechConfig/voiceConfig/prebuiltVoiceConfig/voiceName")
525 .and_then(|v| v.as_str()),
526 Some("Puck")
527 );
528 assert_eq!(
529 cfg.pointer("/systemInstruction/parts/0/text")
530 .and_then(|v| v.as_str()),
531 Some("warm and upbeat")
532 );
533 }
534
535 #[test]
536 fn build_live_config_ignores_blank_persona_and_voice() {
537 let cfg = build_live_config(Some(" "), Some(""));
538 assert!(cfg.get("systemInstruction").is_none());
539 assert!(cfg.pointer("/generationConfig/speechConfig").is_none());
540 }
541
542 #[tokio::test]
543 async fn speak_rejects_text_over_the_length_cap() {
544 let config = GoogleLiveConfig::new("test-key-not-used", "discovered-model");
545 let text: String = "a".repeat(MAX_SPEAK_TEXT_CHARS + 1);
546 let cancel = CancellationToken::new();
547 let err = speak(&config, &text, None, None, &cancel)
548 .await
549 .unwrap_err();
550 assert!(matches!(err, LlmError::InvalidRequest(_)));
551 }
552
553 #[tokio::test]
554 async fn transcribe_rejects_cancelled_request_without_network() {
555 let config = GoogleLiveConfig::new("test-key-not-used", "discovered-model");
556 let cancel = CancellationToken::new();
557 cancel.cancel();
558 let err = transcribe(&config, b"audio", "audio/pcm", &cancel)
559 .await
560 .unwrap_err();
561 assert!(matches!(err, LlmError::Aborted { .. }));
562 }
563
564 #[test]
565 fn provider_status_errors_preserve_classification_and_message() {
566 assert!(
567 matches!(map_status_error(401, r#"{"error":{"message":"bad key"}}"#), LlmError::Auth(message) if message == "bad key")
568 );
569 assert!(
570 matches!(map_status_error(429, r#"{"error":{"message":"slow down"}}"#), LlmError::RateLimit { message, .. } if message == "slow down")
571 );
572 assert!(
573 matches!(map_status_error(503, r#"{"error":{"message":"busy"}}"#), LlmError::Overloaded(message) if message == "busy")
574 );
575 }
576
577 #[test]
578 fn transcription_request_contains_audio_and_strict_instruction() {
579 let request = build_transcribe_request(&[0, 1, 2], "audio/pcm");
580 assert_eq!(
581 request
582 .pointer("/contents/0/parts/0/inline_data/mime_type")
583 .and_then(Value::as_str),
584 Some("audio/pcm")
585 );
586 assert!(
587 request
588 .pointer("/contents/0/parts/0/inline_data/data")
589 .and_then(Value::as_str)
590 .is_some()
591 );
592 assert!(
593 request
594 .pointer("/contents/0/parts/1/text")
595 .and_then(Value::as_str)
596 .unwrap_or_default()
597 .contains("only the spoken words")
598 );
599 }
600
601 /// Opt-in live smoke test. The credential is read only from the process
602 /// environment and is never printed or persisted.
603 #[tokio::test]
604 #[ignore = "requires GEMINI_API_KEY and network access"]
605 async fn live_smoke_synthesizes_audio() {
606 let Ok(key) = smoke_key() else {
607 return;
608 };
609 let Ok(model) = std::env::var("VAK_GEMINI_LIVE_MODEL") else {
610 return;
611 };
612 let config = GoogleLiveConfig::new(key, model);
613 let bytes = speak(
614 &config,
615 "Say hello briefly.",
616 None,
617 None,
618 &CancellationToken::new(),
619 )
620 .await
621 .expect("configured live provider should synthesize");
622 assert!(bytes.starts_with(b"RIFF"));
623 assert!(bytes.len() > 44);
624 }
625
626 #[tokio::test]
627 #[ignore = "requires GEMINI_API_KEY and network access"]
628 async fn live_smoke_transcribes_audio() {
629 let Ok(key) = smoke_key() else {
630 return;
631 };
632 let Ok(model) = std::env::var("VAK_GEMINI_TRANSCRIBE_MODEL") else {
633 return;
634 };
635 let config = GoogleLiveConfig::new(key, model);
636 let audio = vec![0u8; 3200];
637 let text = transcribe(&config, &audio, "audio/pcm", &CancellationToken::new())
638 .await
639 .expect("configured live provider should transcribe");
640 assert!(!text.trim().is_empty());
641 }
642
643 fn smoke_key() -> Result<String, std::env::VarError> {
644 std::env::var("GEMINI_API_KEY").or_else(|_| std::env::var("GOOGLE_API_KEY"))
645 }
646
647 #[test]
648 fn wrap_wav_produces_valid_riff_header() {
649 let pcm = vec![0u8, 1, 2, 3, 4, 5, 6, 7];
650 let wav = wrap_wav(&pcm);
651 assert_eq!(&wav[0..4], b"RIFF");
652 assert_eq!(&wav[8..12], b"WAVE");
653 assert_eq!(&wav[36..40], b"data");
654 assert_eq!(wav.len(), 44 + pcm.len());
655 }
656}
657

Google Live

Indexing the workspace…

Google Live