1. 1//! Speech evidence for a closed utterance.
2. 2//!
3. 3//! The client decides where an utterance starts and stops, but the server
4. 4//! decides whether it is worth a paid transcription call and an Agent turn.
5. 5//! A fixed loudness threshold cannot make that call: in a noisy room every
6. 6//! frame clears it. Evidence is therefore measured against the utterance's
7. 7//! own quiet floor, so steady room noise — however loud — carries no speech.
8. 8//!
9. 9//! `crates/vak-client-ui/src/voice-activity.ts` applies the same constants
10. 10//! live for endpointing; `constants_match_the_client_detector` pins them.
11. 11
12. 12/// Analysis frame length.
13. 13pub const FRAME_MS: u32 = 20;
14. 14/// Normalised RMS below which a frame is silence in any room.
15. 15pub const ABSOLUTE_SPEECH_RMS: f32 = 0.018;
16. 16/// A voiced frame must stand this far above the quiet floor (about 8 dB).
17. 17pub const FLOOR_RATIO: f32 = 2.5;
18. 18/// Voiced audio an utterance needs before it may reach a provider.
19. 19pub const MIN_VOICED_MS: u32 = 240;
20. 20/// Share of frames, from the quietest, that define the floor.
21. 21pub const FLOOR_PERCENTILE: f32 = 0.2;
22. 22
23. 23#[derive(Debug, Clone, Copy, PartialEq)]
24. 24pub struct SpeechEvidence {
25. 25 pub voiced_ms: u32,
26. 26 pub total_ms: u32,
27. 27 /// Normalised RMS of the utterance's quiet floor.
28. 28 pub floor_rms: f32,
29. 29}
30. 30
31. 31impl SpeechEvidence {
32. 32 /// Measure mono 16-bit little-endian PCM.
33. 33 pub fn measure(pcm: &[u8], sample_rate_hz: u32) -> Self {
34. 34 let frame_samples = (sample_rate_hz * FRAME_MS / 1000).max(1) as usize;
35. 35 let samples: Vec<f32> = pcm
36. 36 .as_chunks::<2>()
37. 37 .0
38. 38 .iter()
39. 39 .map(|pair| f32::from(i16::from_le_bytes(*pair)) / 32768.0)
40. 40 .collect();
41. 41 let mut frames: Vec<f32> = samples
42. 42 .chunks(frame_samples)
43. 43 .filter(|frame| frame.len() == frame_samples)
44. 44 .map(|frame| (frame.iter().map(|s| s * s).sum::<f32>() / frame.len() as f32).sqrt())
45. 45 .collect();
46. 46 let total_ms = frames.len() as u32 * FRAME_MS;
47. 47 if frames.is_empty() {
48. 48 return Self {
49. 49 voiced_ms: 0,
50. 50 total_ms,
51. 51 floor_rms: 0.0,
52. 52 };
53. 53 }
54. 54 let rms = frames.clone();
55. 55 frames.sort_by(f32::total_cmp);
56. 56 let floor_index = ((frames.len() - 1) as f32 * FLOOR_PERCENTILE) as usize;
57. 57 let floor_rms = frames[floor_index];
58. 58 let threshold = ABSOLUTE_SPEECH_RMS.max(floor_rms * FLOOR_RATIO);
59. 59 let voiced = rms.iter().filter(|value| **value >= threshold).count() as u32;
60. 60 Self {
61. 61 voiced_ms: voiced * FRAME_MS,
62. 62 total_ms,
63. 63 floor_rms,
64. 64 }
65. 65 }
66. 66
67. 67 pub fn is_speech(&self) -> bool {
68. 68 self.voiced_ms >= MIN_VOICED_MS
69. 69 }
70. 70}
71. 71
72. 72#[cfg(test)]
73. 73#[allow(clippy::expect_used, clippy::unwrap_used)]
74. 74mod tests {
75. 75 use super::*;
76. 76
77. 77 const RATE: u32 = 16_000;
78. 78
79. 79 /// Deterministic PCM: `segments` of (milliseconds, amplitude, hz).
80. 80 fn pcm(segments: &[(u32, f32, f32)]) -> Vec<u8> {
81. 81 let mut out = Vec::new();
82. 82 let mut n = 0u32;
83. 83 for &(ms, amplitude, hz) in segments {
84. 84 for _ in 0..(RATE * ms / 1000) {
85. 85 let t = n as f32 / RATE as f32;
86. 86 // A tone plus a fixed pseudo-noise term, so "noise" segments
87. 87 // are broadband rather than silent.
88. 88 let noise = (((n.wrapping_mul(1_103_515_245).wrapping_add(12_345)) >> 16) & 0x7fff)
89. 89 as f32
90. 90 / 16_384.0
91. 91 - 1.0;
92. 92 let value =
93. 93 amplitude * (0.7 * (std::f32::consts::TAU * hz * t).sin() + 0.3 * noise);
94. 94 out.extend_from_slice(&((value * 32767.0) as i16).to_le_bytes());
95. 95 n += 1;
96. 96 }
97. 97 }
98. 98 out
99. 99 }
100. 100
101. 101 #[test]
102. 102 fn speech_followed_by_a_quiet_tail_is_speech() {
103. 103 let evidence =
104. 104 SpeechEvidence::measure(&pcm(&[(900, 0.25, 220.0), (1500, 0.004, 0.0)]), RATE);
105. 105 assert!(evidence.is_speech(), "{evidence:?}");
106. 106 assert!(evidence.voiced_ms >= 800);
107. 107 }
108. 108
109. 109 #[test]
110. 110 fn loud_steady_room_noise_is_not_speech() {
111. 111 // Far above the absolute threshold, but flat: no frame stands out
112. 112 // from the floor, which is what a fan, traffic or a crowd looks like.
113. 113 let evidence = SpeechEvidence::measure(&pcm(&[(2500, 0.12, 0.0)]), RATE);
114. 114 assert!(!evidence.is_speech(), "{evidence:?}");
115. 115 }
116. 116
117. 117 #[test]
118. 118 fn a_click_is_not_speech() {
119. 119 let evidence =
120. 120 SpeechEvidence::measure(&pcm(&[(100, 0.5, 300.0), (1500, 0.003, 0.0)]), RATE);
121. 121 assert!(!evidence.is_speech(), "{evidence:?}");
122. 122 }
123. 123
124. 124 #[test]
125. 125 fn speech_over_room_noise_is_still_speech() {
126. 126 let evidence = SpeechEvidence::measure(
127. 127 &pcm(&[(400, 0.03, 0.0), (900, 0.3, 180.0), (1500, 0.03, 0.0)]),
128. 128 RATE,
129. 129 );
130. 130 assert!(evidence.is_speech(), "{evidence:?}");
131. 131 }
132. 132
133. 133 #[test]
134. 134 fn empty_or_partial_frames_carry_no_speech() {
135. 135 assert!(!SpeechEvidence::measure(&[], RATE).is_speech());
136. 136 assert!(!SpeechEvidence::measure(&[1, 2, 3], RATE).is_speech());
137. 137 }
138. 138
139. 139 #[test]
140. 140 fn constants_match_the_client_detector() {
141. 141 let client = std::fs::read_to_string(concat!(
142. 142 env!("CARGO_MANIFEST_DIR"),
143. 143 "/../vak-client-ui/src/voice-activity.ts"
144. 144 ))
145. 145 .expect("client detector source");
146. 146 for (name, value) in [
147. 147 ("ABSOLUTE_SPEECH_RMS", ABSOLUTE_SPEECH_RMS.to_string()),
148. 148 ("FLOOR_RATIO", FLOOR_RATIO.to_string()),
149. 149 ("MIN_VOICED_MS", MIN_VOICED_MS.to_string()),
150. 150 ("FLOOR_PERCENTILE", FLOOR_PERCENTILE.to_string()),
151. 151 ] {
152. 152 let needle = format!("export const {name} = {value};");
153. 153 assert!(client.contains(&needle), "client must declare `{needle}`");
154. 154 }
155. 155 }
156. 156}
157. 157