development
rs 611 lines 19.4 KB
Raw
1 //! Loudness, peak/RMS, tempo (spectral flux + autocorrelation) and key
2 //! (chromagram + Krumhansl-Schmuckler) estimates. Tempo and key are estimates, not facts.
3
4 use std::f32::consts::PI;
5 use std::path::Path;
6
7 use anyhow::{Result, anyhow};
8 use ebur128::{EbuR128, Mode};
9 use realfft::RealFftPlanner;
10
11 use super::decode::{StreamInfo, decode_file};
12
13 /// Mono analysis signal is decimated to roughly this rate.
14 const ANALYSIS_RATE: u32 = 11_025;
15 /// Tempo and key look at the first ten minutes only.
16 const MAX_ANALYSIS_SECS: f64 = 600.0;
17
18 #[derive(Debug, Clone)]
19 pub struct TempoEstimate {
20 pub bpm: f64,
21 /// Normalized autocorrelation at the chosen lag, 0..1.
22 pub strength: f64,
23 /// Other plausible readings: (bpm, why).
24 pub alternatives: Vec<(f64, &'static str)>,
25 }
26
27 #[derive(Debug, Clone)]
28 pub struct KeyEstimate {
29 pub name: String,
30 pub correlation: f64,
31 pub alternatives: Vec<(String, f64)>,
32 }
33
34 #[derive(Debug, Clone)]
35 pub struct Analysis {
36 pub info: StreamInfo,
37 pub peak_db: f64,
38 pub rms_db: f64,
39 pub lufs: Option<f64>,
40 pub lra: Option<f64>,
41 pub tempo: Option<TempoEstimate>,
42 pub key: Option<KeyEstimate>,
43 pub analysed_secs: f64,
44 }
45
46 fn to_db(x: f64) -> f64 {
47 if x <= 0.0 {
48 f64::NEG_INFINITY
49 } else {
50 20.0 * x.log10()
51 }
52 }
53
54 pub fn analyze_file(path: &Path) -> Result<Analysis> {
55 let mut peak = 0f32;
56 let mut sum_sq = 0f64;
57 let mut count = 0u64;
58 let mut meter: Option<EbuR128> = None;
59 let mut meter_failed = false;
60 let mut mono: Vec<f32> = Vec::new();
61 let mut decim = 1usize;
62 let mut acc = 0f32;
63 let mut acc_n = 0usize;
64 let mut rate_seen = 0u32;
65
66 let info = decode_file(path, |info, samples| {
67 if rate_seen != info.sample_rate {
68 rate_seen = info.sample_rate;
69 decim = (info.sample_rate / ANALYSIS_RATE).max(1) as usize;
70 }
71 for &s in samples {
72 peak = peak.max(s.abs());
73 sum_sq += (s as f64) * (s as f64);
74 }
75 count += samples.len() as u64;
76 if meter.is_none() && !meter_failed {
77 match EbuR128::new(
78 info.channels as u32,
79 info.sample_rate,
80 Mode::I | Mode::LRA | Mode::SAMPLE_PEAK,
81 ) {
82 Ok(m) => meter = Some(m),
83 Err(_) => meter_failed = true,
84 }
85 }
86 if let Some(m) = meter.as_mut() {
87 let _ = m.add_frames_f32(samples);
88 }
89 let ch = info.channels.max(1);
90 let max_len = (MAX_ANALYSIS_SECS * info.sample_rate as f64 / decim as f64) as usize;
91 if mono.len() < max_len {
92 for frame in samples.chunks_exact(ch) {
93 acc += frame.iter().sum::<f32>() / ch as f32;
94 acc_n += 1;
95 if acc_n == decim {
96 mono.push(acc / decim as f32);
97 acc = 0.0;
98 acc_n = 0;
99 }
100 }
101 }
102 })?;
103
104 let rms = if count == 0 {
105 0.0
106 } else {
107 (sum_sq / count as f64).sqrt()
108 };
109 let lufs = meter
110 .as_ref()
111 .and_then(|m| m.loudness_global().ok())
112 .filter(|v| v.is_finite());
113 let lra = meter
114 .as_ref()
115 .and_then(|m| m.loudness_range().ok())
116 .filter(|v| v.is_finite());
117 let analysis_rate = info.sample_rate as f32 / decim as f32;
118 let silent = peak < 1e-5;
119 Ok(Analysis {
120 peak_db: to_db(peak as f64),
121 rms_db: to_db(rms),
122 lufs,
123 lra,
124 tempo: if silent {
125 None
126 } else {
127 estimate_tempo(&mono, analysis_rate)
128 },
129 key: if silent {
130 None
131 } else {
132 estimate_key(&mono, analysis_rate)
133 },
134 analysed_secs: mono.len() as f64 / analysis_rate as f64,
135 info,
136 })
137 }
138
139 fn hann(n: usize) -> Vec<f32> {
140 (0..n)
141 .map(|i| 0.5 - 0.5 * (2.0 * PI * i as f32 / n as f32).cos())
142 .collect()
143 }
144
145 /// Log-magnitude spectral flux per frame, mean-subtracted and half-wave rectified.
146 fn onset_envelope(mono: &[f32], win: usize, hop: usize) -> Vec<f32> {
147 let mut planner = RealFftPlanner::<f32>::new();
148 let fft = planner.plan_fft_forward(win);
149 let window = hann(win);
150 let mut input = fft.make_input_vec();
151 let mut output = fft.make_output_vec();
152 let mut prev = vec![0f32; win / 2 + 1];
153 let mut env = Vec::new();
154 let mut start = 0;
155 while start + win <= mono.len() {
156 for (i, w) in window.iter().enumerate() {
157 input[i] = mono[start + i] * w;
158 }
159 if fft.process(&mut input, &mut output).is_err() {
160 break;
161 }
162 let mut flux = 0f32;
163 for (k, c) in output.iter().enumerate().skip(2) {
164 let mag = (1.0 + 100.0 * c.norm() / win as f32).ln();
165 flux += (mag - prev[k]).max(0.0);
166 prev[k] = mag;
167 }
168 env.push(flux);
169 start += hop;
170 }
171 // Subtract a local mean (~0.5 s) so steady loudness doesn't dominate the autocorrelation.
172 let half = 22usize;
173 let mut out = Vec::with_capacity(env.len());
174 let mut prefix = vec![0f64];
175 for v in &env {
176 prefix.push(prefix.last().unwrap() + *v as f64);
177 }
178 for i in 0..env.len() {
179 let lo = i.saturating_sub(half);
180 let hi = (i + half + 1).min(env.len());
181 let mean = (prefix[hi] - prefix[lo]) / (hi - lo) as f64;
182 out.push((env[i] as f64 - mean).max(0.0) as f32);
183 }
184 out
185 }
186
187 pub fn estimate_tempo(mono: &[f32], rate: f32) -> Option<TempoEstimate> {
188 const WIN: usize = 1024;
189 const HOP: usize = 128;
190 const MIN_BPM: f64 = 40.0;
191 const MAX_BPM: f64 = 240.0;
192 if (mono.len() as f32) < rate * 4.0 {
193 return None;
194 }
195 let env = onset_envelope(mono, WIN, HOP);
196 let fps = rate as f64 / HOP as f64;
197 let min_lag = (fps * 60.0 / MAX_BPM).floor() as usize;
198 let max_lag = (fps * 60.0 / MIN_BPM).ceil() as usize;
199 if env.len() < max_lag * 3 {
200 return None;
201 }
202 let n = env.len();
203 let mean = env.iter().map(|v| *v as f64).sum::<f64>() / n as f64;
204 let centered: Vec<f64> = env.iter().map(|v| *v as f64 - mean).collect();
205 let r0: f64 = centered.iter().map(|v| v * v).sum::<f64>() / n as f64;
206 if r0 <= 1e-12 {
207 return None;
208 }
209 let acf: Vec<f64> = (0..=2 * max_lag)
210 .map(|lag| {
211 let s: f64 = (0..n - lag).map(|i| centered[i] * centered[i + lag]).sum();
212 s / (n - lag) as f64 / r0
213 })
214 .collect();
215 let bpm_at = |lag: f64| 60.0 * fps / lag;
216 let prior = |bpm: f64| (-0.5 * (bpm / 120.0).log2().powi(2)).exp();
217 let score = |lag: usize| {
218 let mut s = acf[lag].max(0.0);
219 if 2 * lag < acf.len() {
220 s += 0.5 * acf[2 * lag].max(0.0);
221 }
222 s * prior(bpm_at(lag as f64))
223 };
224 let mut best = min_lag.max(1);
225 for lag in min_lag.max(1)..=max_lag {
226 if score(lag) > score(best) {
227 best = lag;
228 }
229 }
230 // Parabolic interpolation around the peak for sub-frame lag.
231 let refined = if best > 1 && best + 1 < acf.len() {
232 let (a, b, c) = (acf[best - 1], acf[best], acf[best + 1]);
233 let denom = a - 2.0 * b + c;
234 if denom.abs() > 1e-12 {
235 best as f64 + 0.5 * (a - c) / denom
236 } else {
237 best as f64
238 }
239 } else {
240 best as f64
241 };
242 let bpm = bpm_at(refined.clamp(best as f64 - 1.0, best as f64 + 1.0));
243 let mut alternatives = Vec::new();
244 if bpm / 2.0 >= MIN_BPM {
245 alternatives.push((bpm / 2.0, "half-time"));
246 }
247 if bpm * 2.0 <= MAX_BPM * 1.25 {
248 alternatives.push((bpm * 2.0, "double-time"));
249 }
250 Some(TempoEstimate {
251 bpm,
252 strength: acf[best].clamp(0.0, 1.0),
253 alternatives,
254 })
255 }
256
257 const MAJOR_PROFILE: [f64; 12] = [
258 6.35, 2.23, 3.48, 2.33, 4.38, 4.09, 2.52, 5.19, 2.39, 3.66, 2.29, 2.88,
259 ];
260 const MINOR_PROFILE: [f64; 12] = [
261 6.33, 2.68, 3.52, 5.38, 2.60, 3.53, 2.54, 4.75, 3.98, 2.69, 3.34, 3.17,
262 ];
263 const MAJOR_NAMES: [&str; 12] = [
264 "C", "Db", "D", "Eb", "E", "F", "F#", "G", "Ab", "A", "Bb", "B",
265 ];
266 const MINOR_NAMES: [&str; 12] = [
267 "C", "C#", "D", "Eb", "E", "F", "F#", "G", "G#", "A", "Bb", "B",
268 ];
269
270 fn pearson(a: &[f64], b: &[f64]) -> f64 {
271 let n = a.len() as f64;
272 let (ma, mb) = (a.iter().sum::<f64>() / n, b.iter().sum::<f64>() / n);
273 let (mut num, mut da, mut db) = (0.0, 0.0, 0.0);
274 for (x, y) in a.iter().zip(b) {
275 num += (x - ma) * (y - mb);
276 da += (x - ma).powi(2);
277 db += (y - mb).powi(2);
278 }
279 if da <= 0.0 || db <= 0.0 {
280 0.0
281 } else {
282 num / (da * db).sqrt()
283 }
284 }
285
286 pub fn chromagram(mono: &[f32], rate: f32) -> [f64; 12] {
287 const WIN: usize = 4096;
288 const HOP: usize = 2048;
289 let mut planner = RealFftPlanner::<f32>::new();
290 let fft = planner.plan_fft_forward(WIN);
291 let window = hann(WIN);
292 let mut input = fft.make_input_vec();
293 let mut output = fft.make_output_vec();
294 let bin_hz = rate as f64 / WIN as f64;
295 let lo = (100.0 / bin_hz).ceil() as usize;
296 let hi = ((2000.0 / bin_hz) as usize).min(WIN / 2);
297 let mut chroma = [0f64; 12];
298 let mut start = 0;
299 while start + WIN <= mono.len() {
300 for (i, w) in window.iter().enumerate() {
301 input[i] = mono[start + i] * w;
302 }
303 if fft.process(&mut input, &mut output).is_err() {
304 break;
305 }
306 for (k, c) in output.iter().enumerate().take(hi + 1).skip(lo) {
307 let hz = k as f64 * bin_hz;
308 let midi = 69.0 + 12.0 * (hz / 440.0).log2();
309 let pc = (midi.round() as i64).rem_euclid(12) as usize;
310 chroma[pc] += c.norm() as f64;
311 }
312 start += HOP;
313 }
314 chroma
315 }
316
317 pub fn estimate_key(mono: &[f32], rate: f32) -> Option<KeyEstimate> {
318 if (mono.len() as f32) < rate * 3.0 {
319 return None;
320 }
321 let chroma = chromagram(mono, rate);
322 if chroma.iter().sum::<f64>() <= 0.0 {
323 return None;
324 }
325 let mut scored: Vec<(String, f64)> = Vec::with_capacity(24);
326 for tonic in 0..12 {
327 for (profile, names, suffix) in [
328 (&MAJOR_PROFILE, &MAJOR_NAMES, "major"),
329 (&MINOR_PROFILE, &MINOR_NAMES, "minor"),
330 ] {
331 let rotated: Vec<f64> = (0..12).map(|i| profile[(i + 12 - tonic) % 12]).collect();
332 scored.push((
333 format!("{} {suffix}", names[tonic]),
334 pearson(&chroma, &rotated),
335 ));
336 }
337 }
338 scored.sort_by(|a, b| b.1.total_cmp(&a.1));
339 let (name, correlation) = scored[0].clone();
340 Some(KeyEstimate {
341 name,
342 correlation,
343 alternatives: scored[1..4].to_vec(),
344 })
345 }
346
347 fn channel_label(n: usize) -> String {
348 match n {
349 1 => "mono".into(),
350 2 => "stereo".into(),
351 n => format!("{n} channels"),
352 }
353 }
354
355 fn fmt_db(v: f64) -> String {
356 if v.is_finite() {
357 format!("{v:.1}")
358 } else {
359 "-inf".into()
360 }
361 }
362
363 /// Human-readable report, written for the model to relay. Tempo and key are labelled estimates.
364 pub fn report(path: &Path, a: &Analysis) -> String {
365 let i = &a.info;
366 let secs = i.duration_secs();
367 let mut out = format!(
368 "File: {}\nFormat: {} ({})\nDuration: {}:{:04.1} ({secs:.1} s), {} Hz, {}{}\n",
369 path.display(),
370 if i.extension.is_empty() {
371 "unknown"
372 } else {
373 &i.extension
374 },
375 i.codec,
376 (secs / 60.0).floor() as u64,
377 secs % 60.0,
378 i.sample_rate,
379 channel_label(i.channels),
380 i.bits_per_sample
381 .map_or(String::new(), |b| format!(", {b}-bit")),
382 );
383 out.push_str(&format!(
384 "Peak: {} dBFS, RMS: {} dBFS\n",
385 fmt_db(a.peak_db),
386 fmt_db(a.rms_db)
387 ));
388 match (a.lufs, a.lra) {
389 (Some(l), Some(r)) => out.push_str(&format!(
390 "Loudness: {l:.1} LUFS integrated, loudness range {r:.1} LU\n"
391 )),
392 (Some(l), None) => out.push_str(&format!("Loudness: {l:.1} LUFS integrated\n")),
393 _ => out.push_str("Loudness: not measurable (silence or too short)\n"),
394 }
395 if a.peak_db.is_infinite() || a.peak_db < -100.0 {
396 out.push_str("The file is silent; tempo and key were not estimated.");
397 return out;
398 }
399 match &a.tempo {
400 Some(t) => {
401 let conf = if t.strength > 0.5 {
402 "high"
403 } else if t.strength > 0.25 {
404 "medium"
405 } else {
406 "low"
407 };
408 let alts: Vec<String> = t
409 .alternatives
410 .iter()
411 .map(|(b, why)| format!("{b:.1} ({why})"))
412 .collect();
413 out.push_str(&format!(
414 "Tempo (estimate): {:.1} BPM, {conf} confidence; also plausible: {}\n",
415 t.bpm,
416 alts.join(", ")
417 ));
418 }
419 None => out.push_str("Tempo (estimate): not enough rhythmic material to estimate\n"),
420 }
421 match &a.key {
422 Some(k) => {
423 let alts: Vec<String> = k
424 .alternatives
425 .iter()
426 .map(|(n, r)| format!("{n} ({r:.2})"))
427 .collect();
428 let conf = if k.correlation > 0.75 {
429 "high"
430 } else if k.correlation > 0.55 {
431 "medium"
432 } else {
433 "low"
434 };
435 out.push_str(&format!(
436 "Key (estimate): {} (correlation {:.2}, {conf} confidence); alternatives: {}\n",
437 k.name,
438 k.correlation,
439 alts.join(", ")
440 ));
441 }
442 None => out.push_str("Key (estimate): not enough tonal material to estimate\n"),
443 }
444 if a.analysed_secs + 1.0 < secs {
445 out.push_str(&format!(
446 "Tempo and key come from the first {:.0} s only.\n",
447 a.analysed_secs
448 ));
449 }
450 out.push_str("Tempo and key are signal-analysis estimates; confirm by ear.");
451 out
452 }
453
454 /// Convenience wrapper for the tool: errors carry context for the model.
455 pub fn analyze_and_report(path: &Path) -> Result<String> {
456 let a = analyze_file(path).map_err(|e| anyhow!("{e:#}"))?;
457 Ok(report(path, &a))
458 }
459
460 #[cfg(test)]
461 mod tests {
462 use super::*;
463
464 const SR: u32 = 44_100;
465
466 fn write_wav(path: &Path, channels: u16, samples: &[f32]) {
467 let spec = hound::WavSpec {
468 channels,
469 sample_rate: SR,
470 bits_per_sample: 16,
471 sample_format: hound::SampleFormat::Int,
472 };
473 let mut w = hound::WavWriter::create(path, spec).unwrap();
474 for s in samples {
475 w.write_sample((s.clamp(-1.0, 1.0) * i16::MAX as f32) as i16)
476 .unwrap();
477 }
478 w.finalize().unwrap();
479 }
480
481 /// Decaying 1 kHz plus broadband-ish clicks on every beat.
482 fn click_track(bpm: f64, secs: f64) -> Vec<f32> {
483 let n = (secs * SR as f64) as usize;
484 let mut out = vec![0f32; n];
485 let beat = 60.0 / bpm * SR as f64;
486 let mut t = 0.0;
487 let mut seed = 12345u32;
488 while (t as usize) < n {
489 for j in 0..1500 {
490 let i = t as usize + j;
491 if i >= n {
492 break;
493 }
494 seed = seed.wrapping_mul(1664525).wrapping_add(1013904223);
495 let noise = (seed >> 8) as f32 / (1 << 23) as f32 - 1.0;
496 let env = (-(j as f32) / 300.0).exp();
497 out[i] += 0.6
498 * env
499 * (0.6 * noise + 0.4 * (2.0 * PI * 1000.0 * j as f32 / SR as f32).sin());
500 }
501 t += beat;
502 }
503 out
504 }
505
506 fn chord_progression(chords: &[&[f32]], secs_each: f64) -> Vec<f32> {
507 let mut out = Vec::new();
508 for freqs in chords {
509 let n = (secs_each * SR as f64) as usize;
510 for i in 0..n {
511 let t = i as f32 / SR as f32;
512 let mut s = 0.0;
513 for f in freqs.iter() {
514 for h in 1..=4 {
515 s += (2.0 * PI * f * h as f32 * t).sin() / h as f32;
516 }
517 }
518 out.push(0.1 * s);
519 }
520 }
521 out
522 }
523
524 fn tmp_wav(name: &str) -> std::path::PathBuf {
525 std::env::temp_dir().join(format!("sf-audio-{name}-{}.wav", uuid::Uuid::new_v4()))
526 }
527
528 #[test]
529 fn tempo_of_click_tracks() {
530 for bpm in [90.0, 120.0, 140.0] {
531 let path = tmp_wav("tempo");
532 write_wav(&path, 1, &click_track(bpm, 24.0));
533 let a = analyze_file(&path).unwrap();
534 let t = a.tempo.expect("tempo");
535 assert!(
536 (t.bpm - bpm).abs() / bpm < 0.02,
537 "wanted {bpm}, got {}",
538 t.bpm
539 );
540 assert!(t.alternatives.iter().any(|(_, w)| *w == "half-time"));
541 std::fs::remove_file(path).ok();
542 }
543 }
544
545 #[test]
546 fn key_of_chord_progressions() {
547 // C F G C
548 let c = [130.81, 164.81, 196.0];
549 let f = [174.61, 220.0, 261.63];
550 let g = [196.0, 246.94, 293.66];
551 let path = tmp_wav("keyc");
552 write_wav(&path, 1, &chord_progression(&[&c, &f, &g, &c], 3.0));
553 let k = analyze_file(&path).unwrap().key.expect("key");
554 assert_eq!(k.name, "C major", "{k:?}");
555 std::fs::remove_file(path).ok();
556 // Am Dm E Am
557 let am = [110.0, 130.81, 164.81];
558 let dm = [146.83, 174.61, 220.0];
559 let e = [164.81, 207.65, 246.94];
560 let path = tmp_wav("keya");
561 write_wav(&path, 1, &chord_progression(&[&am, &dm, &e, &am], 3.0));
562 let k = analyze_file(&path).unwrap().key.expect("key");
563 assert_eq!(k.name, "A minor", "{k:?}");
564 std::fs::remove_file(path).ok();
565 }
566
567 #[test]
568 fn levels_loudness_and_format() {
569 // 5 s of a 997 Hz sine at -20 dBFS peak, stereo.
570 let n = 5 * SR as usize;
571 let amp = 0.1f32;
572 let mut samples = Vec::with_capacity(n * 2);
573 for i in 0..n {
574 let s = amp * (2.0 * PI * 997.0 * i as f32 / SR as f32).sin();
575 samples.push(s);
576 samples.push(s);
577 }
578 let path = tmp_wav("levels");
579 write_wav(&path, 2, &samples);
580 let a = analyze_file(&path).unwrap();
581 assert_eq!(a.info.sample_rate, SR);
582 assert_eq!(a.info.channels, 2);
583 assert!((a.info.duration_secs() - 5.0).abs() < 0.01);
584 assert!((a.peak_db + 20.0).abs() < 0.2, "{}", a.peak_db);
585 // RMS of a sine is 3 dB below its peak.
586 assert!((a.rms_db + 23.0).abs() < 0.3, "{}", a.rms_db);
587 // A 997 Hz sine at -23 dBFS RMS per channel, two channels summed: about -20 LUFS.
588 let lufs = a.lufs.expect("lufs");
589 assert!((lufs + 20.0).abs() < 1.0, "{lufs}");
590 let text = report(&path, &a);
591 assert!(text.contains("44100 Hz, stereo, 16-bit"), "{text}");
592 assert!(text.contains("estimates"), "{text}");
593 std::fs::remove_file(path).ok();
594 }
595
596 #[test]
597 fn silence_and_garbage() {
598 let path = tmp_wav("silence");
599 write_wav(&path, 1, &vec![0.0; SR as usize * 2]);
600 let a = analyze_file(&path).unwrap();
601 assert!(a.tempo.is_none() && a.key.is_none());
602 assert!(report(&path, &a).contains("silent"));
603 std::fs::remove_file(&path).ok();
604
605 let bad = std::env::temp_dir().join(format!("sf-audio-bad-{}.wav", uuid::Uuid::new_v4()));
606 std::fs::write(&bad, b"definitely not audio").unwrap();
607 assert!(analyze_file(&bad).is_err());
608 std::fs::remove_file(bad).ok();
609 assert!(analyze_file(Path::new("/no/such/file.wav")).is_err());
610 }
611 }