Working Example Demo

This commit is contained in:
Andrey Tkachenko 2020-08-18 20:49:31 +04:00
parent 2c98558d53
commit 5e31bee717
14 changed files with 171 additions and 91 deletions

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
/target
/models
Cargo.lock

9
.gitmodules vendored Normal file
View File

@ -0,0 +1,9 @@
[submodule "resources/vosk-api"]
path = resources/vosk-api
url = https://github.com/andreytkachenko/vosk-api.git
[submodule "resources/openfst"]
path = resources/openfst
url = https://github.com/alphacep/openfst
[submodule "resources/kaldi"]
path = resources/kaldi
url = https://github.com/kaldi-asr/kaldi

View File

@ -8,6 +8,8 @@ build = "build.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
cblas = "0.2.0"
openblas-src = { version = "0.9.0", features = ["cache", "static"] }
[build-dependencies]
bindgen = "0.54.1"

128
build.rs
View File

@ -18,15 +18,10 @@ fn main() {
.clang_arg("-x")
.clang_arg("c++")
.parse_callbacks(Box::new(bindgen::CargoCallbacks))
.opaque_type("std::.*")
.opaque_type("kaldi::.*")
.opaque_type("fst::.*")
.opaque_type("KaldiRecognizer")
.opaque_type("Model")
.opaque_type("SpkModel")
.whitelist_type("KaldiRecognizer")
.whitelist_type("Model")
.whitelist_type("SpkModel")
.whitelist_function("vosk_.*")
.rustified_non_exhaustive_enum("*")
.no_copy(".*")
.layout_tests(false)
@ -40,6 +35,7 @@ fn main() {
cc::Build::new()
.warnings(false)
.extra_warnings(false)
.static_flag(true)
.cpp(true)
.include("resources/openfst/src/include")
@ -53,10 +49,14 @@ fn main() {
.file("resources/openfst/src/lib/symbol-table.cc")
.file("resources/openfst/src/lib/util.cc")
.file("resources/openfst/src/lib/weight.cc")
.compile("libopenfst");
.file("resources/openfst/src/extensions/ngram/bitmap-index.cc")
.file("resources/openfst/src/extensions/ngram/nthbit.cc")
.try_compile("libopenfst")
.unwrap();
cc::Build::new()
.warnings(false)
.extra_warnings(false)
.static_flag(true)
.cpp(true)
.include("resources/vosk-api/src")
@ -65,7 +65,9 @@ fn main() {
.file("resources/vosk-api/src/kaldi_recognizer.cc")
.file("resources/vosk-api/src/model.cc")
.file("resources/vosk-api/src/spk_model.cc")
.compile("libvosk");
.file("resources/vosk-api/src/vosk_api.cc")
.try_compile("libvosk")
.unwrap();
let out_dir = env::var("OUT_DIR").unwrap();
let contents = fs::read_to_string("resources/kaldi/src/lat/kaldi-lattice.cc").expect("Something went wrong reading the file");
@ -84,6 +86,7 @@ fn main() {
cc::Build::new()
.warnings(false)
.extra_warnings(false)
.static_flag(true)
.cpp(true)
.define("HAVE_OPENBLAS", "true")
@ -99,17 +102,27 @@ fn main() {
// .file("resources/kaldi/src/base/timer.cc")
// matrix
.file("resources/kaldi/src/matrix/kaldi-matrix.cc")
.file("resources/kaldi/src/matrix/kaldi-vector.cc")
.file("resources/kaldi/src/matrix/matrix-functions.cc")
.file("resources/kaldi/src/matrix/optimization.cc")
.file("resources/kaldi/src/matrix/tp-matrix.cc")
.file("resources/kaldi/src/matrix/sp-matrix.cc")
.file("resources/kaldi/src/matrix/packed-matrix.cc")
.file("resources/kaldi/src/matrix/qr.cc")
.file("resources/kaldi/src/matrix/srfft.cc")
.file("resources/kaldi/src/matrix/compressed-matrix.cc")
// cuda
.file("resources/kaldi/src/cudamatrix/cu-matrix.cc")
.file("resources/kaldi/src/cudamatrix/cu-allocator.cc")
.file("resources/kaldi/src/cudamatrix/cu-common.cc")
// .file("resources/kaldi/src/cudamatrix/cu-math.cc")
.file("resources/kaldi/src/cudamatrix/cu-array.cc")
.file("resources/kaldi/src/cudamatrix/cu-vector.cc")
.file("resources/kaldi/src/cudamatrix/cu-packed-matrix.cc")
.file("resources/kaldi/src/cudamatrix/cu-sp-matrix.cc")
.file("resources/kaldi/src/cudamatrix/cu-rand.cc")
.file("resources/kaldi/src/cudamatrix/cu-math.cc")
// fstext
.file("resources/kaldi/src/fstext/context-fst.cc")
@ -119,17 +132,22 @@ fn main() {
// feat
// .file("resources/kaldi/src/feat/feature-fbank.cc")
// .file("resources/kaldi/src/feat/feature-functions.cc")
.file("resources/kaldi/src/feat/feature-functions.cc")
.file("resources/kaldi/src/feat/feature-mfcc.cc")
// .file("resources/kaldi/src/feat/feature-plp.cc")
// .file("resources/kaldi/src/feat/feature-spectrogram.cc")
// .file("resources/kaldi/src/feat/feature-window.cc")
// .file("resources/kaldi/src/feat/mel-computations.cc")
// .file("resources/kaldi/src/feat/online-feature.cc")
// .file("resources/kaldi/src/feat/pitch-functions.cc")
// .file("resources/kaldi/src/feat/resample.cc")
.file("resources/kaldi/src/feat/feature-window.cc")
.file("resources/kaldi/src/feat/mel-computations.cc")
.file("resources/kaldi/src/feat/online-feature.cc")
.file("resources/kaldi/src/feat/pitch-functions.cc")
.file("resources/kaldi/src/feat/resample.cc")
// .file("resources/kaldi/src/feat/signal.cc")
// .file("resources/kaldi/src/feat/wave-reader.cc")
.file("resources/kaldi/src/feat/feature-plp.cc")
.file("resources/kaldi/src/feat/feature-fbank.cc")
// transform
.file("resources/kaldi/src/transform/cmvn.cc")
// lm
// .file("resources/kaldi/src/lm/arpa-file-parser.cc")
@ -152,13 +170,21 @@ fn main() {
// .file("resources/kaldi/src/rnnlm/sampling-lm-estimate.cc")
// .file("resources/kaldi/src/rnnlm/sampling-lm.cc")
// hmm
.file("resources/kaldi/src/hmm/transition-model.cc")
.file("resources/kaldi/src/hmm/hmm-topology.cc")
.file("resources/kaldi/src/hmm/posterior.cc")
// gmm
.file("resources/kaldi/src/gmm/diag-gmm.cc")
// decoder
// .file("resources/kaldi/src/decoder/decodable-matrix.cc")
// .file("resources/kaldi/src/decoder/decoder-wrappers.cc")
// .file("resources/kaldi/src/decoder/faster-decoder.cc")
// .file("resources/kaldi/src/decoder/grammar-fst.cc")
.file("resources/kaldi/src/decoder/lattice-faster-decoder.cc")
// .file("resources/kaldi/src/decoder/lattice-faster-online-decoder.cc")
.file("resources/kaldi/src/decoder/lattice-faster-online-decoder.cc")
// .file("resources/kaldi/src/decoder/lattice-incremental-decoder.cc")
// .file("resources/kaldi/src/decoder/lattice-incremental-online-decoder.cc")
// .file("resources/kaldi/src/decoder/lattice-simple-decoder.cc")
@ -167,84 +193,87 @@ fn main() {
// nnet3
.file("resources/kaldi/src/nnet3/am-nnet-simple.cc")
// .file("resources/kaldi/src/nnet3/attention.cc")
// .file("resources/kaldi/src/nnet3/convolution.cc")
.file("resources/kaldi/src/nnet3/attention.cc")
.file("resources/kaldi/src/nnet3/convolution.cc")
.file("resources/kaldi/src/nnet3/decodable-online-looped.cc")
.file("resources/kaldi/src/nnet3/decodable-simple-looped.cc")
// .file("resources/kaldi/src/nnet3/discriminative-supervision.cc")
// .file("resources/kaldi/src/nnet3/discriminative-training.cc")
// .file("resources/kaldi/src/nnet3/natural-gradient-online.cc")
.file("resources/kaldi/src/nnet3/natural-gradient-online.cc")
.file("resources/kaldi/src/nnet3/nnet-am-decodable-simple.cc")
// .file("resources/kaldi/src/nnet3/nnet-analyze.cc")
// .file("resources/kaldi/src/nnet3/nnet-attention-component.cc")
.file("resources/kaldi/src/nnet3/nnet-analyze.cc")
.file("resources/kaldi/src/nnet3/nnet-attention-component.cc")
// .file("resources/kaldi/src/nnet3/nnet-batch-compute.cc")
// .file("resources/kaldi/src/nnet3/nnet-chain-diagnostics.cc")
// .file("resources/kaldi/src/nnet3/nnet-chain-diagnostics2.cc")
// .file("resources/kaldi/src/nnet3/nnet-chain-example.cc")
// .file("resources/kaldi/src/nnet3/nnet-chain-training.cc")
// .file("resources/kaldi/src/nnet3/nnet-chain-training2.cc")
// .file("resources/kaldi/src/nnet3/nnet-combined-component.cc")
// .file("resources/kaldi/src/nnet3/nnet-common.cc")
// .file("resources/kaldi/src/nnet3/nnet-compile-looped.cc")
// .file("resources/kaldi/src/nnet3/nnet-compile-utils.cc")
// .file("resources/kaldi/src/nnet3/nnet-compile.cc")
// .file("resources/kaldi/src/nnet3/nnet-component-itf.cc")
// .file("resources/kaldi/src/nnet3/nnet-computation-graph.cc")
.file("resources/kaldi/src/nnet3/nnet-combined-component.cc")
.file("resources/kaldi/src/nnet3/nnet-common.cc")
.file("resources/kaldi/src/nnet3/nnet-compile-looped.cc")
.file("resources/kaldi/src/nnet3/nnet-compile-utils.cc")
.file("resources/kaldi/src/nnet3/nnet-compile.cc")
.file("resources/kaldi/src/nnet3/nnet-component-itf.cc")
.file("resources/kaldi/src/nnet3/nnet-computation-graph.cc")
.file("resources/kaldi/src/nnet3/nnet-computation.cc")
.file("resources/kaldi/src/nnet3/nnet-compute.cc")
// .file("resources/kaldi/src/nnet3/nnet-convolutional-component.cc")
// .file("resources/kaldi/src/nnet3/nnet-descriptor.cc")
.file("resources/kaldi/src/nnet3/nnet-convolutional-component.cc")
.file("resources/kaldi/src/nnet3/nnet-descriptor.cc")
// .file("resources/kaldi/src/nnet3/nnet-diagnostics.cc")
// .file("resources/kaldi/src/nnet3/nnet-discriminative-diagnostics.cc")
// .file("resources/kaldi/src/nnet3/nnet-discriminative-example.cc")
// .file("resources/kaldi/src/nnet3/nnet-discriminative-training.cc")
// .file("resources/kaldi/src/nnet3/nnet-example-utils.cc")
// .file("resources/kaldi/src/nnet3/nnet-example.cc")
// .file("resources/kaldi/src/nnet3/nnet-general-component.cc")
// .file("resources/kaldi/src/nnet3/nnet-graph.cc")
.file("resources/kaldi/src/nnet3/nnet-general-component.cc")
.file("resources/kaldi/src/nnet3/nnet-graph.cc")
.file("resources/kaldi/src/nnet3/nnet-nnet.cc")
// .file("resources/kaldi/src/nnet3/nnet-normalize-component.cc")
// .file("resources/kaldi/src/nnet3/nnet-optimize-utils.cc")
// .file("resources/kaldi/src/nnet3/nnet-optimize.cc")
.file("resources/kaldi/src/nnet3/nnet-normalize-component.cc")
.file("resources/kaldi/src/nnet3/nnet-optimize-utils.cc")
.file("resources/kaldi/src/nnet3/nnet-optimize.cc")
.file("resources/kaldi/src/nnet3/nnet-parse.cc")
// .file("resources/kaldi/src/nnet3/nnet-simple-component.cc")
// .file("resources/kaldi/src/nnet3/nnet-tdnn-component.cc")
.file("resources/kaldi/src/nnet3/nnet-simple-component.cc")
.file("resources/kaldi/src/nnet3/nnet-tdnn-component.cc")
// .file("resources/kaldi/src/nnet3/nnet-training.cc")
.file("resources/kaldi/src/nnet3/nnet-utils.cc")
// lat
// .file("resources/kaldi/src/lat/compose-lattice-pruned.cc")
// .file("resources/kaldi/src/lat/confidence.cc")
// .file("resources/kaldi/src/lat/determinize-lattice-pruned.cc")
.file("resources/kaldi/src/lat/determinize-lattice-pruned.cc")
.file(&kaldi_lattice) // .file("resources/kaldi/src/lat/kaldi-lattice.cc")
.file("resources/kaldi/src/lat/lattice-functions.cc")
// .file("resources/kaldi/src/lat/minimize-lattice.cc")
// .file("resources/kaldi/src/lat/phone-align-lattice.cc")
// .file("resources/kaldi/src/lat/push-lattice.cc")
.file("resources/kaldi/src/lat/push-lattice.cc")
.file("resources/kaldi/src/lat/minimize-lattice.cc")
.file("resources/kaldi/src/lat/sausages.cc")
.file("resources/kaldi/src/lat/word-align-lattice-lexicon.cc")
.file("resources/kaldi/src/lat/word-align-lattice.cc")
// util
// .file("resources/kaldi/src/util/kaldi-holder.cc")
.file("resources/kaldi/src/util/kaldi-holder.cc")
.file("resources/kaldi/src/util/kaldi-io.cc")
// .file("resources/kaldi/src/util/kaldi-semaphore.cc")
// .file("resources/kaldi/src/util/kaldi-table.cc")
// .file("resources/kaldi/src/util/kaldi-thread.cc")
.file("resources/kaldi/src/util/kaldi-semaphore.cc")
.file("resources/kaldi/src/util/kaldi-table.cc")
.file("resources/kaldi/src/util/kaldi-thread.cc")
.file("resources/kaldi/src/util/parse-options.cc")
.file("resources/kaldi/src/util/simple-io-funcs.cc")
// .file("resources/kaldi/src/util/simple-options.cc")
.file("resources/kaldi/src/util/simple-options.cc")
.file("resources/kaldi/src/util/text-utils.cc")
//ivector
.file("resources/kaldi/src/ivector/ivector-extractor.cc")
// online2
.file("resources/kaldi/src/online2/online-endpoint.cc")
.file("resources/kaldi/src/online2/online-feature-pipeline.cc")
// .file("resources/kaldi/src/online2/online-gmm-decodable.cc")
// .file("resources/kaldi/src/online2/online-gmm-decoding.cc")
// .file("resources/kaldi/src/online2/online-ivector-feature.cc")
.file("resources/kaldi/src/online2/online-ivector-feature.cc")
// .file("resources/kaldi/src/online2/online-nnet2-decoding-threaded.cc")
// .file("resources/kaldi/src/online2/online-nnet2-decoding.cc")
// .file("resources/kaldi/src/online2/online-nnet2-feature-pipeline.cc")
.file("resources/kaldi/src/online2/online-nnet2-feature-pipeline.cc")
.file("resources/kaldi/src/online2/online-nnet3-decoding.cc")
// .file("resources/kaldi/src/online2/online-nnet3-incremental-decoding.cc")
// .file("resources/kaldi/src/online2/online-nnet3-wake-word-faster-decoder.cc")
@ -252,5 +281,6 @@ fn main() {
.file("resources/kaldi/src/online2/online-timing.cc")
// .file("resources/kaldi/src/online2/onlinebin-util.cc")
.compile("libkaldi");
.try_compile("libkaldi")
.unwrap();
}

View File

@ -1,3 +1 @@
#include "model.h"
#include "kaldi_recognizer.h"
#include "spk_model.h"
#include "vosk_api.h"

View File

@ -12,29 +12,58 @@ pub fn main() {
let audio_file_path = std::env::args().nth(1)
.expect("Please specify an audio file to run STT on");
let audio_file = File::open(audio_file_path).unwrap();
let mut reader = Reader::new(audio_file).unwrap();
let mut reader = audrey::open(audio_file_path).unwrap();
let desc = reader.description();
assert_eq!(1, desc.channel_count(),
"The channel count is required to be one, at least for now");
let audio_buf :Vec<_> = if desc.sample_rate() == SAMPLE_RATE {
reader.samples().map(|s| s.unwrap()).collect()
} else {
// We need to interpolate to the target sample rate
let interpolator = Linear::new([0i16], [0]);
let conv = Converter::from_hz_to_hz(
from_iter(reader.samples::<i16>().map(|s| [s.unwrap()])),
interpolator,
desc.sample_rate() as f64,
SAMPLE_RATE as f64);
conv.until_exhausted().map(|v| v[0]).collect()
};
let model = VoskModel::new("./models/en-small");
let sess = model.create_session(Default::default());
let mut sess = model.create_session(Default::default());
let mut buff: Vec<i16> = Vec::with_capacity(1600);
let mut samples_reader = reader.samples();
loop {
buff.clear();
while let Some(s) = samples_reader.next() {
buff.push(s.unwrap());
if buff.len() >= 16000 {
break;
}
}
if buff.is_empty() {
break;
}
println!("feed {}", buff.len());
if model.feed(&mut sess, buff.as_slice()) {
println!("{}", model.get_result(&mut sess));
} else {
println!("{}", model.get_partial_result(&mut sess));
}
}
println!("{}", model.get_final_result(&mut sess));
// let audio_buf :Vec<_> = if desc.sample_rate() == SAMPLE_RATE {
// .map(|s| s.unwrap()).collect()
// } else {
// // We need to interpolate to the target sample rate
// let interpolator = Linear::new([0i16], [0]);
// let conv = Converter::from_hz_to_hz(
// from_iter(reader.samples::<i16>().map(|s| [s.unwrap()])),
// interpolator,
// desc.sample_rate() as f64,
// SAMPLE_RATE as f64);
// conv.until_exhausted().map(|v| v[0]).collect()
// };
// audio_buf

0
models/README.md Normal file
View File

1
resources/kaldi Submodule

@ -0,0 +1 @@
Subproject commit 99a8e612b18245022db5102014af08a8634a01da

1
resources/openfst Submodule

@ -0,0 +1 @@
Subproject commit 256f83e52112a5cd37e37a34beff2c4f0eae4660

1
resources/vosk-api Submodule

@ -0,0 +1 @@
Subproject commit de94ef553770919286c35dd4d8aed1df71d24be8

View File

@ -1,3 +1,10 @@
extern crate cblas;
extern crate openblas_src;
#[used]
#[no_mangle]
pub static FLAGS_v: i32 = 0;
mod ffi {
#![allow(non_upper_case_globals)]
#![allow(non_camel_case_types)]

View File

@ -5,7 +5,7 @@ use crate::ffi;
use crate::session::{VoskSession, VoskSessionConfig};
pub struct VoskModel {
pub(crate) inner: ffi::Model,
pub(crate) inner: *mut ffi::VoskModel,
}
impl VoskModel {
@ -13,44 +13,44 @@ impl VoskModel {
let root = unsafe { CString::from_vec_unchecked(root.as_ref().to_string_lossy().as_bytes().to_vec()) };
Self {
inner: unsafe { ffi::Model::new(root.as_c_str().as_ptr()) },
inner: unsafe { ffi::vosk_model_new(root.as_c_str().as_ptr()) },
}
}
#[inline]
pub fn create_session(&self, cfg: VoskSessionConfig) -> VoskSession {
VoskSession::new(&self.inner, cfg)
VoskSession::new(self.inner, cfg)
}
#[inline]
pub fn feed(&self, sess: &mut VoskSession, data: &[i16]) -> bool {
unsafe { ffi::KaldiRecognizer_AcceptWaveform1(&mut sess.inner, data.as_ptr(), data.len() as _) }
unsafe { ffi::vosk_recognizer_accept_waveform_s(sess.inner, data.as_ptr(), data.len() as _) == 1 }
}
#[inline]
pub fn get_result(&self, sess: &mut VoskSession) -> String {
let cstr = unsafe { CStr::from_ptr(ffi::KaldiRecognizer_Result(&mut sess.inner)) };
let cstr = unsafe { CStr::from_ptr(ffi::vosk_recognizer_result(sess.inner)) };
cstr.to_string_lossy().to_string()
cstr.to_string_lossy().to_owned().to_string()
}
#[inline]
pub fn get_partial_result(&self, sess: &mut VoskSession) -> String {
let cstr = unsafe { CStr::from_ptr(ffi::KaldiRecognizer_PartialResult(&mut sess.inner)) };
let cstr = unsafe { CStr::from_ptr(ffi::vosk_recognizer_partial_result(sess.inner)) };
cstr.to_string_lossy().to_string()
cstr.to_string_lossy().to_owned().to_string()
}
#[inline]
pub fn get_final_result(&self, sess: &mut VoskSession) -> String {
let cstr = unsafe { CStr::from_ptr(ffi::KaldiRecognizer_FinalResult(&mut sess.inner)) };
let cstr = unsafe { CStr::from_ptr(ffi::vosk_recognizer_final_result(sess.inner)) };
cstr.to_string_lossy().to_string()
cstr.to_string_lossy().to_owned().to_string()
}
}
impl Drop for VoskModel {
fn drop(&mut self) {
unsafe { self.inner.destruct() }
unsafe { ffi::vosk_model_free(self.inner) }
}
}

View File

@ -79,23 +79,24 @@ impl VoskSessionConfig {
}
pub struct VoskSession {
pub(crate) inner: ffi::KaldiRecognizer
pub(crate) inner: *mut ffi::VoskRecognizer
}
impl VoskSession {
pub(crate) fn new(model: *const ffi::Model, cfg: VoskSessionConfig) -> Self {
pub(crate) fn new(model: *const ffi::VoskModel, cfg: VoskSessionConfig) -> Self {
if let Some(_cfg) = &cfg.spk_root {
unimplemented!()
// VoskSession {
// inner: ffi::KaldiRecognizer::new1(model as *mut ffi::Model, cfg.freq)
// }
} else if let Some(grammar) = &cfg.grammar {
VoskSession {
inner: unsafe { ffi::KaldiRecognizer::new2(model as *mut ffi::Model, cfg.freq, grammar.as_c_str().as_ptr()) }
}
unimplemented!()
// VoskSession {
// inner: unsafe { ffi::KaldiRecognizer::new2(model as *mut ffi::Model, cfg.freq, grammar.as_c_str().as_ptr()) }
// }
} else {
VoskSession {
inner: unsafe { ffi::KaldiRecognizer::new(model as *mut ffi::Model, cfg.freq) }
inner: unsafe { ffi::vosk_recognizer_new(model as *mut ffi::VoskModel, cfg.freq) }
}
}
}

View File

@ -3,7 +3,7 @@ use std::path::Path;
use std::ffi::CString;
pub struct SpeakerModel {
pub(crate) inner: ffi::SpkModel
pub(crate) inner: *mut ffi::VoskSpkModel
}
impl SpeakerModel {
@ -11,13 +11,13 @@ impl SpeakerModel {
let root = unsafe { CString::from_vec_unchecked(root.to_string_lossy().as_bytes().to_vec()) };
Self {
inner: unsafe { ffi::SpkModel::new(root.as_c_str().as_ptr()) }
inner: unsafe { ffi::vosk_spk_model_new(root.as_c_str().as_ptr()) }
}
}
}
impl Drop for SpeakerModel {
fn drop(&mut self) {
unsafe { self.inner.destruct() }
unsafe { ffi::vosk_spk_model_free(self.inner) }
}
}