use std::ffi::CStr; use std::time::{Duration, Instant}; use onnxruntime::*; use structopt::{clap, StructOpt}; use std::path::PathBuf; #[structopt( name = "run", about = "Run a benchmark on an onnx model. Each worker runs the model in a loop in its own thead. Once done it will print the average time to run the model.", setting = clap::AppSettings::ColoredHelp )] #[derive(StructOpt)] struct Opt { /// The path to the onnx files to benchmark onnx: Vec, /// A comma separated list of symbolic_dimension=value. If a symbolic dimension is not /// specified, 1 will be used. #[structopt(long)] dims: Option, /// The number of worker threads to spawn #[structopt(long, default_value = "1")] workers: usize, /// The number of runs each worker will #[structopt(long, default_value = "1")] runs: usize, } use std::collections::HashMap; fn key_val_parse(str: &str) -> HashMap { let mut map = HashMap::new(); if str.is_empty() { return map; } for key_val in str.split(',') { let mut iter = key_val.split('='); let key = iter.next().expect("no ="); let val = iter .next() .expect("nothing after =") .parse() .expect("parse error"); assert!(iter.next().is_none(), "more than 1 ="); map.insert(key.to_owned(), val); } map } /// Get the size of a tensor, substituting symbolic dimentions. fn tensor_size( info: &TensorInfo, named_sizes: &mut HashMap, ) -> (OnnxTensorElementDataType, Vec) { let dims = info .symbolic_dims() .map(|d| match d { SymbolicDim::Symbolic(name) => { let name = name.to_str().unwrap(); named_sizes.get(name).cloned().unwrap_or_else(|| { eprintln!("name {} not specified, setting to 1", name); named_sizes.insert(name.to_owned(), 1); 1 }) } SymbolicDim::Fixed(x) => x, }) .collect(); (info.elem_type(), dims) } fn tensor_mut(name: &str, elem_type: OnnxTensorElementDataType, dims: &[usize]) -> Box> { use OnnxTensorElementDataType::*; println!("{:?} {} {:?}", elem_type, name, dims); match elem_type { Float => Box::new(Tensor::::init(dims, 0.0).unwrap()), Int64 => Box::new(Tensor::::init(dims, 0).unwrap()), Int32 => Box::new(Tensor::::init(dims, 0).unwrap()), t => panic!("Unsupported type {:?}", t), } } fn load_image(filename: &str, height: usize, width: usize) -> Vec { let img = image::open(filename).unwrap() // .resize_exact(width as _, height as _, image::imageops::FilterType::Triangle) .into_rgb(); img.as_flat_samples() .to_vec() .samples .into_iter() .map(|p| (p as f32) / 255.0) .collect() } fn tensor_with_size( name: &str, info: &TensorInfo, named_sizes: &mut HashMap, ) -> Box> { let (ty, dims) = tensor_size(info, named_sizes); use OnnxTensorElementDataType::*; println!("{:?} {} {:?}", ty, name, dims); match ty { Float => match name { "input" => Box::new(Tensor::::new(&dims, load_image("/data/andrey_/Images/me.jpg", dims[2], dims[3])).unwrap()), _ => Box::new(Tensor::::init(&dims, 0.0).unwrap()), }, Int64 => Box::new(Tensor::::init(&dims, 0).unwrap()), Int32 => Box::new(Tensor::::init(&dims, 0).unwrap()), t => panic!("Unsupported type {:?}", t), } } fn main() -> Result<()> { let env = Env::new(LoggingLevel::Fatal, "test")?; let opt = Opt::from_args(); let mut so = SessionOptions::new()?; // so.set_execution_mode(ExecutionMode::Parallel)?; // so.add_tensorrt(0); so.add_cuda(0); // so.add_cpu(true); let mut map = if let Some(dims) = &opt.dims { key_val_parse(dims) } else { HashMap::new() }; let batch_size = 4; map.insert("batch_size".into(), batch_size); for path in &opt.onnx { println!("model {:?}", path); let session = match Session::new(&env, path, &so) { Ok(sess) => sess, Err(err) => { eprintln!("error: {}\n", err); continue; } }; let metadata = session.metadata(); eprintln!("name: {}", metadata.producer_name()); eprintln!("graph_name: {}", metadata.graph_name()); eprintln!("domain: {}", metadata.domain()); eprintln!("description: {}", metadata.description()); let mut input_names: Vec = vec![]; let mut input_tensors: Vec>> = vec![]; for (i, input) in session.inputs().enumerate() { if let Some(tensor_info) = input.tensor_info() { input_names.push(input.name()); input_tensors.push(tensor_with_size(input.name().as_str(), &tensor_info, &mut map)); } else { println!("input {}: {:?} {:?}", i, &*input.name(), input.onnx_type()); } } let mut output_names: Vec = vec![]; let mut output_sizes: Vec<(OnnxTensorElementDataType, Vec)> = vec![]; for (i, output) in session.outputs().enumerate() { if let Some(tensor_info) = output.tensor_info() { output_names.push(output.name()); output_sizes.push(tensor_size(&tensor_info, &mut map)); } else { println!( "output {}: {:?} {:?}", i, &*output.name(), output.onnx_type() ); } } let in_names: Vec<&CStr> = input_names.iter().map(|x| x.as_c_str()).collect(); let in_vals: Vec<&Val> = input_tensors.iter().map(|x| x.as_ref().as_ref()).collect(); let out_names: Vec<&CStr> = output_names.iter().map(|x| x.as_c_str()).collect(); let ro = RunOptions::new(); let before = Instant::now(); let mut res = session .run_raw(&ro, &in_names, &in_vals[..], &out_names) .expect("run"); let tensor = match res.pop().unwrap().as_tensor::() { Ok(t) => t, _ => panic!("something went wrong") }; println!("[{:?}] {}", tensor.dims(), before.elapsed().as_millis()) // println!("{:?}", out_vals[0].as_slice::()); // let out_vals[0].into(); // let total: Duration = times.iter().sum() // let avg = total / (times.len() as u32); // eprintln!("worker {} avg time: {:.2} ms", i, avg.as_secs_f64() * 1e3); } Ok(()) }