andrey/yarnn
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

YARNN v2

Browse Source
This commit is contained in:
Andrey Tkachenko 2019-07-14 13:53:44 +04:00
parent a0c47a5a86
commit d72497421c
58 changed files with 564 additions and 302 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,6 +1,6 @@
# Generated by Cargo
# will have compiled files and executables
/target/
target/
# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html

122
Cargo.lock generated
View File

@ -5,6 +5,24 @@ name = "autocfg"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "blas"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"blas-sys 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.59 (registry+https://github.com/rust-lang/crates.io-index)",
"num-complex 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "blas-sys"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.59 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "byteorder"
version = "1.3.2"
@ -19,13 +37,31 @@ dependencies = [
"ppv-lite86 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "cblas"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cblas-sys 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.59 (registry+https://github.com/rust-lang/crates.io-index)",
"num-complex 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "cblas-sys"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.59 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "getrandom"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.59 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -38,7 +74,7 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.58"
version = "0.2.59"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -50,14 +86,27 @@ dependencies = [
]
[[package]]
name = "poc"
version = "0.1.0"
name = "num-complex"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"mnist 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_distr 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "num-traits"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "openblas-src"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "ppv-lite86"
version = "0.2.5"
@ -69,7 +118,7 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"getrandom 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.59 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_chacha 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_core 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -114,14 +163,69 @@ name = "spin"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "yarnn"
version = "0.1.0"
dependencies = [
"rand 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand_distr 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "yarnn-example-mnist"
version = "0.1.0"
dependencies = [
"mnist 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"yarnn 0.1.0",
"yarnn-model-mnist 0.1.0",
]
[[package]]
name = "yarnn-example-vgg16-demo"
version = "0.1.0"
dependencies = [
"yarnn 0.1.0",
"yarnn-model-vgg16 0.1.0",
]
[[package]]
name = "yarnn-model-mnist"
version = "0.1.0"
dependencies = [
"yarnn 0.1.0",
]
[[package]]
name = "yarnn-model-vgg16"
version = "0.1.0"
dependencies = [
"yarnn 0.1.0",
]
[[package]]
name = "yarnn-native-blas"
version = "0.1.0"
dependencies = [
"blas 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
"cblas 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"openblas-src 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[metadata]
"checksum autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0e49efa51329a5fd37e7c79db4621af617cd4e3e5bc224939808d076077077bf"
"checksum blas 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e4b3b6399f2fe6e70f780b06e278a64770aae273e0005c3860f94fc2f5f01ba7"
"checksum blas-sys 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "13b1b279ceb25d7c4faaea95a5f7addbe7d8c34f9462044bd8e630cebcfc2440"
"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
"checksum c2-chacha 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7d64d04786e0f528460fc884753cf8dddcc466be308f6026f8e355c41a0e4101"
"checksum cblas 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d82f331add33eceb4c41cb28d878049b96f56577016daf190831e94e4aece5db"
"checksum cblas-sys 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b6feecd82cce51b0204cf063f0041d69f24ce83f680d87514b004248e7b0fa65"
"checksum getrandom 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "e65cce4e5084b14874c4e7097f38cab54f47ee554f9194673456ea379dcc4c55"
"checksum lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bc5729f27f159ddd61f4df6228e827e86643d4d3e7c32183cb30a1c08f604a14"
"checksum libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)" = "6281b86796ba5e4366000be6e9e18bf35580adf9e63fbe2294aadb587613a319"
"checksum libc 0.2.59 (registry+https://github.com/rust-lang/crates.io-index)" = "3262021842bf00fe07dbd6cf34ff25c99d7a7ebef8deea84db72be3ea3bb0aff"
"checksum mnist 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "25f19bfda80095b4294000bbb50506f028149ed0ddb7fabf46ebb673b91626bc"
"checksum num-complex 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "fcb0cf31fb3ff77e6d2a6ebd6800df7fdcd106f2ad89113c9130bcd07f93dffc"
"checksum num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "6ba9a427cfca2be13aa6f6403b0b7e7368fe982bfa16fccc450ce74c46cd9b32"
"checksum openblas-src 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3533e568814bee9620fcc529158408384404bae5b277c73c73d66ca03fceb7"
"checksum ppv-lite86 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e3cbf9f658cdb5000fcf6f362b8ea2ba154b9f146a61c7a20d647034c6b6561b"
"checksum rand 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d47eab0e83d9693d40f825f86948aa16eff6750ead4bdffc4ab95b8b3a7f052c"
"checksum rand_chacha 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e193067942ef6f485a349a113329140d0ab9e2168ce92274499bb0e9a4190d9d"

31
Cargo.toml
View File

@ -1,12 +1,21 @@
[package]
name = "poc"
version = "0.1.0"
authors = ["Andrey Tkachenko <andreytkachenko64@gmail.com>"]
edition = "2018"
build = "build.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[workspace]
members = [
"yarnn",
"yarnn-native-blas",
"yarnn-models/mnist",
"yarnn-models/vgg16",
"yarnn-examples/mnist",
# "yarnn-examples/mnist-wasm",
# "yarnn-examples/mnist-blas",
"yarnn-examples/vgg16-demo",
]
[patch.crates-io]
yarnn = { path = './yarnn' }
yarnn-native-blas = { path = './yarnn-native-blas' }
yarnn-model-mnist = { path = './yarnn-models/mnist' }
yarnn-model-vgg16 = { path = './yarnn-models/vgg16' }
[dependencies]
mnist = "0.4.0"
rand = "0.7.0"
rand_distr = "0.2.1"

69
README.md
View File

@ -1,2 +1,67 @@
# poc
Rust Deep Neural Network Framework
# [ W. I. P. ]
## Yet Another Rust Neural Network framework aka YARNN
Inspired by `darknet` and `leaf`
## What it can right now:
* not requires `std` (only `alloc` for tensor allocations, bump allocator is ok, so it can be compiled to stm32f4 board)
* available layers: `Linear`, `ReLu`, `Sigmoid`, `Softmax`(no backward), `Conv2d`, `ZeroPadding2d`, `MaxPool2d`, `AvgPool2d`(no backward), `Flatten`
* available optimizers: `Sgd`, `Adam`, `RMSProp`
* available losses: `CrossEntropy`(no forward), `MeanSquareError`
* available backends: `Native`, `NativeBlas`(no convolution yet)
## What it will can (I hope):
### 1st stage:
* example of running `yarnn` in browser using `WASM`
* example of running `yarnn` on `stm32f4` board
* finish `AvgPool2d` backpropogation
* add `Dropout` layer
* add `BatchNorm` layer
* convolution with BLAS support
### 2nd stage:
* `CUDA` support
* `OpenCL` support
### 3rd stage:
* `DepthwiseConv2d` layer
* `Conv3d` layer
* `Deconv2d` layer
* `k210` backend
## Model definition example
```rust
use yarnn::model;
use yarnn::layer::*;
use yarnn::layers::*;
model! {
MnistConvModel (h: u32, w: u32, c: u32) {
input_shape: (c, h, w),
layers: {
Conv2d<N, B, O> {
filters: 8
},
ReLu<N, B>,
MaxPool2d<N, B> {
pool: (2, 2)
},
Conv2d<N, B, O> {
filters: 8
},
ReLu<N, B>,
MaxPool2d<N, B> {
pool: (2, 2)
},
Flatten<N, B>,
Linear<N, B, O> {
units: 10
},
Sigmoid<N, B>
}
}
}
```
## Contributors are welcome

BIN
dataset/fashion-mnist/t10k-images-idx3-ubyte.gz Normal file
View File

Binary file not shown.

BIN
dataset/fashion-mnist/t10k-labels-idx1-ubyte.gz Normal file
View File

Binary file not shown.

BIN
dataset/fashion-mnist/train-images-idx3-ubyte.gz Normal file
View File

Binary file not shown.

BIN
dataset/fashion-mnist/train-labels-idx1-ubyte.gz Normal file
View File

Binary file not shown.

6
src/backends/mod.rs
View File

@ -1,6 +0,0 @@
mod native;
mod conv2d;
mod pool2d;
mod gemm;
pub use self::native::*;

33
src/conv_model.rs
View File

@ -1,33 +0,0 @@
use crate::layers::*;
use crate::layer::*;
use crate::model;
model! {
ConvModel (h: u32, w: u32, c: u32) {
input_shape: (c, h, w),
layers: {
Conv2d<N, B, O> {
filters: 8
},
ReLu<N, B>,
MaxPool2d<N, B> {
pool: (2, 2)
},
Conv2d<N, B, O> {
filters: 8
},
ReLu<N, B>,
MaxPool2d<N, B> {
pool: (2, 2)
},
Flatten<N, B>,
Linear<N, B, O> {
units: 10
},
Sigmoid<N, B>
}
}
}

20
src/dense_model.rs
View File

@ -1,20 +0,0 @@
use crate::layers::*;
use crate::layer::*;
use crate::model;
model! {
DenseModel (h: u32, w: u32, _c: u32) {
input_shape: (h * w),
layers: {
Flatten<N, B>,
Linear<N, B, O> {
units: 16
},
ReLu<N, B>,
Linear<N, B, O> {
units: 10
},
Softmax<N, B>
}
}
}

9
yarnn-examples/mnist-blas/Cargo.toml Normal file
View File

@ -0,0 +1,9 @@
[package]
name = "yarnn-example-mnist-blas"
version = "0.1.0"
authors = ["Andrey Tkachenko <andreytkachenko64@gmail.com>"]
edition = "2018"
[dependencies]
mnist = "0.4.0"
yarnn = "0.1.0"

3
yarnn-examples/mnist-blas/src/main.rs Normal file
View File

@ -0,0 +1,3 @@
fn main() {
println!("Hello, world!");
}

8
yarnn-examples/mnist-wasm/Cargo.toml Normal file
View File

@ -0,0 +1,8 @@
[package]
name = "yarnn-example-mnist-wasm"
version = "0.1.0"
authors = ["Andrey Tkachenko <andreytkachenko64@gmail.com>"]
edition = "2018"
[dependencies]
yarnn = "0.1.0"

3
yarnn-examples/mnist-wasm/src/main.rs Normal file
View File

@ -0,0 +1,3 @@
fn main() {
println!("Hello, world!");
}

10
yarnn-examples/mnist/Cargo.toml Normal file
View File

@ -0,0 +1,10 @@
[package]
name = "yarnn-example-mnist"
version = "0.1.0"
authors = ["Andrey Tkachenko <andreytkachenko64@gmail.com>"]
edition = "2018"
[dependencies]
mnist = "0.4.0"
yarnn = "0.1.0"
yarnn-model-mnist = "0.1.0"

56
src/main.rs → yarnn-examples/mnist/src/main.rs
View File

@ -1,39 +1,9 @@
#![feature(specialization, trait_alias)]
#![recursion_limit="128"]
pub mod layer;
pub mod layers;
pub mod optimizer;
pub mod optimizers;
pub mod backend;
pub mod backends;
pub mod loss;
pub mod losses;
pub mod tensor;
pub mod params;
mod conv_model;
mod dense_model;
mod vgg_model;
#[macro_use]
mod macros;
use self::backends::{Native, NativeTensorF32};
use self::optimizers::*;
use self::layer::*;
use crate::backend::*;
use self::tensor::Tensor;
use self::loss::Loss;
use self::losses::CrossEntropyLoss;
use yarnn::prelude::*;
use yarnn::native::{Native, NativeTensorF32};
use yarnn_model_mnist::*;
use yarnn::losses::CrossEntropyLoss;
use yarnn::optimizers::Adam;
use mnist::{Mnist, MnistBuilder};
// use crate::conv_model::*;
use crate::dense_model::*;
fn calc_accuracy<N, B: Backend<N>>(back: &B, pred: &B::Tensor, targets: &[u8]) -> f32 {
let mut vec = vec![0.0; pred.shape().size()];
@ -65,17 +35,14 @@ fn calc_accuracy<N, B: Backend<N>>(back: &B, pred: &B::Tensor, targets: &[u8]) -
(positives as f32) / (total as f32)
}
fn main() {
const BATCH_SIZE: usize = 64;
let backend = Native;
// let optimizer = Sgd::new(0.1, 0.01, true);
let optimizer = WeightDecay::new(0.001, Adam::default());
// let optimizer = Adam::default();
let optimizer = Adam::default();
let mut model = DenseModel::new(28, 28, 1);
// let mut model = ConvModel::new(28, 28, 1);
// let mut model = MnistDenseModel::new(28, 28, 1);
let mut model = MnistConvModel::new(28, 28, 1);
println!("{}", &model);
@ -85,7 +52,7 @@ fn main() {
let loss = CrossEntropyLoss::new();
let Mnist { trn_img, trn_lbl, tst_img, tst_lbl, .. } = MnistBuilder::new()
.base_path("./dataset/mnist")
.base_path("../../dataset/mnist")
.label_format_digit()
.finalize();
@ -112,7 +79,7 @@ fn main() {
backend.load_tensor_u8(&mut targets0, &tmp[..]);
for epoch in 1 ..= 10 {
for epoch in 1 ..= 4 {
println!("epoch {}", epoch);
for step in 0 .. (60000 / BATCH_SIZE) {
@ -142,6 +109,3 @@ fn main() {
println!("Accuracy {}", calc_accuracy(&backend, test_ctx.outputs(), targets0_slice));
}
}

9
yarnn-examples/vgg16-demo/Cargo.toml Normal file
View File

@ -0,0 +1,9 @@
[package]
name = "yarnn-example-vgg16-demo"
version = "0.1.0"
authors = ["Andrey Tkachenko <andreytkachenko64@gmail.com>"]
edition = "2018"
[dependencies]
yarnn = "0.1.0"
yarnn-model-vgg16 = "0.1.0"

10
yarnn-examples/vgg16-demo/src/main.rs Normal file
View File

@ -0,0 +1,10 @@
use yarnn::native::Native;
use yarnn::optimizers::Adam;
use yarnn_model_vgg16::Vgg16Model;
fn main() {
let vgg16: Vgg16Model<f32, Native, Adam<_, _>> = Vgg16Model::new(224, 224, 3);
println!("{}", vgg16);
}

8
yarnn-models/mnist/Cargo.toml Normal file
View File

@ -0,0 +1,8 @@
[package]
name = "yarnn-model-mnist"
version = "0.1.0"
authors = ["Andrey Tkachenko <andreytkachenko64@gmail.com>"]
edition = "2018"
[dependencies]
yarnn = "0.1.0"

65
yarnn-models/mnist/src/lib.rs Normal file
View File

@ -0,0 +1,65 @@
#![feature(trait_alias)]
pub use self::dense::MnistDenseModel;
pub use self::conv::MnistConvModel;
mod dense {
use yarnn::model;
use yarnn::layer::*;
use yarnn::layers::*;
model! {
MnistDenseModel (h: u32, w: u32, _c: u32) {
input_shape: (h * w),
layers: {
Flatten<N, B>,
Linear<N, B, O> {
units: 16
},
ReLu<N, B>,
Linear<N, B, O> {
units: 10
},
Softmax<N, B>
}
}
}
}
mod conv {
use yarnn::model;
use yarnn::layer::*;
use yarnn::layers::*;
model! {
MnistConvModel (h: u32, w: u32, c: u32) {
input_shape: (c, h, w),
layers: {
Conv2d<N, B, O> {
filters: 8
},
ReLu<N, B>,
MaxPool2d<N, B> {
pool: (2, 2)
},
Conv2d<N, B, O> {
filters: 8
},
ReLu<N, B>,
MaxPool2d<N, B> {
pool: (2, 2)
},
Flatten<N, B>,
Linear<N, B, O> {
units: 10
},
Sigmoid<N, B>
}
}
}
}

8
yarnn-models/vgg16/Cargo.toml Normal file
View File

@ -0,0 +1,8 @@
[package]
name = "yarnn-model-vgg16"
version = "0.1.0"
authors = ["Andrey Tkachenko <andreytkachenko64@gmail.com>"]
edition = "2018"
[dependencies]
yarnn = "0.1.0"

12
src/vgg_model.rs → yarnn-models/vgg16/src/lib.rs
View File

@ -1,6 +1,8 @@
use crate::layers::*;
use crate::layer::*;
use crate::model;
#![feature(trait_alias)]
use yarnn::layers::*;
use yarnn::layer::*;
use yarnn::model;
model! {
Vgg16Model (h: u32, w: u32, c: u32) {
@ -58,11 +60,11 @@ model! {
Flatten<N, B>,
Linear<N, B, O> { units: 4096 },
ReLu<N, B>,
// Dropout { mu: 0.5 },
// Dropout { 0.5 }, // TODO
Linear<N, B, O> { units: 4096 },
ReLu<N, B>,
// Dropout { mu: 0.5 },
// Dropout { 0.5 }, // TODO
Linear<N, B, O> { units: 1000 },
Softmax<N, B>

1
yarnn-native-blas/Cargo.toml
View File

@ -3,6 +3,7 @@ name = "yarnn-native-blas"
version = "0.1.0"
authors = ["Andrey Tkachenko <andreytkachenko64@gmail.com>"]
edition = "2018"
build = "build.rs"
[dependencies]
openblas-src = {version = "0.7.0", features = ["system"]}

0
build.rs → yarnn-native-blas/build.rs
View File

266
yarnn-native-blas/src/lib.rs
View File

@ -1,164 +1,164 @@
pub struct NativeBlas {
inner: Native,
}
// pub struct NativeBlas {
// inner: Native,
// }
impl NativeBlas {
// impl NativeBlas {
}
// }
impl Backend<f32> for NativeBlas {
type Tensor = NativeTensorF32;
// impl Backend<f32> for NativeBlas {
// type Tensor = NativeTensorF32;
#[inline]
fn store_tensor_f32(&self, t: &Self::Tensor, data: &mut [f32]) {
self.inner.store_tensor_f32(t, data);
}
// #[inline]
// fn store_tensor_f32(&self, t: &Self::Tensor, data: &mut [f32]) {
// self.inner.store_tensor_f32(t, data);
// }
#[inline]
fn load_tensor_u8(&self, t: &mut Self::Tensor, data: &[u8]) {
self.inner.load_tensor_u8(t, data);
}
// #[inline]
// fn load_tensor_u8(&self, t: &mut Self::Tensor, data: &[u8]) {
// self.inner.load_tensor_u8(t, data);
// }
#[inline]
fn load_tensor_f32(&self, t: &mut Self::Tensor, data: &[f32]) {
self.inner.load_tensor_f32(t, data);
}
// #[inline]
// fn load_tensor_f32(&self, t: &mut Self::Tensor, data: &[f32]) {
// self.inner.load_tensor_f32(t, data);
// }
#[inline]
fn scalar_f32(&self, val: f32) -> N {
self.inner.scalar_f32(val);
}
// #[inline]
// fn scalar_f32(&self, val: f32) -> N {
// self.inner.scalar_f32(val);
// }
#[inline]
fn fill_scalar(&self, t: &mut Self::Tensor, scalar: N) {
self.inner.scalar_f32(t, scalar);
}
// #[inline]
// fn fill_scalar(&self, t: &mut Self::Tensor, scalar: N) {
// self.inner.scalar_f32(t, scalar);
// }
#[inline]
fn fill_random(&self, t: &mut Self::Tensor, from: N, to: N) {
self.inner.scalar_f32(t, from, to);
}
// #[inline]
// fn fill_random(&self, t: &mut Self::Tensor, from: N, to: N) {
// self.inner.scalar_f32(t, from, to);
// }
#[inline]
fn print_tensor(&self, t: &Self::Tensor) {
self.inner.scalar_f32(t);
}
}
// #[inline]
// fn print_tensor(&self, t: &Self::Tensor) {
// self.inner.scalar_f32(t);
// }
// }
impl BackendGemm<f32> for Native {
fn matmul(&self, dst: &mut Self::Tensor, a: &Self::Tensor, b: &Self::Tensor) {
let a_shape = a.shape();
let b_shape = b.shape();
let c_shape = dst.shape().clone();
// impl BackendGemm<f32> for Native {
// fn matmul(&self, dst: &mut Self::Tensor, a: &Self::Tensor, b: &Self::Tensor) {
// let a_shape = a.shape();
// let b_shape = b.shape();
// let c_shape = dst.shape().clone();
assert_eq!(a_shape.get(0), c_shape.get(0));
assert_eq!(b_shape.get(1), c_shape.get(1));
// assert_eq!(a_shape.get(0), c_shape.get(0));
// assert_eq!(b_shape.get(1), c_shape.get(1));
assert_eq!(a_shape.dims, 2);
assert_eq!(b_shape.dims, 2);
// assert_eq!(a_shape.dims, 2);
// assert_eq!(b_shape.dims, 2);
let m = a_shape.get(0) as usize;
let n = b_shape.get(1) as usize;
let k = b_shape.get(0) as usize;
// let m = a_shape.get(0) as usize;
// let n = b_shape.get(1) as usize;
// let k = b_shape.get(0) as usize;
unsafe {
sgemm('N' as u8, 'N' as u8,
n, m, k,
1.0,
b.read(), n,
a.read(), k,
0.0,
&mut dst.write(), n);
}
}
// unsafe {
// sgemm('N' as u8, 'N' as u8,
// n, m, k,
// 1.0,
// b.read(), n,
// a.read(), k,
// 0.0,
// &mut dst.write(), n);
// }
// }
fn matmul_nt(&self, dst: &mut Self::Tensor, a: &Self::Tensor, b: &Self::Tensor) {
let a_shape = a.shape();
let b_shape = b.shape();
let c_shape = dst.shape().clone();
// fn matmul_nt(&self, dst: &mut Self::Tensor, a: &Self::Tensor, b: &Self::Tensor) {
// let a_shape = a.shape();
// let b_shape = b.shape();
// let c_shape = dst.shape().clone();
assert_eq!(a_shape.get(0), c_shape.get(0));
assert_eq!(b_shape.get(0), c_shape.get(1));
// assert_eq!(a_shape.get(0), c_shape.get(0));
// assert_eq!(b_shape.get(0), c_shape.get(1));
assert_eq!(a_shape.dims, 2);
assert_eq!(b_shape.dims, 2);
// assert_eq!(a_shape.dims, 2);
// assert_eq!(b_shape.dims, 2);
let m = a_shape.get(0) as usize;
let n = b_shape.get(0) as usize;
let k = b_shape.get(1) as usize;
// let m = a_shape.get(0) as usize;
// let n = b_shape.get(0) as usize;
// let k = b_shape.get(1) as usize;
unsafe {
sgemm('T' as u8, 'N' as u8,
n, m, k,
1.0,
b.read(), k,
a.read(), k,
0.0,
&mut dst.write(), n);
}
}
// unsafe {
// sgemm('T' as u8, 'N' as u8,
// n, m, k,
// 1.0,
// b.read(), k,
// a.read(), k,
// 0.0,
// &mut dst.write(), n);
// }
// }
fn matmul_tn(&self, dst: &mut Self::Tensor, a: &Self::Tensor, b: &Self::Tensor) {
let a_shape = a.shape();
let b_shape = b.shape();
let c_shape = dst.shape().clone();
// fn matmul_tn(&self, dst: &mut Self::Tensor, a: &Self::Tensor, b: &Self::Tensor) {
// let a_shape = a.shape();
// let b_shape = b.shape();
// let c_shape = dst.shape().clone();
assert_eq!(a_shape.get(1), c_shape.get(0));
assert_eq!(b_shape.get(1), c_shape.get(1));
// assert_eq!(a_shape.get(1), c_shape.get(0));
// assert_eq!(b_shape.get(1), c_shape.get(1));
assert_eq!(a_shape.dims, 2);
assert_eq!(b_shape.dims, 2);
// assert_eq!(a_shape.dims, 2);
// assert_eq!(b_shape.dims, 2);
let m = a_shape.get(1) as usize;
let n = b_shape.get(1) as usize;
let k = b_shape.get(0) as usize;
// let m = a_shape.get(1) as usize;
// let n = b_shape.get(1) as usize;
// let k = b_shape.get(0) as usize;
unsafe {
sgemm('N' as u8, 'T' as u8,
n, m, k,
1.0,
b.read(), n,
a.read(), m,
0.0,
&mut dst.write(), n);
}
}
// unsafe {
// sgemm('N' as u8, 'T' as u8,
// n, m, k,
// 1.0,
// b.read(), n,
// a.read(), m,
// 0.0,
// &mut dst.write(), n);
// }
// }
fn matmul_tt(&self, _dst: &mut Self::Tensor, _a: &Self::Tensor, _b: &Self::Tensor) {
unimplemented!();
}
}
// fn matmul_tt(&self, _dst: &mut Self::Tensor, _a: &Self::Tensor, _b: &Self::Tensor) {
// unimplemented!();
// }
// }
impl BackendAxpy<f32> for Native {
fn axpy(&self, dst: &mut Self::Tensor, scale: f32, x: &Self::Tensor) {
let dst_size = dst.shape().size();
// impl BackendAxpy<f32> for Native {
// fn axpy(&self, dst: &mut Self::Tensor, scale: f32, x: &Self::Tensor) {
// let dst_size = dst.shape().size();
assert!(x.shape() == dst.shape());
// assert!(x.shape() == dst.shape());
unsafe {
blas::saxpy(
dst_size as i32,
scale,
x.read(),
1,
dst.write(),
1
);
}
}
}
// unsafe {
// blas::saxpy(
// dst_size as i32,
// scale,
// x.read(),
// 1,
// dst.write(),
// 1
// );
// }
// }
// }
impl BackendScale<f32> for Native {
fn scale(&self, dst: &mut Self::Tensor, scale: f32) {
let dst_size = dst.shape().size();
// impl BackendScale<f32> for Native {
// fn scale(&self, dst: &mut Self::Tensor, scale: f32) {
// let dst_size = dst.shape().size();
unsafe {
blas::sscal(
dst_size as i32,
scale,
dst.write(),
1
);
}
}
}
// unsafe {
// blas::sscal(
// dst_size as i32,
// scale,
// dst.write(),
// 1
// );
// }
// }
// }

9
yarnn/Cargo.toml Normal file
View File

@ -0,0 +1,9 @@
[package]
name = "yarnn"
version = "0.1.0"
authors = ["Andrey Tkachenko <andreytkachenko64@gmail.com>"]
edition = "2018"
[dependencies]
rand = "0.7.0"
rand_distr = "0.2.1"

0
src/backend.rs → yarnn/src/backend.rs
View File

6
src/layer.rs → yarnn/src/layer.rs
View File

@ -34,7 +34,7 @@ impl <T, N, B, O> Optimizable<N, B, O> for T
default fn optimize(&mut self, _backend: &B, _optimizer: &O) {}
}
pub trait AbstractLayer<N, B: Backend<N>, O: Optimizer<N, B>>: std::fmt::Display {
pub trait AbstractLayer<N, B: Backend<N>, O: Optimizer<N, B>>: core::fmt::Display {
type Context: LayerContext<N, B>;
fn forward(&mut self, backend: &B, inputs: &B::Tensor, ctx: &mut Self::Context);
@ -135,12 +135,12 @@ impl <N, B, O, L> LayerImpl<N, B, O, L>
}
}
impl <N, B, O, L> std::fmt::Display for LayerImpl<N, B, O, L>
impl <N, B, O, L> core::fmt::Display for LayerImpl<N, B, O, L>
where B: Backend<N>,
O: Optimizer<N, B>,
L: Layer<N, B>
{
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
write!(f, "{} -> ", self.layer.input_shape())?;
write!(f, "{}", self.layer.name())?;
writeln!(f, " -> {}", self.layer.output_shape())?;

0
src/layers/avgpool2d.rs → yarnn/src/layers/avgpool2d.rs
View File

4
src/layers/chain.rs → yarnn/src/layers/chain.rs
View File

@ -70,13 +70,13 @@ impl<N, B, O, L, R> Chain<N, B, O, L, R>
}
}
impl<N, B, O, L, R> std::fmt::Display for Chain<N, B, O, L, R>
impl<N, B, O, L, R> core::fmt::Display for Chain<N, B, O, L, R>
where B: Backend<N>,
O: Optimizer<N, B>,
L: AbstractLayer<N, B, O>,
R: AbstractLayer<N, B, O>,
{
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
self.left.fmt(f)?;
self.right.fmt(f)?;

0
src/layers/conv2d.rs → yarnn/src/layers/conv2d.rs
View File

2
src/layers/flatten.rs → yarnn/src/layers/flatten.rs
View File

@ -1,7 +1,7 @@
use crate::tensor::TensorShape;
use crate::backend::{Backend, BackendCopy};
use crate::layer::Layer;
use std::marker::PhantomData;
use core::marker::PhantomData;
#[derive(Default)]
pub struct FlattenConfig;

0
src/layers/linear.rs → yarnn/src/layers/linear.rs
View File

0
src/layers/maxpool2d.rs → yarnn/src/layers/maxpool2d.rs
View File

0
src/layers/mod.rs → yarnn/src/layers/mod.rs
View File

2
src/layers/relu.rs → yarnn/src/layers/relu.rs
View File

@ -1,7 +1,7 @@
use crate::tensor::TensorShape;
use crate::backend::{Backend, BackendReLu};
use crate::layer::Layer;
use std::marker::PhantomData;
use core::marker::PhantomData;
#[derive(Default)]
pub struct ReLuConfig;

2
src/layers/sigmoid.rs → yarnn/src/layers/sigmoid.rs
View File

@ -1,7 +1,7 @@
use crate::tensor::TensorShape;
use crate::backend::{Backend, BackendSigmoid};
use crate::layer::Layer;
use std::marker::PhantomData;
use core::marker::PhantomData;
#[derive(Default)]
pub struct SigmoidConfig;

2
src/layers/softmax.rs → yarnn/src/layers/softmax.rs
View File

@ -1,7 +1,7 @@
use crate::tensor::TensorShape;
use crate::backend::{Backend, BackendSoftmax};
use crate::layer::Layer;
use std::marker::PhantomData;
use core::marker::PhantomData;
#[derive(Default)]
pub struct SoftmaxConfig;

2
src/layers/zeropadding2d.rs → yarnn/src/layers/zeropadding2d.rs
View File

@ -1,7 +1,7 @@
use crate::tensor::TensorShape;
use crate::backend::{Backend, BackendCopy};
use crate::layer::Layer;
use std::marker::PhantomData;
use core::marker::PhantomData;
#[derive(Default)]
pub struct ZeroPadding2dConfig {

27
yarnn/src/lib.rs Normal file
View File

@ -0,0 +1,27 @@
#![feature(specialization, trait_alias)]
#![recursion_limit="128"]
pub mod layer;
pub mod layers;
pub mod optimizer;
pub mod optimizers;
pub mod backend;
pub mod native;
pub mod loss;
pub mod losses;
pub mod tensor;
pub mod params;
#[macro_use]
mod macros;
pub mod prelude {
pub use super::backend::*;
pub use super::loss::*;
pub use super::tensor::*;
pub use super::layer::*;
}

0
src/loss.rs → yarnn/src/loss.rs
View File

3
src/losses/cross_entropy.rs → yarnn/src/losses/cross_entropy.rs
View File

@ -1,7 +1,8 @@
use crate::loss::Loss;
use crate::backend::{Backend, BackendSub};
use core::marker::PhantomData;
use std::marker::PhantomData;
pub struct CrossEntropyLoss<N, B> {
_m: PhantomData<fn(N, B)>

0
src/losses/mod.rs → yarnn/src/losses/mod.rs
View File

2
src/losses/mse.rs → yarnn/src/losses/mse.rs
View File

@ -2,7 +2,7 @@ use crate::loss::Loss;
use crate::backend::{Backend, BackendMse};
use crate::tensor::Tensor;
use std::marker::PhantomData;
use core::marker::PhantomData;
pub struct MeanSquareErrorLoss<N, B> {

10
src/macros.rs → yarnn/src/macros.rs
View File

@ -41,7 +41,7 @@ macro_rules! sequential {
macro_rules! sequential_impl {
($p:expr, $t:ty { $($name:ident : $val:expr),* }) => {{
#[allow(unused_imports)]
use std::convert::TryInto;
use core::convert::TryInto;
#[allow(unused_mut)]
let mut params = <$t as $crate::layer::Layer<_, _>>::Config::default();
@ -56,7 +56,7 @@ macro_rules! sequential_impl {
($p:expr, $t:ty { $($name:ident : $val:expr),* }, $($tt:tt)*) => {{
#[allow(unused_imports)]
use std::convert::TryInto;
use core::convert::TryInto;
#[allow(unused_mut)]
let mut params = <$t as $crate::layer::Layer<_, _>>::Config::default();
@ -127,7 +127,7 @@ macro_rules! model_impl {
O: $crate::optimizer::Optimizer<N, B>
{
inner: $crate::sequential_type!($($tt)*),
_m: std::marker::PhantomData<fn(N, B, O)>,
_m: core::marker::PhantomData<fn(N, B, O)>,
}
impl<N, B, O> $name<N, B, O>
@ -149,11 +149,11 @@ macro_rules! model_impl {
}
}
impl<N, B, O> std::fmt::Display for $name<N, B, O>
impl<N, B, O> core::fmt::Display for $name<N, B, O>
where B: $crate::backend::Backend<N> + $trait,
O: $crate::optimizer::Optimizer<N, B>
{
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
writeln!(f, "{} {{", stringify!($name))?;
write!(f, "{}", self.inner)?;
writeln!(f, "}}")?;

0
src/backends/conv2d.rs → yarnn/src/native/conv2d.rs
View File

0
src/backends/gemm.rs → yarnn/src/native/gemm.rs
View File

18
src/backends/native.rs → yarnn/src/native/mod.rs
View File

@ -1,11 +1,17 @@
mod conv2d;
mod pool2d;
mod gemm;
use crate::tensor::*;
use crate::backend::*;
use std::fmt;
use std::fmt::Write;
use self::conv2d::*;
use self::pool2d::*;
use self::gemm::*;
use core::fmt;
use core::fmt::Write;
use rand_distr::{Normal, Distribution};
use super::conv2d::*;
use super::pool2d::*;
use super::gemm::*;
pub struct NativeTensorF32 {
@ -584,7 +590,7 @@ impl BackendSoftmax<f32> for Native {
assert!(i + (axis - 1) < size);
// max(x)
let mut max_x = std::f32::NEG_INFINITY;
let mut max_x = core::f32::NEG_INFINITY;
for j in 0..axis {
let val = x_s[i + j];
if val > max_x {

6
src/backends/pool2d.rs → yarnn/src/native/pool2d.rs
View File

@ -11,7 +11,7 @@ pub fn maxpool2d(y: &mut [f32], x: &[f32],
for y_x in 0..y_cols {
let mut xi = s_row * y_y * x_cols + s_col * y_x;
let mut max = std::f32::NEG_INFINITY;
let mut max = core::f32::NEG_INFINITY;
for _ in 0..w_rows {
for w_x in 0..w_cols {
let val = x[(xi + w_x) as usize];
@ -42,7 +42,7 @@ pub fn maxpool2d_backward(dx: &mut [f32], x: &[f32], dy: &[f32],
for dy_x in 0..y_cols {
let mut xi = s_row * dy_y * x_cols + s_col * dy_x;
let mut max = std::f32::NEG_INFINITY;
let mut max = core::f32::NEG_INFINITY;
let mut max_idx = -1;
for _ in 0..w_rows {
for w_x in 0..w_cols {
@ -108,7 +108,7 @@ pub fn avgpool2d_backward(dx: &mut [f32], x: &[f32], dy: &[f32],
for dy_x in 0..y_cols {
let mut xi = s_row * dy_y * x_cols + s_col * dy_x;
let mut max = std::f32::NEG_INFINITY;
let mut max = core::f32::NEG_INFINITY;
let mut max_idx = -1;
for _ in 0..w_rows {
for w_x in 0..w_cols {

0
src/network.rs → yarnn/src/network.rs
View File

0
src/optimizer.rs → yarnn/src/optimizer.rs
View File

8
src/optimizers/adam.rs → yarnn/src/optimizers/adam.rs
View File

@ -1,8 +1,8 @@
use std::marker::PhantomData;
use crate::backend::{Backend, BackendAdam};
use crate::optimizer::{Optimizer, OptimizerContext};
use crate::tensor::{Tensor, TensorShape};
use std::cell::Cell;
use core::marker::PhantomData;
use core::cell::Cell;
pub struct AdamContext<N, B>
@ -87,10 +87,10 @@ impl<N, B: Backend<N> + BackendAdam<N>> Optimizer<N, B> for Adam<N, B> {
if self.amsgrad {
backend.maximum(&mut ctx.vhats, &ctx.vels);
backend.adam_p(params, backend.scalar_f32(-lr_t), &ctx.moms, &ctx.vhats, backend.scalar_f32(self.epsilon.unwrap_or(std::f32::EPSILON)));
backend.adam_p(params, backend.scalar_f32(-lr_t), &ctx.moms, &ctx.vhats, backend.scalar_f32(self.epsilon.unwrap_or(core::f32::EPSILON)));
} else {
// p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)
backend.adam_p(params, backend.scalar_f32(-lr_t), &ctx.moms, &ctx.vels, backend.scalar_f32(self.epsilon.unwrap_or(std::f32::EPSILON)));
backend.adam_p(params, backend.scalar_f32(-lr_t), &ctx.moms, &ctx.vels, backend.scalar_f32(self.epsilon.unwrap_or(core::f32::EPSILON)));
}
}
}

2
src/optimizers/mod.rs → yarnn/src/optimizers/mod.rs
View File

@ -8,7 +8,7 @@ pub use self::rmsprop::*;
use crate::backend::{Backend, BackendAxpys};
use crate::optimizer::Optimizer;
use std::marker::PhantomData;
use core::marker::PhantomData;
pub struct WeightDecay<N, B, O>
where B: Backend<N>,

4
src/optimizers/rmsprop.rs → yarnn/src/optimizers/rmsprop.rs
View File

@ -1,7 +1,7 @@
use std::marker::PhantomData;
use crate::backend::{Backend, BackendAdam};
use crate::optimizer::{Optimizer, OptimizerContext};
use crate::tensor::{Tensor, TensorShape};
use core::marker::PhantomData;
pub struct RMSPropContext<N, B>
@ -62,6 +62,6 @@ impl<N, B: Backend<N> + BackendAdam<N>> Optimizer<N, B> for RMSProp<N, B> {
backend.axpys(&mut ctx.accum, backend.scalar_f32(1.0 - self.rho), grads);
// new_p = p - lr * g / (K.sqrt(new_a) + self.epsilon)
backend.adam_p(params, backend.scalar_f32(-self.learning_rate), &grads, &ctx.accum, backend.scalar_f32(self.epsilon.unwrap_or(std::f32::EPSILON)));
backend.adam_p(params, backend.scalar_f32(-self.learning_rate), &grads, &ctx.accum, backend.scalar_f32(self.epsilon.unwrap_or(core::f32::EPSILON)));
}
}

2
src/optimizers/sgd.rs → yarnn/src/optimizers/sgd.rs
View File

@ -1,7 +1,7 @@
use std::marker::PhantomData;
use crate::backend::{Backend, BackendScale, BackendAxpy, BackendAdd};
use crate::optimizer::{Optimizer, OptimizerContext};
use crate::tensor::{Tensor, TensorShape};
use core::marker::PhantomData;
pub struct SgdContext<N, B>

0
src/params.rs → yarnn/src/params.rs
View File

4
src/tensor.rs → yarnn/src/tensor.rs
View File

@ -1,5 +1,5 @@
use std::fmt;
use core::fmt;
pub struct TensorShapeIter<'a> {
shape: &'a TensorShape,
@ -144,7 +144,7 @@ impl TensorShape {
self.shape[index] = val;
}
pub fn slice<R: std::ops::RangeBounds<u32>>(&self, range: R) -> TensorShape {
pub fn slice<R: core::ops::RangeBounds<u32>>(&self, range: R) -> TensorShape {
let mut shape = [0u32; 4];
let mut dims = 0;