[WIP] Plugins system MVP

Cargo.toml

@@ -6,6 +6,7 @@ members = [
   "helix-tui",
   "helix-syntax",
   "helix-lsp",
+  "helix-plugin",
 ]
 
 # Build helix-syntax in release mode to make the code path faster in development.

helix-plugin/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "helix-plugin"
+version = "0.1.0"
+edition = "2018"
+
+[dependencies]
+wasmtime = "0.28"
+wasmtime-wasi = { version = "0.28", features = ["tokio"] }
+anyhow = "1"
+futures-util = { version = "0.3", features = ["std", "async-await"], default-features = false }
+
+[dev-dependencies]
+tokio-macros = "1.3"
+tokio = "1.8"

helix-plugin/README.md

@@ -0,0 +1,16 @@
+# Helix plugins system MVP
+
+This is still an early alpha, you should expect frequent breaking changes.
+The intend is to take inspiration from [CodeMirror 6](https://codemirror.net/6/docs/ref/).
+
+Plugins for Helix are written in any language that can be compiled to WASM (Web Assembly).
+This implies:
+- No language language lock-in, user can use what they're most comfortable with to extend and customize their experience.
+- Sandboxing, a plugin can do only what it has been allowed to.
+- Portability, .wasm and .wat files are platform agnostic and can be packaged as is for any platform.
+
+API is described by .witx files and bindings for some languages can be generated by [`witx-bindgen`](https://github.com/bytecodealliance/witx-bindgen).
+As of now, `witx-bindgen` is still in prototype stage.
+
+Host side code is generated with [BLABLA TODO]
+

helix-plugin/src/lib.rs

@@ -0,0 +1,253 @@
+use anyhow::{Context, Result};
+use std::collections::HashMap;
+use std::fmt;
+use std::path::PathBuf;
+use wasmtime_wasi::tokio::WasiCtxBuilder;
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct PluginName(String);
+
+impl From<&str> for PluginName {
+    fn from(name: &str) -> Self {
+        Self(name.to_owned())
+    }
+}
+
+impl From<String> for PluginName {
+    fn from(name: String) -> Self {
+        Self(name)
+    }
+}
+
+impl fmt::Display for PluginName {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+/// Describes a path accessible from sandbox
+#[derive(Debug)]
+pub enum DirDef {
+    Mirrored {
+        path: PathBuf,
+    },
+    Mapped {
+        host_path: PathBuf,
+        guest_path: PathBuf,
+    },
+}
+
+impl DirDef {
+    /// Preopens the directory definition for the given WASI context
+    pub fn preopen(&self, wasi_builder: WasiCtxBuilder) -> Result<WasiCtxBuilder> {
+        use std::fs::File;
+        use wasmtime_wasi::Dir;
+
+        let (host_path, guest_path) = match self {
+            DirDef::Mirrored { path } => (path.as_path(), path.as_path()),
+            DirDef::Mapped {
+                host_path,
+                guest_path,
+            } => (host_path.as_path(), guest_path.as_path()),
+        };
+
+        let host_dir = unsafe {
+            // SAFETY: user is deciding for himself folders that should be accessibles
+            Dir::from_std_file(File::open(host_path)?)
+        };
+
+        wasi_builder.preopened_dir(host_dir, guest_path)
+    }
+}
+
+pub struct HelixCtx {
+    wasi: wasmtime_wasi::WasiCtx,
+}
+
+pub type HelixStore = wasmtime::Store<HelixCtx>;
+
+pub struct PluginDef {
+    pub name: PluginName,
+    pub path: PathBuf,
+    pub dependencies: Vec<PluginName>,
+}
+
+pub struct Plugin {
+    instance: wasmtime::Instance,
+}
+
+impl Plugin {
+    pub fn get_typed_func<Params, Results>(
+        &self,
+        store: &mut HelixStore,
+        name: &str,
+    ) -> Result<wasmtime::TypedFunc<Params, Results>>
+    where
+        Params: wasmtime::WasmParams,
+        Results: wasmtime::WasmResults,
+    {
+        let func = self
+            .instance
+            .get_typed_func::<Params, Results, _>(store, name)?;
+        Ok(func)
+    }
+
+    pub fn get_func(&self, store: &mut HelixStore, name: &str) -> Option<wasmtime::Func> {
+        self.instance.get_func(store, name)
+    }
+}
+
+pub struct PluginsSystem {
+    pub store: HelixStore,
+    pub plugins: HashMap<PluginName, Plugin>,
+}
+
+impl PluginsSystem {
+    pub fn builder() -> PluginsSystemBuilder {
+        PluginsSystemBuilder::default()
+    }
+}
+
+pub struct PluginsSystemBuilder {
+    debug_info: bool,
+    definitions: Vec<PluginDef>,
+    preopened_dirs: Vec<DirDef>,
+    linker_fn: Box<dyn Fn(&mut wasmtime::Linker<HelixCtx>) -> Result<()>>,
+}
+
+impl Default for PluginsSystemBuilder {
+    fn default() -> Self {
+        Self {
+            debug_info: false,
+            definitions: vec![],
+            preopened_dirs: vec![],
+            linker_fn: Box::new(|_| Ok(())),
+        }
+    }
+}
+
+impl PluginsSystemBuilder {
+    pub fn plugin(&mut self, def: PluginDef) -> &mut Self {
+        self.definitions.push(def);
+        self
+    }
+
+    pub fn plugins(&mut self, mut defs: Vec<PluginDef>) -> &mut Self {
+        self.definitions.append(&mut defs);
+        self
+    }
+
+    pub fn dir(&mut self, dir: DirDef) -> &mut Self {
+        self.preopened_dirs.push(dir);
+        self
+    }
+
+    pub fn dirs(&mut self, mut dirs: Vec<DirDef>) -> &mut Self {
+        self.preopened_dirs.append(&mut dirs);
+        self
+    }
+
+    pub fn debug_info(&mut self, debug: bool) -> &mut Self {
+        self.debug_info = debug;
+        self
+    }
+
+    pub fn linker<F>(&mut self, linker_fn: F) -> &mut Self
+    where
+        F: Fn(&mut wasmtime::Linker<HelixCtx>) -> Result<()> + 'static,
+    {
+        self.linker_fn = Box::new(linker_fn);
+        self
+    }
+
+    /// Instanciate the plugins system, compiling and linking WASM modules as appropriate.
+    pub async fn build(&self) -> Result<PluginsSystem> {
+        use wasmtime::{Config, Engine, Linker, Module, Store};
+
+        let mut config = Config::new();
+        config.debug_info(self.debug_info);
+        config.async_support(true);
+
+        let engine = Engine::new(&config)?;
+
+        // Compile plugins
+        let modules: HashMap<PluginName, Module> = self
+            .definitions
+            .iter()
+            .map(|def| {
+                println!("Compile {}", def.name);
+                Module::from_file(&engine, &def.path)
+                    .with_context(|| format!("module creation failed for `{}`", def.name))
+                    .map(|module| (def.name.clone(), module))
+            })
+            .collect::<Result<HashMap<PluginName, Module>>>()?;
+
+        // Dumb link order resolution: a good one would detect cycles to give a better error, in
+        // our case a link error will arise at link time
+        let mut link_order = Vec::new();
+        for def in &self.definitions {
+            let insert_pos = if let Some(pos) = link_order.iter().position(|name| name == &def.name)
+            {
+                pos
+            } else {
+                link_order.push(def.name.clone());
+                link_order.len() - 1
+            };
+
+            for dep in &def.dependencies {
+                if let Some(pos) = link_order.iter().position(|name| name == dep) {
+                    if pos > insert_pos {
+                        link_order.remove(pos);
+                        link_order.insert(insert_pos, dep.clone());
+                    }
+                } else {
+                    link_order.insert(insert_pos, dep.clone());
+                };
+            }
+        }
+
+        // Link and create instances
+        let mut wasi_builder = WasiCtxBuilder::new();
+        for dir in &self.preopened_dirs {
+            wasi_builder = dir
+                .preopen(wasi_builder)
+                .with_context(|| format!("couldn't preopen directory `{:?}`", dir))?;
+        }
+        let wasi = wasi_builder.build();
+
+        let ctx = HelixCtx { wasi };
+
+        let mut store = Store::new(&engine, ctx);
+
+        let mut linker = Linker::new(&engine);
+        wasmtime_wasi::add_to_linker(&mut linker, |s: &mut HelixCtx| &mut s.wasi)?;
+
+        (self.linker_fn)(&mut linker).context("couldn't add host-provided modules to linker")?;
+
+        let mut plugins: HashMap<PluginName, Plugin> = HashMap::new();
+
+        for name in link_order {
+            let module = modules.get(&name).expect("this module was compiled above");
+
+            let instance = linker
+                .instantiate_async(&mut store, module)
+                .await
+                .with_context(|| format!("couldn't instanciate `{}`", name))?;
+
+            // Register the instance with the linker for the next linking
+            linker.instance(&mut store, &name.0, instance)?;
+
+            plugins.insert(name, Plugin { instance });
+        }
+
+        // Call `init` function on all loaded plugins
+        for plugin in plugins.values_mut() {
+            // Call this plugin's `init` function if one is defined
+            if let Ok(func) = plugin.get_typed_func::<(), ()>(&mut store, "init") {
+                func.call_async(&mut store, ()).await?;
+            }
+        }
+
+        Ok(PluginsSystem { store, plugins })
+    }
+}

helix-plugin/tests/call_host.wat

@@ -0,0 +1,7 @@
+(module
+  (import "host" "callback" (func $callback))
+
+  (func (export "init")
+    call $callback
+  )
+)

helix-plugin/tests/linking.rs

@@ -0,0 +1,88 @@
+use helix_plugin::{DirDef, PluginDef, PluginName, PluginsSystem};
+use std::path::PathBuf;
+
+/// Based on https://docs.wasmtime.dev/examples-rust-linking.html
+#[tokio_macros::test]
+async fn two_plugins() {
+    let linking1 = PluginName::from("linking1");
+    let linking2 = PluginName::from("linking2");
+
+    let mut system = PluginsSystem::builder()
+        .plugin(PluginDef {
+            name: linking1.clone(),
+            path: PathBuf::from("tests/linking1.wat"),
+            dependencies: vec![linking2.clone()],
+        })
+        .plugin(PluginDef {
+            name: linking2.clone(),
+            path: PathBuf::from("tests/linking2.wat"),
+            dependencies: vec![],
+        })
+        .build()
+        .await
+        .unwrap();
+
+    let run = system
+        .plugins
+        .get(&linking1)
+        .unwrap()
+        .get_typed_func::<(), ()>(&mut system.store, "run")
+        .unwrap();
+    run.call_async(&mut system.store, ()).await.unwrap();
+
+    let double = system
+        .plugins
+        .get(&linking2)
+        .unwrap()
+        .get_typed_func::<i32, i32>(&mut system.store, "double")
+        .unwrap();
+    assert_eq!(double.call_async(&mut system.store, 5).await.unwrap(), 10);
+}
+
+#[tokio_macros::test(flavor = "multi_thread")]
+async fn wasi_preopen_dir() {
+    let name = PluginName::from("read_file");
+
+    // read_file.wasm is a program reading `./Cargo.toml` file and does nothing with it
+
+    PluginsSystem::builder()
+        .dir(DirDef::Mirrored {
+            path: PathBuf::from("./"),
+        })
+        .plugin(PluginDef {
+            name: name.clone(),
+            path: PathBuf::from("tests/read_file.wasm"),
+            dependencies: vec![],
+        })
+        .build()
+        .await
+        .unwrap();
+}
+
+#[tokio_macros::test]
+async fn callback_to_host() {
+    use std::sync::atomic::{AtomicBool, Ordering};
+
+    static CANARY: AtomicBool = AtomicBool::new(false);
+
+    PluginsSystem::builder()
+        .plugin(PluginDef {
+            name: PluginName::from("call_host"),
+            path: PathBuf::from("tests/call_host.wat"),
+            dependencies: vec![],
+        })
+        .linker(|l| {
+            l.func_wrap("host", "callback", || CANARY.store(true, Ordering::Relaxed))?;
+            Ok(())
+        })
+        .build()
+        .await
+        .unwrap();
+
+    assert_eq!(CANARY.load(Ordering::Relaxed), true);
+}
+
+#[tokio_macros::test]
+async fn callback_to_content() {
+    todo!()
+}

helix-plugin/tests/linking1.wat

@@ -0,0 +1,24 @@
+;; from https://docs.wasmtime.dev/examples-rust-linking.html
+
+(module
+  (import "linking2" "double" (func $double (param i32) (result i32)))
+  (import "linking2" "log" (func $log (param i32 i32)))
+  (import "linking2" "memory" (memory 1))
+  (import "linking2" "memory_offset" (global $offset i32))
+
+  (func (export "run")
+    ;; Call into the other module to double our number, and we could print it
+    ;; here but for now we just drop it
+    i32.const 2
+    call $double
+    drop
+
+    ;; Our `data` segment initialized our imported memory, so let's print the
+    ;; string there now.
+    global.get $offset
+    i32.const 14
+    call $log
+  )
+
+  (data (global.get $offset) "Hello, world!\n")
+)

helix-plugin/tests/linking2.wat

@@ -0,0 +1,36 @@
+;; from https://docs.wasmtime.dev/examples-rust-linking.html
+
+(module
+  (type $fd_write_ty (func (param i32 i32 i32 i32) (result i32)))
+  (import "wasi_snapshot_preview1" "fd_write" (func $fd_write (type $fd_write_ty)))
+
+  (func (export "double") (param i32) (result i32)
+    local.get 0
+    i32.const 2
+    i32.mul
+  )
+
+  (func (export "log") (param i32 i32)
+    ;; store the pointer in the first iovec field
+    i32.const 4
+    local.get 0
+    i32.store
+
+    ;; store the length in the first iovec field
+    i32.const 4
+    local.get 1
+    i32.store offset=4
+
+    ;; call the `fd_write` import
+    i32.const 1     ;; stdout fd
+    i32.const 4     ;; iovs start
+    i32.const 1     ;; number of iovs
+    i32.const 0     ;; where to write nwritten bytes
+    call $fd_write
+    drop
+  )
+
+  (memory (export "memory") 2)
+  (global (export "memory_offset") i32 (i32.const 65536))
+)
+