fetch and compile tree-sitter grammars in helix-term build

This restores much of the behavior that existed before this PR:
helix will build the grammars when compiling. The difference is that
now fetching is also done during the build phase and is done much
more quickly - both shallow and in parallel.

.github/workflows/build.yml

@@ -77,27 +77,14 @@ jobs:
           path: target
           key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
 
+      - name: Copy minimal languages config
+        run: cp .github/workflows/languages.toml ./languages.toml
+
       - name: Cache test tree-sitter grammar
         uses: actions/cache@v2.1.7
         with:
           path: runtime/grammars
-          key: ${{ runner.os }}-v2-tree-sitter-grammars-${{ hashFiles('**/Cargo.lock') }}
-
-      - run: cp .github/workflows/languages.toml ./languages.toml
-
-      - name: Download test tree-sitter grammar
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: ${{ matrix.cross }}
-          command: run
-          args: -- --fetch-grammars
-
-      - name: Build test tree-sitter grammar
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: ${{ matrix.cross }}
-          command: run
-          args: -- --build-grammars
+          key: ${{ runner.os }}-v2-tree-sitter-grammars-${{ hashFiles('languages.toml') }}
 
       - name: Run cargo test
         uses: actions-rs/cargo@v1

.github/workflows/release.yml

@@ -62,20 +62,6 @@ jobs:
           target: ${{ matrix.target }}
           override: true
 
-      - name: Fetch tree-sitter grammars
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: ${{ matrix.cross }}
-          command: run
-          args: --release --locked --target ${{ matrix.target }} -- --fetch-grammars
-
-      - name: Build tree-sitter grammars
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: ${{ matrix.cross }}
-          command: run
-          args: --release --locked --target ${{ matrix.target }} -- --build-grammars
-
       - name: Run cargo test
         uses: actions-rs/cargo@v1
         with:

flake.nix

@@ -35,6 +35,8 @@
       overrides = {
         crateOverrides = common: _: rec {
           helix-term = prev: {
+            # disable fetching and building of tree-sitter grammars in the helix-term build.rs
+            HELIX_DISABLE_AUTO_GRAMMAR_BUILD = "1";
             buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ];
             nativeBuildInputs = (prev.nativeBuildInputs or [ ]) ++ [ common.pkgs.makeWrapper ];
             preConfigure = ''

helix-term/Cargo.toml

@@ -67,9 +67,8 @@ grep-searcher = "0.1.8"
 # Remove once retain_mut lands in stable rust
 retain_mut = "0.1.7"
 
-# compiling grammars
-cc = { version = "1" }
-threadpool = { version = "1.0" }
-
 [target.'cfg(not(windows))'.dependencies]  # https://github.com/vorner/signal-hook/issues/100
 signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] }
+
+[build-dependencies]
+helix-loader = { version = "0.6", path = "../helix-loader" }

helix-term/build.rs

@@ -1,3 +1,4 @@
+use helix_loader::grammar::{build_grammars, fetch_grammars};
 use std::borrow::Cow;
 use std::process::Command;
 
@@ -14,5 +15,12 @@ fn main() {
         None => env!("CARGO_PKG_VERSION").into(),
     };
 
+    if std::env::var("HELIX_DISABLE_AUTO_GRAMMAR_BUILD").is_err() {
+        fetch_grammars().expect("Failed to fetch tree-sitter grammars");
+        build_grammars().expect("Failed to compile tree-sitter grammars");
+    }
+
+    println!("cargo:rerun-if-changed=../runtime/grammars/");
+
     println!("cargo:rustc-env=VERSION_AND_GIT_HASH={}", version);
 }