syntax: Split parsing and highlighting

Cargo.lock

@@ -381,6 +381,7 @@ dependencies = [
  "serde",
  "serde_json",
  "similar",
+ "slotmap",
  "smallvec",
  "tendril",
  "toml",

helix-core/Cargo.toml

@@ -22,6 +22,7 @@ unicode-segmentation = "1.8"
 unicode-width = "0.1"
 unicode-general-category = "0.4"
 # slab = "0.4.2"
+slotmap = "1.0"
 tree-sitter = "0.20"
 once_cell = "1.9"
 arc-swap = "1"

helix-core/src/indent.rs

@@ -454,7 +454,7 @@ pub fn change<I>(document: &Document, changes: I) -> Self
 
         let language_config = loader.language_config_for_scope("source.rust").unwrap();
         let highlight_config = language_config.highlight_config(&[]).unwrap();
-        let syntax = Syntax::new(&doc, highlight_config.clone());
+        let syntax = Syntax::new(&doc, highlight_config.clone(), std::sync::Arc::new(loader));
         let text = doc.slice(..);
         let tab_width = 4;
 

helix-term/src/ui/editor.rs

@@ -68,13 +68,12 @@ pub fn render_view(
         surface: &mut Surface,
         theme: &Theme,
         is_focused: bool,
-        loader: &syntax::Loader,
         config: &helix_view::editor::Config,
     ) {
         let inner = view.inner_area();
         let area = view.area;
 
-        let highlights = Self::doc_syntax_highlights(doc, view.offset, inner.height, theme, loader);
+        let highlights = Self::doc_syntax_highlights(doc, view.offset, inner.height, theme);
         let highlights = syntax::merge(highlights, Self::doc_diagnostics_highlights(doc, theme));
         let highlights: Box<dyn Iterator<Item = HighlightEvent>> = if is_focused {
             Box::new(syntax::merge(
@@ -121,8 +120,7 @@ pub fn doc_syntax_highlights<'doc>(
         doc: &'doc Document,
         offset: Position,
         height: u16,
-        theme: &Theme,
+        _theme: &Theme,
-        loader: &syntax::Loader,
     ) -> Box<dyn Iterator<Item = HighlightEvent> + 'doc> {
         let text = doc.text().slice(..);
         let last_line = std::cmp::min(
@@ -142,25 +140,8 @@ pub fn doc_syntax_highlights<'doc>(
         // TODO: range doesn't actually restrict source, just highlight range
         let highlights = match doc.syntax() {
             Some(syntax) => {
-                let scopes = theme.scopes();
                 syntax
-                    .highlight_iter(text.slice(..), Some(range), None, |language| {
+                    .highlight_iter(text.slice(..), Some(range), None)
-                        loader.language_configuration_for_injection_string(language)
-                            .and_then(|language_config| {
-                                let config = language_config.highlight_config(scopes)?;
-                                let config_ref = config.as_ref();
-                                // SAFETY: the referenced `HighlightConfiguration` behind
-                                // the `Arc` is guaranteed to remain valid throughout the
-                                // duration of the highlight.
-                                let config_ref = unsafe {
-                                    std::mem::transmute::<
-                                        _,
-                                        &'static syntax::HighlightConfiguration,
-                                    >(config_ref)
-                                };
-                                Some(config_ref)
-                            })
-                    })
                     .map(|event| event.unwrap())
                     .collect() // TODO: we collect here to avoid holding the lock, fix later
             }
@@ -1070,7 +1051,6 @@ fn render(&mut self, area: Rect, surface: &mut Surface, cx: &mut Context) {
 
         for (view, is_focused) in cx.editor.tree.views() {
             let doc = cx.editor.document(view.doc).unwrap();
-            let loader = &cx.editor.syn_loader;
             self.render_view(
                 doc,
                 view,
@@ -1078,7 +1058,6 @@ fn render(&mut self, area: Rect, surface: &mut Surface, cx: &mut Context) {
                 surface,
                 &cx.editor.theme,
                 is_focused,
-                loader,
                 &cx.editor.config,
             );
         }

helix-term/src/ui/markdown.rs

@@ -38,7 +38,7 @@ pub fn new(contents: String, config_loader: Arc<syntax::Loader>) -> Self {
 fn parse<'a>(
     contents: &'a str,
     theme: Option<&Theme>,
-    loader: &syntax::Loader,
+    loader: Arc<syntax::Loader>,
 ) -> tui::text::Text<'a> {
     // // also 2021-03-04T16:33:58.553 helix_lsp::transport [INFO] <- {"contents":{"kind":"markdown","value":"\n```rust\ncore::num\n```\n\n```rust\npub const fn saturating_sub(self, rhs:Self) ->Self\n```\n\n---\n\n```rust\n```"},"range":{"end":{"character":61,"line":101},"start":{"character":47,"line":101}}}
     // let text = "\n```rust\ncore::iter::traits::iterator::Iterator\n```\n\n```rust\nfn collect<B: FromIterator<Self::Item>>(self) -> B\nwhere\n        Self: Sized,\n```\n\n---\n\nTransforms an iterator into a collection.\n\n`collect()` can take anything iterable, and turn it into a relevant\ncollection. This is one of the more powerful methods in the standard\nlibrary, used in a variety of contexts.\n\nThe most basic pattern in which `collect()` is used is to turn one\ncollection into another. You take a collection, call [`iter`](https://doc.rust-lang.org/nightly/core/iter/traits/iterator/trait.Iterator.html) on it,\ndo a bunch of transformations, and then `collect()` at the end.\n\n`collect()` can also create instances of types that are not typical\ncollections. For example, a [`String`](https://doc.rust-lang.org/nightly/core/iter/std/string/struct.String.html) can be built from [`char`](type@char)s,\nand an iterator of [`Result<T, E>`](https://doc.rust-lang.org/nightly/core/result/enum.Result.html) items can be collected\ninto `Result<Collection<T>, E>`. See the examples below for more.\n\nBecause `collect()` is so general, it can cause problems with type\ninference. As such, `collect()` is one of the few times you'll see\nthe syntax affectionately known as the 'turbofish': `::<>`. This\nhelps the inference algorithm understand specifically which collection\nyou're trying to collect into.\n\n# Examples\n\nBasic usage:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled: Vec<i32> = a.iter()\n                         .map(|&x| x * 2)\n                         .collect();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nNote that we needed the `: Vec<i32>` on the left-hand side. This is because\nwe could collect into, for example, a [`VecDeque<T>`](https://doc.rust-lang.org/nightly/core/iter/std/collections/struct.VecDeque.html) instead:\n\n```rust\nuse std::collections::VecDeque;\n\nlet a = [1, 2, 3];\n\nlet doubled: VecDeque<i32> = a.iter().map(|&x| x * 2).collect();\n\nassert_eq!(2, doubled[0]);\nassert_eq!(4, doubled[1]);\nassert_eq!(6, doubled[2]);\n```\n\nUsing the 'turbofish' instead of annotating `doubled`:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled = a.iter().map(|x| x * 2).collect::<Vec<i32>>();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nBecause `collect()` only cares about what you're collecting into, you can\nstill use a partial type hint, `_`, with the turbofish:\n\n```rust\nlet a = [1, 2, 3];\n\nlet doubled = a.iter().map(|x| x * 2).collect::<Vec<_>>();\n\nassert_eq!(vec![2, 4, 6], doubled);\n```\n\nUsing `collect()` to make a [`String`](https://doc.rust-lang.org/nightly/core/iter/std/string/struct.String.html):\n\n```rust\nlet chars = ['g', 'd', 'k', 'k', 'n'];\n\nlet hello: String = chars.iter()\n    .map(|&x| x as u8)\n    .map(|x| (x + 1) as char)\n    .collect();\n\nassert_eq!(\"hello\", hello);\n```\n\nIf you have a list of [`Result<T, E>`](https://doc.rust-lang.org/nightly/core/result/enum.Result.html)s, you can use `collect()` to\nsee if any of them failed:\n\n```rust\nlet results = [Ok(1), Err(\"nope\"), Ok(3), Err(\"bad\")];\n\nlet result: Result<Vec<_>, &str> = results.iter().cloned().collect();\n\n// gives us the first error\nassert_eq!(Err(\"nope\"), result);\n\nlet results = [Ok(1), Ok(3)];\n\nlet result: Result<Vec<_>, &str> = results.iter().cloned().collect();\n\n// gives us the list of answers\nassert_eq!(Ok(vec![1, 3]), result);\n```";
@@ -98,14 +98,13 @@ fn to_span(text: pulldown_cmark::CowStr) -> Span {
                         let syntax = loader
                             .language_configuration_for_injection_string(language)
                             .and_then(|config| config.highlight_config(theme.scopes()))
-                            .map(|config| Syntax::new(&rope, config));
+                            .map(|config| Syntax::new(&rope, config, loader.clone()));
 
                         if let Some(syntax) = syntax {
                             // if we have a syntax available, highlight_iter and generate spans
                             let mut highlights = Vec::new();
 
-                            for event in syntax.highlight_iter(rope.slice(..), None, None, |_| None)
+                            for event in syntax.highlight_iter(rope.slice(..), None, None) {
-                            {
                                 match event.unwrap() {
                                     HighlightEvent::HighlightStart(span) => {
                                         highlights.push(span);
@@ -211,7 +210,11 @@ impl Component for Markdown {
     fn render(&mut self, area: Rect, surface: &mut Surface, cx: &mut Context) {
         use tui::widgets::{Paragraph, Widget, Wrap};
 
-        let text = parse(&self.contents, Some(&cx.editor.theme), &self.config_loader);
+        let text = parse(
+            &self.contents,
+            Some(&cx.editor.theme),
+            self.config_loader.clone(),
+        );
 
         let par = Paragraph::new(text)
             .wrap(Wrap { trim: false })
@@ -229,7 +232,7 @@ fn required_size(&mut self, viewport: (u16, u16)) -> Option<(u16, u16)> {
         if padding >= viewport.1 || padding >= viewport.0 {
             return None;
         }
-        let contents = parse(&self.contents, None, &self.config_loader);
+        let contents = parse(&self.contents, None, self.config_loader.clone());
         // TODO: account for tab width
         let max_text_width = (viewport.0 - padding).min(120);
         let mut text_width = 0;

helix-term/src/ui/picker.rs

@@ -221,13 +221,8 @@ fn render(&mut self, area: Rect, surface: &mut Surface, cx: &mut Context) {
 
             let offset = Position::new(first_line, 0);
 
-            let highlights = EditorView::doc_syntax_highlights(
+            let highlights =
-                doc,
+                EditorView::doc_syntax_highlights(doc, offset, area.height, &cx.editor.theme);
-                offset,
-                area.height,
-                &cx.editor.theme,
-                &cx.editor.syn_loader,
-            );
             EditorView::render_text_highlights(
                 doc,
                 offset,

helix-view/src/document.rs

@@ -359,7 +359,7 @@ pub fn open(
         path: &Path,
         encoding: Option<&'static encoding::Encoding>,
         theme: Option<&Theme>,
-        config_loader: Option<&syntax::Loader>,
+        config_loader: Option<Arc<syntax::Loader>>,
     ) -> Result<Self, Error> {
         // Open the file if it exists, otherwise assume it is a new file (and thus empty).
         let (rope, encoding) = if path.exists() {
@@ -498,12 +498,12 @@ fn save_impl<F: Future<Output = LspFormatting>>(
     }
 
     /// Detect the programming language based on the file type.
-    pub fn detect_language(&mut self, theme: Option<&Theme>, config_loader: &syntax::Loader) {
+    pub fn detect_language(&mut self, theme: Option<&Theme>, config_loader: Arc<syntax::Loader>) {
         if let Some(path) = &self.path {
             let language_config = config_loader
                 .language_config_for_file_name(path)
                 .or_else(|| config_loader.language_config_for_shebang(self.text()));
-            self.set_language(theme, language_config);
+            self.set_language(theme, language_config, Some(config_loader));
         }
     }
 
@@ -579,11 +579,12 @@ pub fn set_language(
         &mut self,
         theme: Option<&Theme>,
         language_config: Option<Arc<helix_core::syntax::LanguageConfiguration>>,
+        loader: Option<Arc<helix_core::syntax::Loader>>,
     ) {
-        if let Some(language_config) = language_config {
+        if let (Some(language_config), Some(loader)) = (language_config, loader) {
             let scopes = theme.map(|theme| theme.scopes()).unwrap_or(&[]);
             if let Some(highlight_config) = language_config.highlight_config(scopes) {
-                let syntax = Syntax::new(&self.text, highlight_config);
+                let syntax = Syntax::new(&self.text, highlight_config, loader);
                 self.syntax = Some(syntax);
                 // TODO: config.configure(scopes) is now delayed, is that ok?
             }
@@ -605,7 +606,7 @@ pub fn set_language2(
     ) {
         let language_config = config_loader.language_config_for_scope(scope);
 
-        self.set_language(theme, language_config);
+        self.set_language(theme, language_config, Some(config_loader));
     }
 
     /// Set the LSP.

helix-view/src/editor.rs

@@ -283,7 +283,7 @@ pub fn set_theme(&mut self, theme: Theme) {
     /// Refreshes the language server for a given document
     pub fn refresh_language_server(&mut self, doc_id: DocumentId) -> Option<()> {
         let doc = self.documents.get_mut(&doc_id)?;
-        doc.detect_language(Some(&self.theme), &self.syn_loader);
+        doc.detect_language(Some(&self.theme), self.syn_loader.clone());
         Self::launch_language_server(&mut self.language_servers, doc)
     }
 
@@ -462,7 +462,12 @@ pub fn open(&mut self, path: PathBuf, action: Action) -> Result<DocumentId, Erro
         let id = if let Some(id) = id {
             id
         } else {
-            let mut doc = Document::open(&path, None, Some(&self.theme), Some(&self.syn_loader))?;
+            let mut doc = Document::open(
+                &path,
+                None,
+                Some(&self.theme),
+                Some(self.syn_loader.clone()),
+            )?;
 
             let _ = Self::launch_language_server(&mut self.language_servers, &mut doc);