From 3cb95be452491a72d18c98ebc619b0d2abb1b746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20Hrastnik?= Date: Mon, 6 Sep 2021 13:18:16 +0900 Subject: [PATCH] Update tree-sitter to 0.20 0.20 includes querying improvements, we no longer have to convert fragments to strings but can return an iterator of chunks instead. --- Cargo.lock | 4 +- helix-core/Cargo.toml | 2 +- helix-core/src/syntax.rs | 79 +++++++++++++++++++++------------------- helix-syntax/Cargo.toml | 2 +- 4 files changed, 45 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 298ad15b8..1b2492b84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1034,9 +1034,9 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.19.5" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad726ec26496bf4c083fff0f43d4eb3a2ad1bba305323af5ff91383c0b6ecac0" +checksum = "63ec02a07a782abef91279b72fe8fd2bee4c168a22112cedec5d3b0d49b9e4f9" dependencies = [ "cc", "regex", diff --git a/helix-core/Cargo.toml b/helix-core/Cargo.toml index 8c83816cf..5d2db642a 100644 --- a/helix-core/Cargo.toml +++ b/helix-core/Cargo.toml @@ -22,7 +22,7 @@ unicode-segmentation = "1.8" unicode-width = "0.1" unicode-general-category = "0.4" # slab = "0.4.2" -tree-sitter = "0.19" +tree-sitter = "0.20" once_cell = "1.8" arc-swap = "1" regex = "1" diff --git a/helix-core/src/syntax.rs b/helix-core/src/syntax.rs index 64b921e65..a7a5d022d 100644 --- a/helix-core/src/syntax.rs +++ b/helix-core/src/syntax.rs @@ -276,16 +276,6 @@ fn byte_range_to_str(range: std::ops::Range, source: RopeSlice) -> Cow(node: Node, source: RopeSlice<'a>) -> Cow<'a, [u8]> { - let start_char = source.byte_to_char(node.start_byte()); - let end_char = source.byte_to_char(node.end_byte()); - let fragment = source.slice(start_char..end_char); - match fragment.as_str() { - Some(fragment) => Cow::Borrowed(fragment.as_bytes()), - None => Cow::Owned(String::from(fragment).into_bytes()), - } -} - impl Syntax { // buffer, grammar, config, grammars, sync_timeout? pub fn new( @@ -380,14 +370,11 @@ pub fn highlight_iter<'a>( // TODO: if reusing cursors this might need resetting if let Some(range) = &range { - cursor_ref.set_byte_range(range.start, range.end); + cursor_ref.set_byte_range(range.clone()); } let captures = cursor_ref - .captures(query_ref, tree_ref.root_node(), move |n: Node| { - // &source[n.byte_range()] - node_to_bytes(n, source) - }) + .captures(query_ref, tree_ref.root_node(), RopeProvider(source)) .peekable(); // manually craft the root layer based on the existing tree @@ -501,10 +488,7 @@ fn parse( // let mut injections_by_pattern_index = // vec![(None, Vec::new(), false); combined_injections_query.pattern_count()]; // let matches = - // cursor.matches(combined_injections_query, tree.root_node(), |n: Node| { - // // &source[n.byte_range()] - // node_to_bytes(n, source) - // }); + // cursor.matches(combined_injections_query, tree.root_node(), RopeProvider(source)); // for mat in matches { // let entry = &mut injections_by_pattern_index[mat.pattern_index]; // let (language_name, content_node, include_children) = @@ -716,7 +700,7 @@ fn update( use std::{iter, mem, ops, str, usize}; use tree_sitter::{ Language as Grammar, Node, Parser, Point, Query, QueryCaptures, QueryCursor, QueryError, - QueryMatch, Range, Tree, + QueryMatch, Range, TextProvider, Tree, }; const CANCELLATION_CHECK_INTERVAL: usize = 100; @@ -776,7 +760,7 @@ struct LocalScope<'a> { } #[derive(Debug)] -struct HighlightIter<'a, 'tree: 'a, F> +struct HighlightIter<'a, F> where F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a, { @@ -784,16 +768,41 @@ struct HighlightIter<'a, 'tree: 'a, F> byte_offset: usize, injection_callback: F, cancellation_flag: Option<&'a AtomicUsize>, - layers: Vec>, + layers: Vec>, iter_count: usize, next_event: Option, last_highlight_range: Option<(usize, usize, usize)>, } -struct HighlightIterLayer<'a, 'tree: 'a> { +// Adapter to convert rope chunks to bytes +struct ChunksBytes<'a> { + chunks: ropey::iter::Chunks<'a>, +} +impl<'a> Iterator for ChunksBytes<'a> { + type Item = &'a [u8]; + fn next(&mut self) -> Option { + self.chunks.next().map(str::as_bytes) + } +} + +struct RopeProvider<'a>(RopeSlice<'a>); +impl<'a> TextProvider<'a> for RopeProvider<'a> { + type I = ChunksBytes<'a>; + + fn text(&mut self, node: Node) -> Self::I { + let start_char = self.0.byte_to_char(node.start_byte()); + let end_char = self.0.byte_to_char(node.end_byte()); + let fragment = self.0.slice(start_char..end_char); + ChunksBytes { + chunks: fragment.chunks(), + } + } +} + +struct HighlightIterLayer<'a> { _tree: Option, cursor: QueryCursor, - captures: iter::Peekable>>, + captures: iter::Peekable>>, config: &'a HighlightConfiguration, highlight_end_stack: Vec, scope_stack: Vec>, @@ -801,7 +810,7 @@ struct HighlightIterLayer<'a, 'tree: 'a> { depth: usize, } -impl<'a, 'tree: 'a> fmt::Debug for HighlightIterLayer<'a, 'tree> { +impl<'a> fmt::Debug for HighlightIterLayer<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("HighlightIterLayer").finish() } @@ -972,7 +981,7 @@ pub fn configure(&self, recognized_names: &[String]) { } } -impl<'a, 'tree: 'a> HighlightIterLayer<'a, 'tree> { +impl<'a> HighlightIterLayer<'a> { /// Create a new 'layer' of highlighting for this document. /// /// In the even that the new layer contains "combined injections" (injections where multiple @@ -1029,10 +1038,7 @@ fn new Option<&'a HighlightConfiguration> + 'a>( let matches = cursor.matches( combined_injections_query, tree.root_node(), - |n: Node| { - // &source[n.byte_range()] - node_to_bytes(n, source) - }, + RopeProvider(source), ); for mat in matches { let entry = &mut injections_by_pattern_index[mat.pattern_index]; @@ -1079,10 +1085,7 @@ fn new Option<&'a HighlightConfiguration> + 'a>( let cursor_ref = unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) }; let captures = cursor_ref - .captures(&config.query, tree_ref.root_node(), move |n: Node| { - // &source[n.byte_range()] - node_to_bytes(n, source) - }) + .captures(&config.query, tree_ref.root_node(), RopeProvider(source)) .peekable(); result.push(HighlightIterLayer { @@ -1236,7 +1239,7 @@ fn sort_key(&mut self) -> Option<(usize, bool, isize)> { } } -impl<'a, 'tree: 'a, F> HighlightIter<'a, 'tree, F> +impl<'a, F> HighlightIter<'a, F> where F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a, { @@ -1287,7 +1290,7 @@ fn sort_layers(&mut self) { } } - fn insert_layer(&mut self, mut layer: HighlightIterLayer<'a, 'tree>) { + fn insert_layer(&mut self, mut layer: HighlightIterLayer<'a>) { if let Some(sort_key) = layer.sort_key() { let mut i = 1; while i < self.layers.len() { @@ -1306,7 +1309,7 @@ fn insert_layer(&mut self, mut layer: HighlightIterLayer<'a, 'tree>) { } } -impl<'a, 'tree: 'a, F> Iterator for HighlightIter<'a, 'tree, F> +impl<'a, F> Iterator for HighlightIter<'a, F> where F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a, { @@ -1570,7 +1573,7 @@ fn next(&mut self) -> Option { fn injection_for_match<'a>( config: &HighlightConfiguration, query: &'a Query, - query_match: &QueryMatch<'a>, + query_match: &QueryMatch<'a, 'a>, source: RopeSlice<'a>, ) -> (Option>, Option>, bool) { let content_capture_index = config.injection_content_capture_index; diff --git a/helix-syntax/Cargo.toml b/helix-syntax/Cargo.toml index 73eda4720..9c2b82759 100644 --- a/helix-syntax/Cargo.toml +++ b/helix-syntax/Cargo.toml @@ -11,7 +11,7 @@ homepage = "https://helix-editor.com" include = ["src/**/*", "languages/**/*", "build.rs", "!**/docs/**/*", "!**/test/**/*", "!**/examples/**/*", "!**/build/**/*"] [dependencies] -tree-sitter = "0.19" +tree-sitter = "0.20" libloading = "0.7" anyhow = "1"