andrey/helix-mirror
1
0
Fork 0
mirror of https://github.com/helix-editor/helix.git synced 2024-11-22 01:16:18 +04:00
Code Issues Projects Releases Wiki Activity

tmp

Browse Source
This commit is contained in:
Pascal Kuthe 2024-07-27 15:43:44 +02:00
parent f7686d7af2
commit ae0d4189e1
No known key found for this signature in database
GPG Key ID: D715E8655AE166A6
15 changed files with 1799 additions and 1500 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
helix-core/src/syntax.rs
View File

@ -246,7 +246,7 @@ fn load_query(&self, kind: &str) -> Option<Query> {
if query_text.is_empty() { if query_text.is_empty() {
return None; return None;
} }
let lang = &self.highlight_config.get()?.as_ref()?.language; let lang = &self.highlight_config.get()?.as_ref()?.grammar;
Query::new(lang, &query_text) Query::new(lang, &query_text)
.map_err(|e| { .map_err(|e| {
log::error!( log::error!(

253
helix-syntax/src/config.rs
View File

@ -1,36 +1,38 @@
use std::borrow::Cow;
use std::path::Path; use std::path::Path;
use std::sync::Arc; use std::sync::Arc;
use crate::tree_sitter::query::{Capture, Pattern, QueryStr, UserPredicate};
use crate::tree_sitter::{query, Grammar, Query, QueryMatch, SyntaxTreeNode};
use arc_swap::ArcSwap; use arc_swap::ArcSwap;
use helix_stdx::rope::{self, RopeSliceExt}; use helix_stdx::rope::{self, RopeSliceExt};
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use regex::Regex; use regex::Regex;
use ropey::RopeSlice; use ropey::RopeSlice;
use tree_sitter::{Language as Grammar, Node, Query, QueryError, QueryMatch};
use crate::byte_range_to_str;
use crate::highlighter::Highlight; use crate::highlighter::Highlight;
use crate::{byte_range_to_str, IncludedChildren, InjectionLanguageMarker, SHEBANG};
/// Contains the data needed to highlight code written in a particular language. /// Contains the data needed to highlight code written in a particular language.
/// ///
/// This struct is immutable and can be shared between threads. /// This struct is immutable and can be shared between threads.
#[derive(Debug)] #[derive(Debug)]
pub struct HighlightConfiguration { pub struct HighlightConfiguration {
pub language: Grammar, pub grammar: Grammar,
pub query: Query, pub query: Query,
pub(crate) injections_query: Query, pub(crate) injections_query: Query,
pub(crate) combined_injections_patterns: Vec<usize>, pub(crate) combined_injections_patterns: Vec<Pattern>,
pub(crate) highlights_pattern_index: usize, first_highlights_pattern: Pattern,
pub(crate) highlight_indices: ArcSwap<Vec<Option<Highlight>>>, pub(crate) highlight_indices: ArcSwap<Vec<Highlight>>,
pub(crate) non_local_variable_patterns: Vec<bool>, pub(crate) non_local_variable_patterns: Vec<bool>,
pub(crate) injection_content_capture_index: Option<u32>, pub(crate) injection_content_capture: Option<Capture>,
pub(crate) injection_language_capture_index: Option<u32>, pub(crate) injection_language_capture: Option<Capture>,
pub(crate) injection_filename_capture_index: Option<u32>, pub(crate) injection_filename_capture: Option<Capture>,
pub(crate) injection_shebang_capture_index: Option<u32>, pub(crate) injection_shebang_capture: Option<Capture>,
pub(crate) local_scope_capture_index: Option<u32>, pub(crate) local_scope_capture: Option<Capture>,
pub(crate) local_def_capture_index: Option<u32>, pub(crate) local_def_capture: Option<Capture>,
pub(crate) local_def_value_capture_index: Option<u32>, pub(crate) local_def_value_capture: Option<Capture>,
pub(crate) local_ref_capture_index: Option<u32>, pub(crate) local_ref_capture: Option<Capture>,
} }
impl HighlightConfiguration { impl HighlightConfiguration {
@ -49,105 +51,89 @@ impl HighlightConfiguration {
/// ///
/// Returns a `HighlightConfiguration` that can then be used with the `highlight` method. /// Returns a `HighlightConfiguration` that can then be used with the `highlight` method.
pub fn new( pub fn new(
language: Grammar, grammar: Grammar,
path: impl AsRef<Path>,
highlights_query: &str, highlights_query: &str,
injection_query: &str, injection_query: &str,
locals_query: &str, locals_query: &str,
) -> Result<Self, QueryError> { ) -> Result<Self, query::ParseError> {
// Concatenate the query strings, keeping track of the start offset of each section. // Concatenate the query strings, keeping track of the start offset of each section.
let mut query_source = String::new(); let mut query_source = String::new();
query_source.push_str(locals_query); query_source.push_str(locals_query);
let highlights_query_offset = query_source.len(); let highlights_query_offset = query_source.len();
query_source.push_str(highlights_query); query_source.push_str(highlights_query);
let mut non_local_variable_patterns = Vec::with_capacity(32);
// Construct a single query by concatenating the three query strings, but record the // Construct a single query by concatenating the three query strings, but record the
// range of pattern indices that belong to each individual string. // range of pattern indices that belong to each individual string.
let query = Query::new(&language, &query_source)?; let query = Query::new(grammar, &query_source, path, |pattern, predicate| {
let mut highlights_pattern_index = 0; match predicate {
for i in 0..(query.pattern_count()) { UserPredicate::IsPropertySet {
let pattern_offset = query.start_byte_for_pattern(i); negate: true,
if pattern_offset < highlights_query_offset { key: "local",
highlights_pattern_index += 1; val: None,
} => {
if non_local_variable_patterns.len() < pattern.idx() {
non_local_variable_patterns.resize(pattern.idx(), false)
}
non_local_variable_patterns[pattern.idx()] = true;
}
predicate => {
return Err(format!("unsupported predicate {predicate}").into());
} }
} }
Ok(())
})?;
let injections_query = Query::new(&language, injection_query)?; let mut combined_injections_patterns = Vec::new();
let combined_injections_patterns = (0..injections_query.pattern_count()) let injections_query = Query::new(grammar, injection_query, path, |pattern, predicate| {
.filter(|&i| { match predicate {
injections_query UserPredicate::SetProperty {
.property_settings(i) key: "injection.combined",
.iter() val: None,
.any(|s| &*s.key == "injection.combined") } => combined_injections_patterns.push(pattern),
}) predicate => {
.collect(); return Err(format!("unsupported predicate {predicate}").into());
// Find all of the highlighting patterns that are disabled for nodes that
// have been identified as local variables.
let non_local_variable_patterns = (0..query.pattern_count())
.map(|i| {
query
.property_predicates(i)
.iter()
.any(|(prop, positive)| !*positive && prop.key.as_ref() == "local")
})
.collect();
// Store the numeric ids for all of the special captures.
let mut injection_content_capture_index = None;
let mut injection_language_capture_index = None;
let mut injection_filename_capture_index = None;
let mut injection_shebang_capture_index = None;
let mut local_def_capture_index = None;
let mut local_def_value_capture_index = None;
let mut local_ref_capture_index = None;
let mut local_scope_capture_index = None;
for (i, name) in query.capture_names().iter().enumerate() {
let i = Some(i as u32);
match *name {
"local.definition" => local_def_capture_index = i,
"local.definition-value" => local_def_value_capture_index = i,
"local.reference" => local_ref_capture_index = i,
"local.scope" => local_scope_capture_index = i,
_ => {}
} }
} }
Ok(())
})?;
for (i, name) in injections_query.capture_names().iter().enumerate() { let first_highlights_pattern = query
let i = Some(i as u32); .patterns()
match *name { .find(|pattern| query.start_byte_for_pattern(*pattern) >= highlights_query_offset)
"injection.content" => injection_content_capture_index = i, .unwrap_or(Pattern::SENTINEL);
"injection.language" => injection_language_capture_index = i,
"injection.filename" => injection_filename_capture_index = i,
"injection.shebang" => injection_shebang_capture_index = i,
_ => {}
}
}
let highlight_indices = ArcSwap::from_pointee(vec![None; query.capture_names().len()]); let injection_content_capture = query.get_capture("injection.content");
let injection_language_capture = query.get_capture("injection.language");
let injection_filename_capture = query.get_capture("injection.filename");
let injection_shebang_capture = query.get_capture("injection.shebang");
let local_def_capture = query.get_capture("local.definition");
let local_def_value_capture = query.get_capture("local.definition-value");
let local_ref_capture = query.get_capture("local.reference");
let local_scope_capture = query.get_capture("local.scope");
let highlight_indices =
ArcSwap::from_pointee(vec![Highlight::NONE; query.num_captures() as usize]);
Ok(Self { Ok(Self {
language, grammar,
query, query,
injections_query, injections_query,
combined_injections_patterns, combined_injections_patterns,
highlights_pattern_index, first_highlights_pattern,
highlight_indices, highlight_indices,
non_local_variable_patterns, non_local_variable_patterns,
injection_content_capture_index, injection_content_capture,
injection_language_capture_index, injection_language_capture,
injection_filename_capture_index, injection_filename_capture,
injection_shebang_capture_index, injection_shebang_capture,
local_scope_capture_index, local_scope_capture,
local_def_capture_index, local_def_capture,
local_def_value_capture_index, local_def_value_capture,
local_ref_capture_index, local_ref_capture,
}) })
} }
/// Get a slice containing all of the highlight names used in the configuration.
pub fn names(&self) -> &[&str] {
self.query.capture_names()
}
/// Set the list of recognized highlight names. /// Set the list of recognized highlight names.
/// ///
/// Tree-sitter syntax-highlighting queries specify highlights in the form of dot-separated /// Tree-sitter syntax-highlighting queries specify highlights in the form of dot-separated
@ -162,13 +148,12 @@ pub fn configure(&self, recognized_names: &[String]) {
let mut capture_parts = Vec::new(); let mut capture_parts = Vec::new();
let indices: Vec<_> = self let indices: Vec<_> = self
.query .query
.capture_names() .captures()
.iter() .map(move |(_, capture_name)| {
.map(move |capture_name| {
capture_parts.clear(); capture_parts.clear();
capture_parts.extend(capture_name.split('.')); capture_parts.extend(capture_name.split('.'));
let mut best_index = None; let mut best_index = u32::MAX;
let mut best_match_len = 0; let mut best_match_len = 0;
for (i, recognized_name) in recognized_names.iter().enumerate() { for (i, recognized_name) in recognized_names.iter().enumerate() {
let mut len = 0; let mut len = 0;
@ -183,11 +168,11 @@ pub fn configure(&self, recognized_names: &[String]) {
} }
} }
if matches && len > best_match_len { if matches && len > best_match_len {
best_index = Some(i); best_index = i as u32;
best_match_len = len; best_match_len = len;
} }
} }
best_index.map(Highlight) Highlight(best_index)
}) })
.collect(); .collect();
@ -198,21 +183,24 @@ fn injection_pair<'a>(
&self, &self,
query_match: &QueryMatch<'a, 'a>, query_match: &QueryMatch<'a, 'a>,
source: RopeSlice<'a>, source: RopeSlice<'a>,
) -> (Option<InjectionLanguageMarker<'a>>, Option<Node<'a>>) { ) -> (
Option<InjectionLanguageMarker<'a>>,
Option<SyntaxTreeNode<'a>>,
) {
let mut injection_capture = None; let mut injection_capture = None;
let mut content_node = None; let mut content_node = None;
for capture in query_match.captures { for matched_node in query_match.matched_nodes() {
let index = Some(capture.index); let capture = Some(matched_node.capture);
if index == self.injection_language_capture_index { if capture == self.injection_language_capture {
let name = byte_range_to_str(capture.node.byte_range(), source); let name = byte_range_to_str(matched_node.syntax_node.byte_range(), source);
injection_capture = Some(InjectionLanguageMarker::Name(name)); injection_capture = Some(InjectionLanguageMarker::Name(name));
} else if index == self.injection_filename_capture_index { } else if capture == self.injection_filename_capture {
let name = byte_range_to_str(capture.node.byte_range(), source); let name = byte_range_to_str(matched_node.syntax_node.byte_range(), source);
let path = Path::new(name.as_ref()).to_path_buf(); let path = Path::new(name.as_ref()).to_path_buf();
injection_capture = Some(InjectionLanguageMarker::Filename(path.into())); injection_capture = Some(InjectionLanguageMarker::Filename(path.into()));
} else if index == self.injection_shebang_capture_index { } else if capture == self.injection_shebang_capture {
let node_slice = source.byte_slice(capture.node.byte_range()); let node_slice = source.byte_slice(matched_node.syntax_node.byte_range());
// some languages allow space and newlines before the actual string content // some languages allow space and newlines before the actual string content
// so a shebang could be on either the first or second line // so a shebang could be on either the first or second line
@ -222,9 +210,6 @@ fn injection_pair<'a>(
node_slice node_slice
}; };
static SHEBANG_REGEX: Lazy<rope::Regex> =
Lazy::new(|| rope::Regex::new(SHEBANG).unwrap());
injection_capture = SHEBANG_REGEX injection_capture = SHEBANG_REGEX
.captures_iter(lines.regex_input()) .captures_iter(lines.regex_input())
.map(|cap| { .map(|cap| {
@ -232,8 +217,8 @@ fn injection_pair<'a>(
InjectionLanguageMarker::Shebang(cap.into()) InjectionLanguageMarker::Shebang(cap.into())
}) })
.next() .next()
} else if index == self.injection_content_capture_index { } else if capture == self.injection_content_capture {
content_node = Some(capture.node); content_node = Some(matched_node.syntax_node.clone());
} }
} }
(injection_capture, content_node) (injection_capture, content_node)
@ -246,7 +231,7 @@ pub(super) fn injection_for_match<'a>(
source: RopeSlice<'a>, source: RopeSlice<'a>,
) -> ( ) -> (
Option<InjectionLanguageMarker<'a>>, Option<InjectionLanguageMarker<'a>>,
Option<Node<'a>>, Option<SyntaxTreeNode<'a>>,
IncludedChildren, IncludedChildren,
) { ) {
let (mut injection_capture, content_node) = self.injection_pair(query_match, source); let (mut injection_capture, content_node) = self.injection_pair(query_match, source);
@ -282,18 +267,20 @@ pub(super) fn injection_for_match<'a>(
(injection_capture, content_node, included_children) (injection_capture, content_node, included_children)
} }
pub fn load_query(
&self, // pub fn load_query(
language: &str, // &self,
filename: &str, // language: &str,
read_query_text: impl FnMut(&str, &str) -> String, // filename: &str,
) -> Result<Option<Query>, QueryError> { // read_query_text: impl FnMut(&str, &str) -> String,
let query_text = read_query(language, filename, read_query_text); // ) -> Result<Option<Query>, QueryError> {
if query_text.is_empty() { // let query_text = read_query(language, filename, read_query_text);
return Ok(None); // if query_text.is_empty() {
} // return Ok(None);
Query::new(&self.language, &query_text).map(Some) // }
}
// Query::new(&self.grammar, &query_text, ).map(Some)
// }
} }
/// reads a query by invoking `read_query_text`, handeles any `inherits` directives /// reads a query by invoking `read_query_text`, handeles any `inherits` directives
@ -329,3 +316,31 @@ fn read_query_impl(
} }
read_query_impl(language, filename, &mut read_query_text) read_query_impl(language, filename, &mut read_query_text)
} }
const SHEBANG: &str = r"#!\s*(?:\S*[/\\](?:env\s+(?:\-\S+\s+)*)?)?([^\s\.\d]+)";
static SHEBANG_REGEX: Lazy<rope::Regex> = Lazy::new(|| rope::Regex::new(SHEBANG).unwrap());
struct InjectionSettings {
include_children: IncludedChildren,
language: Option<QueryStr>,
}
#[derive(Debug, Clone)]
pub enum InjectionLanguageMarker<'a> {
Name(Cow<'a, str>),
Filename(Cow<'a, Path>),
Shebang(String),
}
#[derive(Clone)]
enum IncludedChildren {
None,
All,
Unnamed,
}
impl Default for IncludedChildren {
fn default() -> Self {
Self::None
}
}

791
helix-syntax/src/highlighter.rs
View File

@ -1,439 +1,438 @@
use std::borrow::Cow; pub use super::highlighter2::*;
use std::cell::RefCell;
use std::sync::atomic::{self, AtomicUsize};
use std::{fmt, iter, mem, ops};
use ropey::RopeSlice; // use std::borrow::Cow;
use tree_sitter::{QueryCaptures, QueryCursor, Tree}; // use std::cell::RefCell;
// use std::sync::atomic::{self, AtomicUsize};
// use std::{fmt, iter, mem, ops};
use crate::ropey::RopeProvider; // use ropey::RopeSlice;
use crate::{ // use tree_sitter::{QueryCaptures, QueryCursor, Tree};
byte_range_to_str, Error, HighlightConfiguration, Syntax, PARSER, TREE_SITTER_MATCH_LIMIT,
};
const CANCELLATION_CHECK_INTERVAL: usize = 100; // use crate::{byte_range_to_str, Error, HighlightConfiguration, Syntax, TREE_SITTER_MATCH_LIMIT};
/// Indicates which highlight should be applied to a region of source code. // const CANCELLATION_CHECK_INTERVAL: usize = 100;
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub struct Highlight(pub usize);
/// Represents a single step in rendering a syntax-highlighted document. // /// Indicates which highlight should be applied to a region of source code.
#[derive(Copy, Clone, Debug)] // #[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum HighlightEvent { // pub struct Highlight(pub usize);
Source { start: usize, end: usize },
HighlightStart(Highlight),
HighlightEnd,
}
#[derive(Debug)] // /// Represents a single step in rendering a syntax-highlighted document.
struct LocalDef<'a> { // #[derive(Copy, Clone, Debug)]
name: Cow<'a, str>, // pub enum HighlightEvent {
value_range: ops::Range<usize>, // Source { start: usize, end: usize },
highlight: Option<Highlight>, // HighlightStart(Highlight),
} // HighlightEnd,
// }
#[derive(Debug)] // #[derive(Debug)]
struct LocalScope<'a> { // struct LocalDef<'a> {
inherits: bool, // name: Cow<'a, str>,
range: ops::Range<usize>, // value_range: ops::Range<usize>,
local_defs: Vec<LocalDef<'a>>, // highlight: Option<Highlight>,
} // }
#[derive(Debug)] // #[derive(Debug)]
struct HighlightIter<'a> { // struct LocalScope<'a> {
source: RopeSlice<'a>, // inherits: bool,
byte_offset: usize, // range: ops::Range<usize>,
cancellation_flag: Option<&'a AtomicUsize>, // local_defs: Vec<LocalDef<'a>>,
layers: Vec<HighlightIterLayer<'a>>, // }
iter_count: usize,
next_event: Option<HighlightEvent>,
last_highlight_range: Option<(usize, usize, u32)>,
}
struct HighlightIterLayer<'a> { // #[derive(Debug)]
_tree: Option<Tree>, // struct HighlightIter<'a> {
cursor: QueryCursor, // source: RopeSlice<'a>,
captures: RefCell<iter::Peekable<QueryCaptures<'a, 'a, RopeProvider<'a>, &'a [u8]>>>, // byte_offset: usize,
config: &'a HighlightConfiguration, // cancellation_flag: Option<&'a AtomicUsize>,
highlight_end_stack: Vec<usize>, // layers: Vec<HighlightIterLayer<'a>>,
scope_stack: Vec<LocalScope<'a>>, // iter_count: usize,
depth: u32, // next_event: Option<HighlightEvent>,
} // last_highlight_range: Option<(usize, usize, u32)>,
// }
impl<'a> fmt::Debug for HighlightIterLayer<'a> { // struct HighlightIterLayer<'a> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // _tree: Option<Tree>,
f.debug_struct("HighlightIterLayer").finish() // cursor: QueryCursor,
} // captures: RefCell<iter::Peekable<QueryCaptures<'a, 'a, RopeProvider<'a>, &'a [u8]>>>,
} // config: &'a HighlightConfiguration,
// highlight_end_stack: Vec<usize>,
// scope_stack: Vec<LocalScope<'a>>,
// depth: u32,
// }
impl<'a> HighlightIterLayer<'a> { // impl<'a> fmt::Debug for HighlightIterLayer<'a> {
// First, sort scope boundaries by their byte offset in the document. At a // fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// given position, emit scope endings before scope beginnings. Finally, emit // f.debug_struct("HighlightIterLayer").finish()
// scope boundaries from deeper layers first. // }
fn sort_key(&self) -> Option<(usize, bool, isize)> { // }
let depth = -(self.depth as isize);
let next_start = self
.captures
.borrow_mut()
.peek()
.map(|(m, i)| m.captures[*i].node.start_byte());
let next_end = self.highlight_end_stack.last().cloned();
match (next_start, next_end) {
(Some(start), Some(end)) => {
if start < end {
Some((start, true, depth))
} else {
Some((end, false, depth))
}
}
(Some(i), None) => Some((i, true, depth)),
(None, Some(j)) => Some((j, false, depth)),
_ => None,
}
}
}
impl<'a> HighlightIter<'a> { // impl<'a> HighlightIterLayer<'a> {
fn emit_event( // // First, sort scope boundaries by their byte offset in the document. At a
&mut self, // // given position, emit scope endings before scope beginnings. Finally, emit
offset: usize, // // scope boundaries from deeper layers first.
event: Option<HighlightEvent>, // fn sort_key(&self) -> Option<(usize, bool, isize)> {
) -> Option<Result<HighlightEvent, Error>> { // let depth = -(self.depth as isize);
let result; // let next_start = self
if self.byte_offset < offset { // .captures
result = Some(Ok(HighlightEvent::Source { // .borrow_mut()
start: self.byte_offset, // .peek()
end: offset, // .map(|(m, i)| m.captures[*i].node.start_byte());
})); // let next_end = self.highlight_end_stack.last().cloned();
self.byte_offset = offset; // match (next_start, next_end) {
self.next_event = event; // (Some(start), Some(end)) => {
} else { // if start < end {
result = event.map(Ok); // Some((start, true, depth))
} // } else {
self.sort_layers(); // Some((end, false, depth))
result // }
} // }
// (Some(i), None) => Some((i, true, depth)),
// (None, Some(j)) => Some((j, false, depth)),
// _ => None,
// }
// }
// }
fn sort_layers(&mut self) { // impl<'a> HighlightIter<'a> {
while !self.layers.is_empty() { // fn emit_event(
if let Some(sort_key) = self.layers[0].sort_key() { // &mut self,
let mut i = 0; // offset: usize,
while i + 1 < self.layers.len() { // event: Option<HighlightEvent>,
if let Some(next_offset) = self.layers[i + 1].sort_key() { // ) -> Option<Result<HighlightEvent, Error>> {
if next_offset < sort_key { // let result;
i += 1; // if self.byte_offset < offset {
continue; // result = Some(Ok(HighlightEvent::Source {
} // start: self.byte_offset,
} else { // end: offset,
let layer = self.layers.remove(i + 1); // }));
PARSER.with(|ts_parser| { // self.byte_offset = offset;
let highlighter = &mut ts_parser.borrow_mut(); // self.next_event = event;
highlighter.cursors.push(layer.cursor); // } else {
}); // result = event.map(Ok);
} // }
break; // self.sort_layers();
} // result
if i > 0 { // }
self.layers[0..(i + 1)].rotate_left(1);
}
break;
} else {
let layer = self.layers.remove(0);
PARSER.with(|ts_parser| {
let highlighter = &mut ts_parser.borrow_mut();
highlighter.cursors.push(layer.cursor);
});
}
}
}
}
impl<'a> Iterator for HighlightIter<'a> { // fn sort_layers(&mut self) {
type Item = Result<HighlightEvent, Error>; // while !self.layers.is_empty() {
// if let Some(sort_key) = self.layers[0].sort_key() {
// let mut i = 0;
// while i + 1 < self.layers.len() {
// if let Some(next_offset) = self.layers[i + 1].sort_key() {
// if next_offset < sort_key {
// i += 1;
// continue;
// }
// } else {
// let layer = self.layers.remove(i + 1);
// PARSER.with(|ts_parser| {
// let highlighter = &mut ts_parser.borrow_mut();
// highlighter.cursors.push(layer.cursor);
// });
// }
// break;
// }
// if i > 0 {
// self.layers[0..(i + 1)].rotate_left(1);
// }
// break;
// } else {
// let layer = self.layers.remove(0);
// PARSER.with(|ts_parser| {
// let highlighter = &mut ts_parser.borrow_mut();
// highlighter.cursors.push(layer.cursor);
// });
// }
// }
// }
// }
fn next(&mut self) -> Option<Self::Item> { // impl<'a> Iterator for HighlightIter<'a> {
'main: loop { // type Item = Result<HighlightEvent, Error>;
// If we've already determined the next highlight boundary, just return it.
if let Some(e) = self.next_event.take() {
return Some(Ok(e));
}
// Periodically check for cancellation, returning `Cancelled` error if the // fn next(&mut self) -> Option<Self::Item> {
// cancellation flag was flipped. // 'main: loop {
if let Some(cancellation_flag) = self.cancellation_flag { // // If we've already determined the next highlight boundary, just return it.
self.iter_count += 1; // if let Some(e) = self.next_event.take() {
if self.iter_count >= CANCELLATION_CHECK_INTERVAL { // return Some(Ok(e));
self.iter_count = 0; // }
if cancellation_flag.load(atomic::Ordering::Relaxed) != 0 {
return Some(Err(Error::Cancelled));
}
}
}
// If none of the layers have any more highlight boundaries, terminate. // // Periodically check for cancellation, returning `Cancelled` error if the
if self.layers.is_empty() { // // cancellation flag was flipped.
let len = self.source.len_bytes(); // if let Some(cancellation_flag) = self.cancellation_flag {
return if self.byte_offset < len { // self.iter_count += 1;
let result = Some(Ok(HighlightEvent::Source { // if self.iter_count >= CANCELLATION_CHECK_INTERVAL {
start: self.byte_offset, // self.iter_count = 0;
end: len, // if cancellation_flag.load(atomic::Ordering::Relaxed) != 0 {
})); // return Some(Err(Error::Cancelled));
self.byte_offset = len; // }
result // }
} else { // }
None
};
}
// Get the next capture from whichever layer has the earliest highlight boundary. // // If none of the layers have any more highlight boundaries, terminate.
let range; // if self.layers.is_empty() {
let layer = &mut self.layers[0]; // let len = self.source.len_bytes();
let captures = layer.captures.get_mut(); // return if self.byte_offset < len {
if let Some((next_match, capture_index)) = captures.peek() { // let result = Some(Ok(HighlightEvent::Source {
let next_capture = next_match.captures[*capture_index]; // start: self.byte_offset,
range = next_capture.node.byte_range(); // end: len,
// }));
// self.byte_offset = len;
// result
// } else {
// None
// };
// }
// If any previous highlight ends before this node starts, then before // // Get the next capture from whichever layer has the earliest highlight boundary.
// processing this capture, emit the source code up until the end of the // let range;
// previous highlight, and an end event for that highlight. // let layer = &mut self.layers[0];
if let Some(end_byte) = layer.highlight_end_stack.last().cloned() { // let captures = layer.captures.get_mut();
if end_byte <= range.start { // if let Some((next_match, capture_index)) = captures.peek() {
layer.highlight_end_stack.pop(); // let next_capture = next_match.captures[*capture_index];
return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd)); // range = next_capture.node.byte_range();
}
}
}
// If there are no more captures, then emit any remaining highlight end events.
// And if there are none of those, then just advance to the end of the document.
else if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {
layer.highlight_end_stack.pop();
return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));
} else {
return self.emit_event(self.source.len_bytes(), None);
};
let (mut match_, capture_index) = captures.next().unwrap(); // // If any previous highlight ends before this node starts, then before
let mut capture = match_.captures[capture_index]; // // processing this capture, emit the source code up until the end of the
// // previous highlight, and an end event for that highlight.
// if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {
// if end_byte <= range.start {
// layer.highlight_end_stack.pop();
// return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));
// }
// }
// }
// // If there are no more captures, then emit any remaining highlight end events.
// // And if there are none of those, then just advance to the end of the document.
// else if let Some(end_byte) = layer.highlight_end_stack.last().cloned() {
// layer.highlight_end_stack.pop();
// return self.emit_event(end_byte, Some(HighlightEvent::HighlightEnd));
// } else {
// return self.emit_event(self.source.len_bytes(), None);
// };
// Remove from the local scope stack any local scopes that have already ended. // let (mut match_, capture_index) = captures.next().unwrap();
while range.start > layer.scope_stack.last().unwrap().range.end { // let mut capture = match_.captures[capture_index];
layer.scope_stack.pop();
}
// If this capture is for tracking local variables, then process the // // Remove from the local scope stack any local scopes that have already ended.
// local variable info. // while range.start > layer.scope_stack.last().unwrap().range.end {
let mut reference_highlight = None; // layer.scope_stack.pop();
let mut definition_highlight = None; // }
while match_.pattern_index < layer.config.highlights_pattern_index {
// If the node represents a local scope, push a new local scope onto
// the scope stack.
if Some(capture.index) == layer.config.local_scope_capture_index {
definition_highlight = None;
let mut scope = LocalScope {
inherits: true,
range: range.clone(),
local_defs: Vec::new(),
};
for prop in layer.config.query.property_settings(match_.pattern_index) {
if let "local.scope-inherits" = prop.key.as_ref() {
scope.inherits =
prop.value.as_ref().map_or(true, |r| r.as_ref() == "true");
}
}
layer.scope_stack.push(scope);
}
// If the node represents a definition, add a new definition to the
// local scope at the top of the scope stack.
else if Some(capture.index) == layer.config.local_def_capture_index {
reference_highlight = None;
let scope = layer.scope_stack.last_mut().unwrap();
let mut value_range = 0..0; // // If this capture is for tracking local variables, then process the
for capture in match_.captures { // // local variable info.
if Some(capture.index) == layer.config.local_def_value_capture_index { // let mut reference_highlight = None;
value_range = capture.node.byte_range(); // let mut definition_highlight = None;
} // while match_.pattern_index < layer.config.highlights_pattern_index {
} // // If the node represents a local scope, push a new local scope onto
// // the scope stack.
// if Some(capture.index) == layer.config.local_scope_capture_index {
// definition_highlight = None;
// let mut scope = LocalScope {
// inherits: true,
// range: range.clone(),
// local_defs: Vec::new(),
// };
// for prop in layer.config.query.property_settings(match_.pattern_index) {
// if let "local.scope-inherits" = prop.key.as_ref() {
// scope.inherits =
// prop.value.as_ref().map_or(true, |r| r.as_ref() == "true");
// }
// }
// layer.scope_stack.push(scope);
// }
// // If the node represents a definition, add a new definition to the
// // local scope at the top of the scope stack.
// else if Some(capture.index) == layer.config.local_def_capture_index {
// reference_highlight = None;
// let scope = layer.scope_stack.last_mut().unwrap();
let name = byte_range_to_str(range.clone(), self.source); // let mut value_range = 0..0;
scope.local_defs.push(LocalDef { // for capture in match_.captures {
name, // if Some(capture.index) == layer.config.local_def_value_capture_index {
value_range, // value_range = capture.node.byte_range();
highlight: None, // }
}); // }
definition_highlight = scope.local_defs.last_mut().map(|s| &mut s.highlight);
}
// If the node represents a reference, then try to find the corresponding
// definition in the scope stack.
else if Some(capture.index) == layer.config.local_ref_capture_index
&& definition_highlight.is_none()
{
definition_highlight = None;
let name = byte_range_to_str(range.clone(), self.source);
for scope in layer.scope_stack.iter().rev() {
if let Some(highlight) = scope.local_defs.iter().rev().find_map(|def| {
if def.name == name && range.start >= def.value_range.end {
Some(def.highlight)
} else {
None
}
}) {
reference_highlight = highlight;
break;
}
if !scope.inherits {
break;
}
}
}
// Continue processing any additional matches for the same node. // let name = byte_range_to_str(range.clone(), self.source);
if let Some((next_match, next_capture_index)) = captures.peek() { // scope.local_defs.push(LocalDef {
let next_capture = next_match.captures[*next_capture_index]; // name,
if next_capture.node == capture.node { // value_range,
capture = next_capture; // highlight: None,
match_ = captures.next().unwrap().0; // });
continue; // definition_highlight = scope.local_defs.last_mut().map(|s| &mut s.highlight);
} // }
} // // If the node represents a reference, then try to find the corresponding
// // definition in the scope stack.
// else if Some(capture.index) == layer.config.local_ref_capture_index
// && definition_highlight.is_none()
// {
// definition_highlight = None;
// let name = byte_range_to_str(range.clone(), self.source);
// for scope in layer.scope_stack.iter().rev() {
// if let Some(highlight) = scope.local_defs.iter().rev().find_map(|def| {
// if def.name == name && range.start >= def.value_range.end {
// Some(def.highlight)
// } else {
// None
// }
// }) {
// reference_highlight = highlight;
// break;
// }
// if !scope.inherits {
// break;
// }
// }
// }
self.sort_layers(); // // Continue processing any additional matches for the same node.
continue 'main; // if let Some((next_match, next_capture_index)) = captures.peek() {
} // let next_capture = next_match.captures[*next_capture_index];
// if next_capture.node == capture.node {
// capture = next_capture;
// match_ = captures.next().unwrap().0;
// continue;
// }
// }
// Otherwise, this capture must represent a highlight. // self.sort_layers();
// If this exact range has already been highlighted by an earlier pattern, or by // continue 'main;
// a different layer, then skip over this one. // }
if let Some((last_start, last_end, last_depth)) = self.last_highlight_range {
if range.start == last_start && range.end == last_end && layer.depth < last_depth {
self.sort_layers();
continue 'main;
}
}
// If the current node was found to be a local variable, then skip over any // // Otherwise, this capture must represent a highlight.
// highlighting patterns that are disabled for local variables. // // If this exact range has already been highlighted by an earlier pattern, or by
if definition_highlight.is_some() || reference_highlight.is_some() { // // a different layer, then skip over this one.
while layer.config.non_local_variable_patterns[match_.pattern_index] { // if let Some((last_start, last_end, last_depth)) = self.last_highlight_range {
match_.remove(); // if range.start == last_start && range.end == last_end && layer.depth < last_depth {
if let Some((next_match, next_capture_index)) = captures.peek() { // self.sort_layers();
let next_capture = next_match.captures[*next_capture_index]; // continue 'main;
if next_capture.node == capture.node { // }
capture = next_capture; // }
match_ = captures.next().unwrap().0;
continue;
}
}
self.sort_layers(); // // If the current node was found to be a local variable, then skip over any
continue 'main; // // highlighting patterns that are disabled for local variables.
} // if definition_highlight.is_some() || reference_highlight.is_some() {
} // while layer.config.non_local_variable_patterns[match_.pattern_index] {
// match_.remove();
// if let Some((next_match, next_capture_index)) = captures.peek() {
// let next_capture = next_match.captures[*next_capture_index];
// if next_capture.node == capture.node {
// capture = next_capture;
// match_ = captures.next().unwrap().0;
// continue;
// }
// }
// Once a highlighting pattern is found for the current node, skip over // self.sort_layers();
// any later highlighting patterns that also match this node. Captures // continue 'main;
// for a given node are ordered by pattern index, so these subsequent // }
// captures are guaranteed to be for highlighting, not injections or // }
// local variables.
while let Some((next_match, next_capture_index)) = captures.peek() {
let next_capture = next_match.captures[*next_capture_index];
if next_capture.node == capture.node {
captures.next();
} else {
break;
}
}
let current_highlight = layer.config.highlight_indices.load()[capture.index as usize]; // // Once a highlighting pattern is found for the current node, skip over
// // any later highlighting patterns that also match this node. Captures
// // for a given node are ordered by pattern index, so these subsequent
// // captures are guaranteed to be for highlighting, not injections or
// // local variables.
// while let Some((next_match, next_capture_index)) = captures.peek() {
// let next_capture = next_match.captures[*next_capture_index];
// if next_capture.node == capture.node {
// captures.next();
// } else {
// break;
// }
// }
// If this node represents a local definition, then store the current // let current_highlight = layer.config.highlight_indices.load()[capture.index as usize];
// highlight value on the local scope entry representing this node.
if let Some(definition_highlight) = definition_highlight {
*definition_highlight = current_highlight;
}
// Emit a scope start event and push the node's end position to the stack. // // If this node represents a local definition, then store the current
if let Some(highlight) = reference_highlight.or(current_highlight) { // // highlight value on the local scope entry representing this node.
self.last_highlight_range = Some((range.start, range.end, layer.depth)); // if let Some(definition_highlight) = definition_highlight {
layer.highlight_end_stack.push(range.end); // *definition_highlight = current_highlight;
return self // }
.emit_event(range.start, Some(HighlightEvent::HighlightStart(highlight)));
}
self.sort_layers(); // // Emit a scope start event and push the node's end position to the stack.
} // if let Some(highlight) = reference_highlight.or(current_highlight) {
} // self.last_highlight_range = Some((range.start, range.end, layer.depth));
} // layer.highlight_end_stack.push(range.end);
// return self
// .emit_event(range.start, Some(HighlightEvent::HighlightStart(highlight)));
// }
impl Syntax { // self.sort_layers();
/// Iterate over the highlighted regions for a given slice of source code. // }
pub fn highlight_iter<'a>( // }
&'a self, // }
source: RopeSlice<'a>,
range: Option<std::ops::Range<usize>>,
cancellation_flag: Option<&'a AtomicUsize>,
) -> impl Iterator<Item = Result<HighlightEvent, Error>> + 'a {
let mut layers = self
.layers
.iter()
.filter_map(|(_, layer)| {
// TODO: if range doesn't overlap layer range, skip it
// Reuse a cursor from the pool if available. // impl Syntax {
let mut cursor = PARSER.with(|ts_parser| { // /// Iterate over the highlighted regions for a given slice of source code.
let highlighter = &mut ts_parser.borrow_mut(); // pub fn highlight_iter<'a>(
highlighter.cursors.pop().unwrap_or_else(QueryCursor::new) // &'a self,
}); // source: RopeSlice<'a>,
// range: Option<std::ops::Range<usize>>,
// cancellation_flag: Option<&'a AtomicUsize>,
// ) -> impl Iterator<Item = Result<HighlightEvent, Error>> + 'a {
// let mut layers = self
// .layers
// .iter()
// .filter_map(|(_, layer)| {
// // TODO: if range doesn't overlap layer range, skip it
// The `captures` iterator borrows the `Tree` and the `QueryCursor`, which // // Reuse a cursor from the pool if available.
// prevents them from being moved. But both of these values are really just // let mut cursor = PARSER.with(|ts_parser| {
// pointers, so it's actually ok to move them. // let highlighter = &mut ts_parser.borrow_mut();
let cursor_ref = // highlighter.cursors.pop().unwrap_or_else(QueryCursor::new)
unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) }; // });
// if reusing cursors & no range this resets to whole range // // The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
cursor_ref.set_byte_range(range.clone().unwrap_or(0..usize::MAX)); // // prevents them from being moved. But both of these values are really just
cursor_ref.set_match_limit(TREE_SITTER_MATCH_LIMIT); // // pointers, so it's actually ok to move them.
// let cursor_ref =
// unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };
let mut captures = cursor_ref // // if reusing cursors & no range this resets to whole range
.captures( // cursor_ref.set_byte_range(range.clone().unwrap_or(0..usize::MAX));
&layer.config.query, // cursor_ref.set_match_limit(TREE_SITTER_MATCH_LIMIT);
layer.tree().root_node(),
RopeProvider(source),
)
.peekable();
// If there's no captures, skip the layer // let mut captures = cursor_ref
captures.peek()?; // .captures(
// &layer.config.query,
// layer.tree().root_node(),
// RopeProvider(source),
// )
// .peekable();
Some(HighlightIterLayer { // // If there's no captures, skip the layer
highlight_end_stack: Vec::new(), // captures.peek()?;
scope_stack: vec![LocalScope {
inherits: false,
range: 0..usize::MAX,
local_defs: Vec::new(),
}],
cursor,
_tree: None,
captures: RefCell::new(captures),
config: layer.config.as_ref(), // TODO: just reuse `layer`
depth: layer.depth, // TODO: just reuse `layer`
})
})
.collect::<Vec<_>>();
layers.sort_unstable_by_key(|layer| layer.sort_key()); // Some(HighlightIterLayer {
// highlight_end_stack: Vec::new(),
// scope_stack: vec![LocalScope {
// inherits: false,
// range: 0..usize::MAX,
// local_defs: Vec::new(),
// }],
// cursor,
// _tree: None,
// captures: RefCell::new(captures),
// config: layer.config.as_ref(), // TODO: just reuse `layer`
// depth: layer.depth, // TODO: just reuse `layer`
// })
// })
// .collect::<Vec<_>>();
let mut result = HighlightIter { // layers.sort_unstable_by_key(|layer| layer.sort_key());
source,
byte_offset: range.map_or(0, |r| r.start), // let mut result = HighlightIter {
cancellation_flag, // source,
iter_count: 0, // byte_offset: range.map_or(0, |r| r.start),
layers, // cancellation_flag,
next_event: None, // iter_count: 0,
last_highlight_range: None, // layers,
}; // next_event: None,
result.sort_layers(); // last_highlight_range: None,
result // };
} // result.sort_layers();
} // result
// }
// }

206
helix-syntax/src/highlighter2.rs Normal file
View File

@ -0,0 +1,206 @@
use std::borrow::Cow;
use std::iter::{self, Peekable};
use std::mem::{replace, take};
use std::slice;
use hashbrown::HashMap;
use crate::query_iter::{MatchedNode, QueryIter, QueryIterEvent};
use crate::{Injection, LayerId, Range, Syntax};
/// Indicates which highlight should be applied to a region of source code.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub struct Highlight(pub u32);
impl Highlight{
pub(crate) const NONE = Highlight(u32::MAX);
}
#[derive(Debug)]
struct LocalDef<'a> {
name: Cow<'a, str>,
value_range: Range,
highlight: Option<Highlight>,
}
#[derive(Debug)]
struct LocalScope<'a> {
inherits: bool,
range: Range,
local_defs: Vec<LocalDef<'a>>,
}
#[derive(Debug)]
struct HighlightedNode {
end: u32,
highlight: Highlight,
}
#[derive(Debug, Default)]
struct LayerData<'a> {
parent_highlights: usize,
dormant_highlights: Vec<HighlightedNode>,
scope_stack: Vec<LocalDef<'a>>,
}
struct HighlighterConfig<'a> {
new_precedance: bool,
highlight_indices: &'a [Highlight],
}
pub struct Highligther<'a> {
query: QueryIter<'a, LayerData<'a>>,
next_query_event: Option<QueryIterEvent<LayerData<'a>>>,
active_highlights: Vec<HighlightedNode>,
next_highlight_end: u32,
next_highlight_start: u32,
config: HighlighterConfig<'a>,
}
pub struct HighlightList<'a>(slice::Iter<'a, HighlightedNode>);
impl<'a> Iterator for HighlightList<'a> {
type Item = Highlight;
fn next(&mut self) -> Option<Highlight> {
self.0.next().map(|node| node.highlight)
}
}
pub enum HighlighEvent<'a> {
RefreshHiglights(HighlightList<'a>),
PushHighlights(HighlightList<'a>),
}
impl<'a> Highligther<'a> {
pub fn active_highlights(&self) -> HighlightList<'_> {
HighlightList(self.active_highlights.iter())
}
pub fn next_event_offset(&self) -> u32 {
self.next_highlight_start.min(self.next_highlight_end)
}
pub fn advance(&mut self) -> HighlighEvent<'_> {
let mut refresh = false;
let prev_stack_size = self.active_highlights.len();
let pos = self.next_event_offset();
if self.next_highlight_end == pos {
self.process_injection_ends();
self.process_higlight_end();
refresh = true;
}
let mut first_highlight = true;
while self.next_highlight_start == pos {
let Some(query_event) = self.adance_query_iter() else {
break;
};
match query_event {
QueryIterEvent::EnterInjection(_) => self.enter_injection(),
QueryIterEvent::Match(node) => self.start_highlight(node, &mut first_highlight),
QueryIterEvent::ExitInjection { injection, state } => {
// state is returned if the layer is finifhed, if it isn't we have
// a combined injection and need to deactive its highlights
if state.is_none() {
self.deactive_layer(injection.layer);
refresh = true;
}
}
}
}
self.next_highlight_end = self
.active_highlights
.last()
.map_or(u32::MAX, |node| node.end);
if refresh {
HighlighEvent::RefreshHiglights(HighlightList(self.active_highlights.iter()))
} else {
HighlighEvent::PushHighlights(HighlightList(
self.active_highlights[prev_stack_size..].iter(),
))
}
}
fn adance_query_iter(&mut self) -> Option<QueryIterEvent<LayerData<'a>>> {
let event = replace(&mut self.next_query_event, self.query.next());
self.next_highlight_start = self
.next_query_event
.as_ref()
.map_or(u32::MAX, |event| event.start());
event
}
fn process_higlight_end(&mut self) {
let i = self
.active_highlights
.iter()
.rposition(|highlight| highlight.end != self.next_highlight_end)
.unwrap();
self.active_highlights.truncate(i);
}
/// processes injections that end at the same position as highlights first.
fn process_injection_ends(&mut self) {
while self.next_highlight_end == self.next_highlight_start {
match self.next_query_event.as_ref() {
Some(QueryIterEvent::ExitInjection { injection, state }) => {
if state.is_none() {
self.deactive_layer(injection.layer);
}
}
Some(QueryIterEvent::Match(matched_node)) if matched_node.byte_range.is_empty() => {
}
_ => break,
}
}
}
fn enter_injection(&mut self) {
self.query.current_layer_state().parent_highlights = self.active_highlights.len();
}
fn deactive_layer(&mut self, layer: LayerId) {
let LayerData {
parent_highlights,
ref mut dormant_highlights,
..
} = *self.query.layer_state(layer);
let i = self.active_highlights[parent_highlights..]
.iter()
.rposition(|highlight| highlight.end != self.next_highlight_end)
.unwrap();
self.active_highlights.truncate(parent_highlights + i);
dormant_highlights.extend(self.active_highlights.drain(parent_highlights..))
}
fn start_highlight(&mut self, node: MatchedNode, first_highlight: &mut bool) {
if node.byte_range.is_empty() {
return;
}
// if there are multiple matches for the exact same node
// only use one of the (the last with new/nvim precedance)
if !*first_highlight
&& self.active_highlights.last().map_or(false, |prev_node| {
prev_node.end == node.byte_range.end as u32
})
{
if self.config.new_precedance {
self.active_highlights.pop();
} else {
return;
}
}
let highlight = self.config.highlight_indices[node.capture.idx()];
if highlight.0 == u32::MAX {
return;
}
self.active_highlights.push(HighlightedNode {
end: node.byte_range.end as u32,
highlight,
});
*first_highlight = false;
}
}

268
helix-syntax/src/injections_tree.rs
View File

@ -1,268 +0,0 @@
use core::slice;
use std::cell::RefCell;
use std::iter::Peekable;
use std::mem::replace;
use std::sync::Arc;
use hashbrown::HashMap;
use ropey::RopeSlice;
use slotmap::{new_key_type, SlotMap};
use crate::parse::LayerUpdateFlags;
use crate::tree_sitter::{
self, Capture, InactiveQueryCursor, Parser, Query, QueryCursor, RopeTsInput, SyntaxTree,
SyntaxTreeNode,
};
use crate::HighlightConfiguration;
// TODO(perf): replace std::ops::Range<usize> with helix_stdx::Range<u32> once added
type Range = std::ops::Range<usize>;
new_key_type! {
/// The default slot map key type.
pub struct LayerId;
}
#[derive(Debug)]
pub struct LanguageLayer {
pub config: Arc<HighlightConfiguration>,
pub(crate) parse_tree: Option<SyntaxTree>,
/// internal flags used during parsing to track incremental invalidation
pub(crate) flags: LayerUpdateFlags,
ranges: Vec<tree_sitter::Range>,
pub(crate) parent: Option<LayerId>,
/// a list of **sorted** non-overlapping injection ranges. Note that
/// injection ranges are not relative to the start of this layer but the
/// start of the root layer
pub(crate) injections: Box<[Injection]>,
}
#[derive(Debug, Clone)]
pub(crate) struct Injection {
pub byte_range: Range,
pub layer: LayerId,
}
impl LanguageLayer {
/// Returns the injection range **within this layers** that contains `idx`.
/// This function will not descend into nested injections
pub(crate) fn injection_at_byte_idx(&self, idx: usize) -> Option<&Injection> {
let i = self
.injections
.partition_point(|range| range.byte_range.start <= idx);
self.injections
.get(i)
.filter(|injection| injection.byte_range.end > idx)
}
}
struct InjectionTree {
layers: SlotMap<LayerId, LanguageLayer>,
root: LayerId,
}
impl InjectionTree {
pub fn layer_for_byte_range(&self, start: usize, end: usize) -> LayerId {
let mut cursor = self.root;
loop {
let layer = &self.layers[cursor];
let Some(start_injection) = layer.injection_at_byte_idx(start) else {
break;
};
let Some(end_injection) = layer.injection_at_byte_idx(end) else {
break;
};
if start_injection.layer == end_injection.layer {
cursor = start_injection.layer;
} else {
break;
}
}
cursor
}
}
#[derive(Clone)]
pub struct MatchedNode {
pub capture: Capture,
pub byte_range: Range,
}
struct LayerQueryIter<'a> {
cursor: QueryCursor<'a, 'a, RopeTsInput<'a>>,
peeked: Option<MatchedNode>,
}
impl<'a> LayerQueryIter<'a> {
fn peek(&mut self) -> Option<&MatchedNode> {
if self.peeked.is_none() {
let (query_match, node_idx) = self.cursor.next_matched_node()?;
let matched_node = query_match.matched_node(node_idx);
self.peeked = Some(MatchedNode {
capture: matched_node.capture,
byte_range: matched_node.syntax_node.byte_range(),
});
}
self.peeked.as_ref()
}
fn consume(&mut self) -> MatchedNode {
self.peeked.take().unwrap()
}
}
struct ActiveLayer<'a> {
query_iter: LayerQueryIter<'a>,
injections: Peekable<slice::Iter<'a, Injection>>,
}
struct QueryBuilder<'a, 'tree> {
query: &'a Query,
node: &'a SyntaxTreeNode<'tree>,
src: RopeSlice<'a>,
injection_tree: &'a InjectionTree,
}
pub struct QueryIter<'a, 'tree> {
query_builder: Box<QueryBuilder<'a, 'tree>>,
active_layers: HashMap<LayerId, ActiveLayer<'a>>,
active_injections: Vec<Injection>,
current_injection: Injection,
}
impl<'a> QueryIter<'a, '_> {
fn enter_injection(&mut self, injection: Injection) -> bool {
self.active_layers
.entry(injection.layer)
.or_insert_with(|| {
let layer = &self.query_builder.injection_tree.layers[injection.layer];
let injection_start = layer
.injections
.partition_point(|child| child.byte_range.start < injection.byte_range.start);
let cursor = get_cursor().execute_query(
self.query_builder.query,
self.query_builder.node,
RopeTsInput::new(self.query_builder.src),
);
ActiveLayer {
query_iter: LayerQueryIter {
cursor,
peeked: None,
},
injections: layer.injections[injection_start..].iter().peekable(),
}
});
let old_injection = replace(&mut self.current_injection, injection);
self.active_injections.push(old_injection);
true
}
fn exit_injection(&mut self) -> Option<Injection> {
let injection = replace(&mut self.current_injection, self.active_injections.pop()?);
let finished_layer = self.active_layers[&injection.layer]
.query_iter
.peeked
.is_none();
if finished_layer {
let layer = self.active_layers.remove(&injection.layer).unwrap();
reuse_cursor(layer.query_iter.cursor.reuse());
}
Some(injection)
}
}
pub enum QueryIterEvent {
EnterInjection(Injection),
Match(MatchedNode),
ExitInjection(Injection),
}
impl<'a> Iterator for QueryIter<'a, '_> {
type Item = QueryIterEvent;
fn next(&mut self) -> Option<QueryIterEvent> {
loop {
let active_layer = self
.active_layers
.get_mut(&self.current_injection.layer)
.unwrap();
let next_injection = active_layer.injections.peek().filter(|injection| {
injection.byte_range.start < self.current_injection.byte_range.end
});
let next_match = active_layer.query_iter.peek().filter(|matched_node| {
matched_node.byte_range.start < self.current_injection.byte_range.end
});
match (next_match, next_injection) {
(None, None) => {
return self.exit_injection().map(QueryIterEvent::ExitInjection);
}
(Some(_), None) => {
// consume match
let matched_node = active_layer.query_iter.consume();
return Some(QueryIterEvent::Match(matched_node));
}
(Some(matched_node), Some(injection))
if matched_node.byte_range.start <= injection.byte_range.end =>
{
// consume match
let matched_node = active_layer.query_iter.consume();
// ignore nodes that are overlapped by the injection
if matched_node.byte_range.start <= injection.byte_range.start {
return Some(QueryIterEvent::Match(matched_node));
}
}
(Some(_), Some(_)) | (None, Some(_)) => {
// consume injection
let injection = active_layer.injections.next().unwrap();
if self.enter_injection(injection.clone()) {
return Some(QueryIterEvent::EnterInjection(injection.clone()));
}
}
}
}
}
}
struct TsParser {
parser: crate::tree_sitter::Parser,
pub cursors: Vec<crate::tree_sitter::InactiveQueryCursor>,
}
// could also just use a pool, or a single instance?
thread_local! {
static PARSER: RefCell<TsParser> = RefCell::new(TsParser {
parser: Parser::new(),
cursors: Vec::new(),
})
}
pub fn with_cursor<T>(f: impl FnOnce(&mut InactiveQueryCursor) -> T) -> T {
PARSER.with(|parser| {
let mut parser = parser.borrow_mut();
let mut cursor = parser
.cursors
.pop()
.unwrap_or_else(InactiveQueryCursor::new);
let res = f(&mut cursor);
parser.cursors.push(cursor);
res
})
}
pub fn get_cursor() -> InactiveQueryCursor {
PARSER.with(|parser| {
let mut parser = parser.borrow_mut();
parser
.cursors
.pop()
.unwrap_or_else(InactiveQueryCursor::new)
})
}
pub fn reuse_cursor(cursor: InactiveQueryCursor) {
PARSER.with(|parser| {
let mut parser = parser.borrow_mut();
parser.cursors.push(cursor)
})
}

459
helix-syntax/src/lib.rs
View File

@ -1,211 +1,32 @@
use ::ropey::RopeSlice; use ::ropey::RopeSlice;
use ::tree_sitter::{Node, Parser, Point, Query, QueryCursor, Range, Tree}; use slotmap::{new_key_type, HopSlotMap};
use slotmap::HopSlotMap;
use std::borrow::Cow; use std::borrow::Cow;
use std::cell::RefCell;
use std::hash::{Hash, Hasher}; use std::hash::{Hash, Hasher};
use std::path::Path; use std::path::Path;
use std::str; use std::str;
use std::sync::Arc; use std::sync::Arc;
use crate::injections_tree::LayerId;
use crate::parse::LayerUpdateFlags; use crate::parse::LayerUpdateFlags;
pub use crate::config::{read_query, HighlightConfiguration}; pub use crate::config::{read_query, HighlightConfiguration};
pub use crate::ropey::RopeProvider; use crate::tree_sitter::{SyntaxTree, SyntaxTreeNode};
pub use merge::merge;
pub use pretty_print::pretty_print_tree; pub use pretty_print::pretty_print_tree;
pub use tree_cursor::TreeCursor; pub use tree_cursor::TreeCursor;
mod config; mod config;
pub mod highlighter; pub mod highlighter;
mod injections_tree; pub mod highlighter2;
mod merge;
mod parse; mod parse;
mod pretty_print; mod pretty_print;
mod ropey; mod query_iter;
pub mod text_object;
mod tree_cursor; mod tree_cursor;
pub mod tree_sitter; pub mod tree_sitter;
#[derive(Debug)] new_key_type! {
pub struct Syntax { /// The default slot map key type.
layers: HopSlotMap<LayerId, LanguageLayer>, pub struct LayerId;
root: LayerId,
}
impl Syntax {
pub fn new(
source: RopeSlice,
config: Arc<HighlightConfiguration>,
injection_callback: impl Fn(&InjectionLanguageMarker) -> Option<Arc<HighlightConfiguration>>,
) -> Option<Self> {
let root_layer = LanguageLayer {
tree: None,
config,
depth: 0,
flags: LayerUpdateFlags::empty(),
ranges: vec![Range {
start_byte: 0,
end_byte: usize::MAX,
start_point: Point::new(0, 0),
end_point: Point::new(usize::MAX, usize::MAX),
}],
parent: None,
};
// track scope_descriptor: a Vec of scopes for item in tree
let mut layers = HopSlotMap::default();
let root = layers.insert(root_layer);
let mut syntax = Self { root, layers };
let res = syntax.update(source, Vec::new(), injection_callback);
if res.is_err() {
log::error!("TS parser failed, disabling TS for the current buffer: {res:?}");
return None;
}
Some(syntax)
}
pub fn tree(&self) -> &Tree {
self.layers[self.root].tree()
}
pub fn tree_for_byte_range(&self, start: usize, end: usize) -> &Tree {
let mut container_id = self.root;
for (layer_id, layer) in self.layers.iter() {
if layer.depth > self.layers[container_id].depth
&& layer.contains_byte_range(start, end)
{
container_id = layer_id;
}
}
self.layers[container_id].tree()
}
pub fn named_descendant_for_byte_range(&self, start: usize, end: usize) -> Option<Node<'_>> {
self.tree_for_byte_range(start, end)
.root_node()
.named_descendant_for_byte_range(start, end)
}
pub fn descendant_for_byte_range(&self, start: usize, end: usize) -> Option<Node<'_>> {
self.tree_for_byte_range(start, end)
.root_node()
.descendant_for_byte_range(start, end)
}
pub fn walk(&self) -> TreeCursor<'_> {
TreeCursor::new(&self.layers, self.root)
}
}
#[derive(Debug)]
pub struct LanguageLayer {
// mode
// grammar
pub config: Arc<HighlightConfiguration>,
pub(crate) tree: Option<Tree>,
pub ranges: Vec<Range>,
pub depth: u32,
flags: LayerUpdateFlags,
parent: Option<LayerId>,
}
/// This PartialEq implementation only checks if that
/// two layers are theoretically identical (meaning they highlight the same text range with the same language).
/// It does not check whether the layers have the same internal treesitter
/// state.
impl PartialEq for LanguageLayer {
fn eq(&self, other: &Self) -> bool {
self.depth == other.depth
&& self.config.language == other.config.language
&& self.ranges == other.ranges
}
}
/// Hash implementation belongs to PartialEq implementation above.
/// See its documentation for details.
impl Hash for LanguageLayer {
fn hash<H: Hasher>(&self, state: &mut H) {
self.depth.hash(state);
self.config.language.hash(state);
self.ranges.hash(state);
}
}
impl LanguageLayer {
pub fn tree(&self) -> &Tree {
// TODO: no unwrap
self.tree.as_ref().unwrap()
}
/// Whether the layer contains the given byte range.
///
/// If the layer has multiple ranges (i.e. combined injections), the
/// given range is considered contained if it is within the start and
/// end bytes of the first and last ranges **and** if the given range
/// starts or ends within any of the layer's ranges.
fn contains_byte_range(&self, start: usize, end: usize) -> bool {
let layer_start = self
.ranges
.first()
.expect("ranges should not be empty")
.start_byte;
let layer_end = self
.ranges
.last()
.expect("ranges should not be empty")
.end_byte;
layer_start <= start
&& layer_end >= end
&& self.ranges.iter().any(|range| {
let byte_range = range.start_byte..range.end_byte;
byte_range.contains(&start) || byte_range.contains(&end)
})
}
}
#[derive(Debug, Clone)]
pub enum InjectionLanguageMarker<'a> {
Name(Cow<'a, str>),
Filename(Cow<'a, Path>),
Shebang(String),
}
const SHEBANG: &str = r"#!\s*(?:\S*[/\\](?:env\s+(?:\-\S+\s+)*)?)?([^\s\.\d]+)";
#[derive(Debug)]
pub enum CapturedNode<'a> {
Single(Node<'a>),
/// Guaranteed to be not empty
Grouped(Vec<Node<'a>>),
}
impl<'a> CapturedNode<'a> {
pub fn start_byte(&self) -> usize {
match self {
Self::Single(n) => n.start_byte(),
Self::Grouped(ns) => ns[0].start_byte(),
}
}
pub fn end_byte(&self) -> usize {
match self {
Self::Single(n) => n.end_byte(),
Self::Grouped(ns) => ns.last().unwrap().end_byte(),
}
}
pub fn byte_range(&self) -> std::ops::Range<usize> {
self.start_byte()..self.end_byte()
}
} }
/// The maximum number of in-progress matches a TS cursor can consider at once. /// The maximum number of in-progress matches a TS cursor can consider at once.
@ -226,72 +47,192 @@ pub fn byte_range(&self) -> std::ops::Range<usize> {
/// Neovim chose 64 for this value somewhat arbitrarily (<https://github.com/neovim/neovim/pull/18397>). /// Neovim chose 64 for this value somewhat arbitrarily (<https://github.com/neovim/neovim/pull/18397>).
/// 64 is too low for some languages though. In particular, it breaks some highlighting for record fields in Erlang record definitions. /// 64 is too low for some languages though. In particular, it breaks some highlighting for record fields in Erlang record definitions.
/// This number can be increased if new syntax highlight breakages are found, as long as the performance penalty is not too high. /// This number can be increased if new syntax highlight breakages are found, as long as the performance penalty is not too high.
const TREE_SITTER_MATCH_LIMIT: u32 = 256; pub const TREE_SITTER_MATCH_LIMIT: u32 = 256;
// TODO(perf): replace std::ops::Range<usize> with helix_stdx::Range<u32> once added
type Range = std::ops::Range<usize>;
/// The Tree siitter syntax tree for a single language.
/// This is really multipe nested different syntax trees due to tree sitter
/// injections. A single syntax tree/parser is called layer. Each layer
/// is parsed as a single "file" by tree sitter. There can be multiple layers
/// for the same language. A layer corresponds to one of three things:
/// * the root layer
/// * a singular injection limited to a single node in it's parent layer
/// * Multiple injections (multiple disjoint nodes in parent layer) that are
/// parsed as tough they are a single uninterrupted file.
///
/// An injection always refer to a single node into which another layer is
/// injected. As injections only correspond to syntax tree nodes injections in
/// the same layer do not intersect. However, the syntax tree in a an injected
/// layer can have nodes that intersect with nodes from the parent layer. For
/// example:
/// ```
/// layer2: | Sibling A | Sibling B (layer3) | Sibling C |
/// layer1: | Sibling A (layer2) | Sibling B | Sibling C (layer2) |
/// ````
/// In this case Sibling B really spans across a "GAP" in layer2. While the syntax
/// node can not be split up by tree sitter directly, we can treat Sibling B as two
/// seperate injections. That is done while parsing/running the query capture. As
/// a result the injections from a tree. Note that such other queries must account for
/// such multi injection nodes.
#[derive(Debug)] #[derive(Debug)]
pub struct TextObjectQuery { pub struct Syntax {
pub query: Query, layers: HopSlotMap<LayerId, LanguageLayer>,
root: LayerId,
} }
impl TextObjectQuery { impl Syntax {
/// Run the query on the given node and return sub nodes which match given pub fn new(
/// capture ("function.inside", "class.around", etc). source: RopeSlice,
/// config: Arc<HighlightConfiguration>,
/// Captures may contain multiple nodes by using quantifiers (+, *, etc), injection_callback: impl Fn(&InjectionLanguageMarker) -> Option<Arc<HighlightConfiguration>>,
/// and support for this is partial and could use improvement. ) -> Option<Self> {
/// let root_layer = LanguageLayer {
/// ```query parse_tree: None,
/// (comment)+ @capture config,
/// flags: LayerUpdateFlags::empty(),
/// ; OR ranges: vec![tree_sitter::Range {
/// ( start_byte: 0,
/// (comment)* end_byte: u32::MAX,
/// . start_point: tree_sitter::Point { row: 0, col: 0 },
/// (function) end_point: tree_sitter::Point {
/// ) @capture row: u32::MAX,
/// ``` col: u32::MAX,
pub fn capture_nodes<'a>( },
&'a self, }]
capture_name: &str, .into_boxed_slice(),
node: Node<'a>, injections: Box::new([]),
slice: RopeSlice<'a>, parent: None,
cursor: &'a mut QueryCursor, };
) -> Option<impl Iterator<Item = CapturedNode<'a>>> {
self.capture_nodes_any(&[capture_name], node, slice, cursor) // track scope_descriptor: a Vec of scopes for item in tree
let mut layers = HopSlotMap::default();
let root = layers.insert(root_layer);
let mut syntax = Self { root, layers };
let res = syntax.update(source, Vec::new(), injection_callback);
if res.is_err() {
log::error!("TS parser failed, disabling TS for the current buffer: {res:?}");
return None;
}
Some(syntax)
} }
/// Find the first capture that exists out of all given `capture_names` pub fn tree(&self) -> &SyntaxTree {
/// and return sub nodes that match this capture. self.layers[self.root].tree()
pub fn capture_nodes_any<'a>( }
&'a self,
capture_names: &[&str],
node: Node<'a>,
slice: RopeSlice<'a>,
cursor: &'a mut QueryCursor,
) -> Option<impl Iterator<Item = CapturedNode<'a>>> {
let capture_idx = capture_names
.iter()
.find_map(|cap| self.query.capture_index_for_name(cap))?;
cursor.set_match_limit(TREE_SITTER_MATCH_LIMIT); pub fn tree_for_byte_range(&self, start: usize, end: usize) -> &SyntaxTree {
let layer = self.layer_for_byte_range(start, end);
self.layers[layer].tree()
}
let nodes = cursor pub fn named_descendant_for_byte_range(
.captures(&self.query, node, RopeProvider(slice)) &self,
.filter_map(move |(mat, _)| { start: usize,
let nodes: Vec<_> = mat end: usize,
.captures ) -> Option<SyntaxTreeNode<'_>> {
.iter() self.tree_for_byte_range(start, end)
.filter_map(|cap| (cap.index == capture_idx).then_some(cap.node)) .root_node()
.collect(); .named_descendant_for_byte_range(start, end)
}
if nodes.len() > 1 { pub fn descendant_for_byte_range(
Some(CapturedNode::Grouped(nodes)) &self,
start: usize,
end: usize,
) -> Option<SyntaxTreeNode<'_>> {
self.tree_for_byte_range(start, end)
.root_node()
.descendant_for_byte_range(start, end)
}
pub fn layer_for_byte_range(&self, start: usize, end: usize) -> LayerId {
let mut cursor = self.root;
loop {
let layer = &self.layers[cursor];
let Some(start_injection) = layer.injection_at_byte_idx(start) else {
break;
};
let Some(end_injection) = layer.injection_at_byte_idx(end) else {
break;
};
if start_injection.layer == end_injection.layer {
cursor = start_injection.layer;
} else { } else {
nodes.into_iter().map(CapturedNode::Single).next() break;
}
}
cursor
} }
});
Some(nodes) pub fn walk(&self) -> TreeCursor<'_> {
TreeCursor::new(&self.layers, self.root)
}
}
#[derive(Debug, Clone)]
pub(crate) struct Injection {
pub byte_range: Range,
pub layer: LayerId,
}
#[derive(Debug)]
pub struct LanguageLayer {
pub config: Arc<HighlightConfiguration>,
parse_tree: Option<SyntaxTree>,
ranges: Box<[tree_sitter::Range]>,
/// a list of **sorted** non-overlapping injection ranges. Note that
/// injection ranges are not relative to the start of this layer but the
/// start of the root layer
injections: Box<[Injection]>,
/// internal flags used during parsing to track incremental invalidation
flags: LayerUpdateFlags,
parent: Option<LayerId>,
}
/// This PartialEq implementation only checks if that
/// two layers are theoretically identical (meaning they highlight the same text range with the same language).
/// It does not check whether the layers have the same internal treesitter
/// state.
impl PartialEq for LanguageLayer {
fn eq(&self, other: &Self) -> bool {
self.parent == other.parent
&& self.config.grammar == other.config.grammar
&& self.ranges == other.ranges
}
}
/// Hash implementation belongs to PartialEq implementation above.
/// See its documentation for details.
impl Hash for LanguageLayer {
fn hash<H: Hasher>(&self, state: &mut H) {
self.parent.hash(state);
self.config.grammar.hash(state);
self.ranges.hash(state);
}
}
impl LanguageLayer {
pub fn tree(&self) -> &SyntaxTree {
// TODO: no unwrap
self.parse_tree.as_ref().unwrap()
}
/// Returns the injection range **within this layers** that contains `idx`.
/// This function will not descend into nested injections
pub(crate) fn injection_at_byte_idx(&self, idx: usize) -> Option<&Injection> {
let i = self
.injections
.partition_point(|range| range.byte_range.start < idx);
self.injections
.get(i)
.filter(|injection| injection.byte_range.end > idx)
} }
} }
@ -304,42 +245,6 @@ pub enum Error {
Unknown, Unknown,
} }
#[derive(Clone)]
enum IncludedChildren {
None,
All,
Unnamed,
}
impl Default for IncludedChildren {
fn default() -> Self {
Self::None
}
}
fn byte_range_to_str(range: std::ops::Range<usize>, source: RopeSlice) -> Cow<str> { fn byte_range_to_str(range: std::ops::Range<usize>, source: RopeSlice) -> Cow<str> {
Cow::from(source.byte_slice(range)) Cow::from(source.byte_slice(range))
} }
struct TsParser {
parser: ::tree_sitter::Parser,
pub cursors: Vec<QueryCursor>,
}
// could also just use a pool, or a single instance?
thread_local! {
static PARSER: RefCell<TsParser> = RefCell::new(TsParser {
parser: Parser::new(),
cursors: Vec::new(),
})
}
pub fn with_cursor<T>(f: impl FnOnce(&mut QueryCursor) -> T) -> T {
PARSER.with(|parser| {
let mut parser = parser.borrow_mut();
let mut cursor = parser.cursors.pop().unwrap_or_default();
let res = f(&mut cursor);
parser.cursors.push(cursor);
res
})
}

750
helix-syntax/src/parse.rs
View File

@ -1,18 +1,18 @@
use std::collections::VecDeque; // use std::collections::VecDeque;
use std::mem::replace; // use std::mem::replace;
use std::sync::Arc; // use std::sync::Arc;
use ahash::RandomState; // use ahash::RandomState;
use bitflags::bitflags; use bitflags::bitflags;
use hashbrown::raw::RawTable; // use hashbrown::raw::RawTable;
use ropey::RopeSlice; // use ropey::RopeSlice;
use tree_sitter::{Node, Parser, Point, QueryCursor, Range}; // use tree_sitter::{Node, Parser, Point, QueryCursor, Range};
use crate::ropey::RopeProvider; // use crate::ropey::RopeProvider;
use crate::{ // use crate::{
Error, HighlightConfiguration, IncludedChildren, InjectionLanguageMarker, LanguageLayer, // Error, HighlightConfiguration, IncludedChildren, InjectionLanguageMarker, LanguageLayer,
Syntax, PARSER, TREE_SITTER_MATCH_LIMIT, // Syntax, PARSER, TREE_SITTER_MATCH_LIMIT,
}; // };
bitflags! { bitflags! {
/// Flags that track the status of a layer /// Flags that track the status of a layer
@ -25,405 +25,405 @@ pub(crate) struct LayerUpdateFlags : u32{
} }
} }
impl Syntax { // impl Syntax {
pub fn update( // pub fn update(
&mut self, // &mut self,
source: RopeSlice, // source: RopeSlice,
edits: Vec<tree_sitter::InputEdit>, // edits: Vec<tree_sitter::InputEdit>,
injection_callback: impl Fn(&InjectionLanguageMarker) -> Option<Arc<HighlightConfiguration>>, // injection_callback: impl Fn(&InjectionLanguageMarker) -> Option<Arc<HighlightConfiguration>>,
) -> Result<(), Error> { // ) -> Result<(), Error> {
let mut queue = VecDeque::new(); // let mut queue = VecDeque::new();
queue.push_back(self.root); // queue.push_back(self.root);
// This table allows inverse indexing of `layers`. // // This table allows inverse indexing of `layers`.
// That is by hashing a `Layer` you can find // // That is by hashing a `Layer` you can find
// the `LayerId` of an existing equivalent `Layer` in `layers`. // // the `LayerId` of an existing equivalent `Layer` in `layers`.
// // //
// It is used to determine if a new layer exists for an injection // // It is used to determine if a new layer exists for an injection
// or if an existing layer needs to be updated. // // or if an existing layer needs to be updated.
let mut layers_table = RawTable::with_capacity(self.layers.len()); // let mut layers_table = RawTable::with_capacity(self.layers.len());
let layers_hasher = RandomState::new(); // let layers_hasher = RandomState::new();
// Use the edits to update all layers markers // // Use the edits to update all layers markers
fn point_add(a: Point, b: Point) -> Point { // fn point_add(a: Point, b: Point) -> Point {
if b.row > 0 { // if b.row > 0 {
Point::new(a.row.saturating_add(b.row), b.column) // Point::new(a.row.saturating_add(b.row), b.column)
} else { // } else {
Point::new(0, a.column.saturating_add(b.column)) // Point::new(0, a.column.saturating_add(b.column))
} // }
} // }
fn point_sub(a: Point, b: Point) -> Point { // fn point_sub(a: Point, b: Point) -> Point {
if a.row > b.row { // if a.row > b.row {
Point::new(a.row.saturating_sub(b.row), a.column) // Point::new(a.row.saturating_sub(b.row), a.column)
} else { // } else {
Point::new(0, a.column.saturating_sub(b.column)) // Point::new(0, a.column.saturating_sub(b.column))
} // }
} // }
for (layer_id, layer) in self.layers.iter_mut() { // for (layer_id, layer) in self.layers.iter_mut() {
// The root layer always covers the whole range (0..usize::MAX) // // The root layer always covers the whole range (0..usize::MAX)
if layer.depth == 0 { // if layer.depth == 0 {
layer.flags = LayerUpdateFlags::MODIFIED; // layer.flags = LayerUpdateFlags::MODIFIED;
continue; // continue;
} // }
if !edits.is_empty() { // if !edits.is_empty() {
for range in &mut layer.ranges { // for range in &mut layer.ranges {
// Roughly based on https://github.com/tree-sitter/tree-sitter/blob/ddeaa0c7f534268b35b4f6cb39b52df082754413/lib/src/subtree.c#L691-L720 // // Roughly based on https://github.com/tree-sitter/tree-sitter/blob/ddeaa0c7f534268b35b4f6cb39b52df082754413/lib/src/subtree.c#L691-L720
for edit in edits.iter().rev() { // for edit in edits.iter().rev() {
let is_pure_insertion = edit.old_end_byte == edit.start_byte; // let is_pure_insertion = edit.old_end_byte == edit.start_byte;
// if edit is after range, skip // // if edit is after range, skip
if edit.start_byte > range.end_byte { // if edit.start_byte > range.end_byte {
// TODO: || (is_noop && edit.start_byte == range.end_byte) // // TODO: || (is_noop && edit.start_byte == range.end_byte)
continue; // continue;
} // }
// if edit is before range, shift entire range by len // // if edit is before range, shift entire range by len
if edit.old_end_byte < range.start_byte { // if edit.old_end_byte < range.start_byte {
range.start_byte = // range.start_byte =
edit.new_end_byte + (range.start_byte - edit.old_end_byte); // edit.new_end_byte + (range.start_byte - edit.old_end_byte);
range.start_point = point_add( // range.start_point = point_add(
edit.new_end_position, // edit.new_end_position,
point_sub(range.start_point, edit.old_end_position), // point_sub(range.start_point, edit.old_end_position),
); // );
range.end_byte = edit // range.end_byte = edit
.new_end_byte // .new_end_byte
.saturating_add(range.end_byte - edit.old_end_byte); // .saturating_add(range.end_byte - edit.old_end_byte);
range.end_point = point_add( // range.end_point = point_add(
edit.new_end_position, // edit.new_end_position,
point_sub(range.end_point, edit.old_end_position), // point_sub(range.end_point, edit.old_end_position),
); // );
layer.flags |= LayerUpdateFlags::MOVED; // layer.flags |= LayerUpdateFlags::MOVED;
} // }
// if the edit starts in the space before and extends into the range // // if the edit starts in the space before and extends into the range
else if edit.start_byte < range.start_byte { // else if edit.start_byte < range.start_byte {
range.start_byte = edit.new_end_byte; // range.start_byte = edit.new_end_byte;
range.start_point = edit.new_end_position; // range.start_point = edit.new_end_position;
range.end_byte = range // range.end_byte = range
.end_byte // .end_byte
.saturating_sub(edit.old_end_byte) // .saturating_sub(edit.old_end_byte)
.saturating_add(edit.new_end_byte); // .saturating_add(edit.new_end_byte);
range.end_point = point_add( // range.end_point = point_add(
edit.new_end_position, // edit.new_end_position,
point_sub(range.end_point, edit.old_end_position), // point_sub(range.end_point, edit.old_end_position),
); // );
layer.flags = LayerUpdateFlags::MODIFIED; // layer.flags = LayerUpdateFlags::MODIFIED;
} // }
// If the edit is an insertion at the start of the tree, shift // // If the edit is an insertion at the start of the tree, shift
else if edit.start_byte == range.start_byte && is_pure_insertion { // else if edit.start_byte == range.start_byte && is_pure_insertion {
range.start_byte = edit.new_end_byte; // range.start_byte = edit.new_end_byte;
range.start_point = edit.new_end_position; // range.start_point = edit.new_end_position;
layer.flags |= LayerUpdateFlags::MOVED; // layer.flags |= LayerUpdateFlags::MOVED;
} else { // } else {
range.end_byte = range // range.end_byte = range
.end_byte // .end_byte
.saturating_sub(edit.old_end_byte) // .saturating_sub(edit.old_end_byte)
.saturating_add(edit.new_end_byte); // .saturating_add(edit.new_end_byte);
range.end_point = point_add( // range.end_point = point_add(
edit.new_end_position, // edit.new_end_position,
point_sub(range.end_point, edit.old_end_position), // point_sub(range.end_point, edit.old_end_position),
); // );
layer.flags = LayerUpdateFlags::MODIFIED; // layer.flags = LayerUpdateFlags::MODIFIED;
} // }
} // }
} // }
} // }
let hash = layers_hasher.hash_one(layer); // let hash = layers_hasher.hash_one(layer);
// Safety: insert_no_grow is unsafe because it assumes that the table // // Safety: insert_no_grow is unsafe because it assumes that the table
// has enough capacity to hold additional elements. // // has enough capacity to hold additional elements.
// This is always the case as we reserved enough capacity above. // // This is always the case as we reserved enough capacity above.
unsafe { layers_table.insert_no_grow(hash, layer_id) }; // unsafe { layers_table.insert_no_grow(hash, layer_id) };
} // }
PARSER.with(|ts_parser| { // PARSER.with(|ts_parser| {
let ts_parser = &mut ts_parser.borrow_mut(); // let ts_parser = &mut ts_parser.borrow_mut();
ts_parser.parser.set_timeout_micros(1000 * 500); // half a second is pretty generours // ts_parser.parser.set_timeout_micros(1000 * 500); // half a second is pretty generours
let mut cursor = ts_parser.cursors.pop().unwrap_or_else(QueryCursor::new); // let mut cursor = ts_parser.cursors.pop().unwrap_or_else(QueryCursor::new);
// TODO: might need to set cursor range // // TODO: might need to set cursor range
cursor.set_byte_range(0..usize::MAX); // cursor.set_byte_range(0..usize::MAX);
cursor.set_match_limit(TREE_SITTER_MATCH_LIMIT); // cursor.set_match_limit(TREE_SITTER_MATCH_LIMIT);
let source_slice = source.slice(..); // let source_slice = source.slice(..);
while let Some(layer_id) = queue.pop_front() { // while let Some(layer_id) = queue.pop_front() {
let layer = &mut self.layers[layer_id]; // let layer = &mut self.layers[layer_id];
// Mark the layer as touched // // Mark the layer as touched
layer.flags |= LayerUpdateFlags::TOUCHED; // layer.flags |= LayerUpdateFlags::TOUCHED;
// If a tree already exists, notify it of changes. // // If a tree already exists, notify it of changes.
if let Some(tree) = &mut layer.tree { // if let Some(tree) = &mut layer.parse_tree {
if layer // if layer
.flags // .flags
.intersects(LayerUpdateFlags::MODIFIED | LayerUpdateFlags::MOVED) // .intersects(LayerUpdateFlags::MODIFIED | LayerUpdateFlags::MOVED)
{ // {
for edit in edits.iter().rev() { // for edit in edits.iter().rev() {
// Apply the edits in reverse. // // Apply the edits in reverse.
// If we applied them in order then edit 1 would disrupt the positioning of edit 2. // // If we applied them in order then edit 1 would disrupt the positioning of edit 2.
tree.edit(edit); // tree.edit(edit);
} // }
} // }
if layer.flags.contains(LayerUpdateFlags::MODIFIED) { // if layer.flags.contains(LayerUpdateFlags::MODIFIED) {
// Re-parse the tree. // // Re-parse the tree.
layer.parse(&mut ts_parser.parser, source)?; // layer.parse(&mut ts_parser.parser, source)?;
} // }
} else { // } else {
// always parse if this layer has never been parsed before // // always parse if this layer has never been parsed before
layer.parse(&mut ts_parser.parser, source)?; // layer.parse(&mut ts_parser.parser, source)?;
} // }
// Switch to an immutable borrow. // // Switch to an immutable borrow.
let layer = &self.layers[layer_id]; // let layer = &self.layers[layer_id];
// Process injections. // // Process injections.
let matches = cursor.matches( // let matches = cursor.matches(
&layer.config.injections_query, // &layer.config.injections_query,
layer.tree().root_node(), // layer.tree().root_node(),
RopeProvider(source_slice), // RopeProvider(source_slice),
); // );
let mut combined_injections = vec![ // let mut combined_injections = vec![
(None, Vec::new(), IncludedChildren::default()); // (None, Vec::new(), IncludedChildren::default());
layer.config.combined_injections_patterns.len() // layer.config.combined_injections_patterns.len()
]; // ];
let mut injections = Vec::new(); // let mut injections = Vec::new();
let mut last_injection_end = 0; // let mut last_injection_end = 0;
for mat in matches { // for mat in matches {
let (injection_capture, content_node, included_children) = layer // let (injection_capture, content_node, included_children) = layer
.config // .config
.injection_for_match(&layer.config.injections_query, &mat, source_slice); // .injection_for_match(&layer.config.injections_query, &mat, source_slice);
// in case this is a combined injection save it for more processing later // // in case this is a combined injection save it for more processing later
if let Some(combined_injection_idx) = layer // if let Some(combined_injection_idx) = layer
.config // .config
.combined_injections_patterns // .combined_injections_patterns
.iter() // .iter()
.position(|&pattern| pattern == mat.pattern_index) // .position(|&pattern| pattern == mat.pattern_index)
{ // {
let entry = &mut combined_injections[combined_injection_idx]; // let entry = &mut combined_injections[combined_injection_idx];
if injection_capture.is_some() { // if injection_capture.is_some() {
entry.0 = injection_capture; // entry.0 = injection_capture;
} // }
if let Some(content_node) = content_node { // if let Some(content_node) = content_node {
if content_node.start_byte() >= last_injection_end { // if content_node.start_byte() >= last_injection_end {
entry.1.push(content_node); // entry.1.push(content_node);
last_injection_end = content_node.end_byte(); // last_injection_end = content_node.end_byte();
} // }
} // }
entry.2 = included_children; // entry.2 = included_children;
continue; // continue;
} // }
// Explicitly remove this match so that none of its other captures will remain // // Explicitly remove this match so that none of its other captures will remain
// in the stream of captures. // // in the stream of captures.
mat.remove(); // mat.remove();
// If a language is found with the given name, then add a new language layer // // If a language is found with the given name, then add a new language layer
// to the highlighted document. // // to the highlighted document.
if let (Some(injection_capture), Some(content_node)) = // if let (Some(injection_capture), Some(content_node)) =
(injection_capture, content_node) // (injection_capture, content_node)
{ // {
if let Some(config) = (injection_callback)(&injection_capture) { // if let Some(config) = (injection_callback)(&injection_capture) {
let ranges = // let ranges =
intersect_ranges(&layer.ranges, &[content_node], included_children); // intersect_ranges(&layer.ranges, &[content_node], included_children);
if !ranges.is_empty() { // if !ranges.is_empty() {
if content_node.start_byte() < last_injection_end { // if content_node.start_byte() < last_injection_end {
continue; // continue;
} // }
last_injection_end = content_node.end_byte(); // last_injection_end = content_node.end_byte();
injections.push((config, ranges)); // injections.push((config, ranges));
} // }
} // }
} // }
} // }
for (lang_name, content_nodes, included_children) in combined_injections { // for (lang_name, content_nodes, included_children) in combined_injections {
if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) { // if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) {
if let Some(config) = (injection_callback)(&lang_name) { // if let Some(config) = (injection_callback)(&lang_name) {
let ranges = // let ranges =
intersect_ranges(&layer.ranges, &content_nodes, included_children); // intersect_ranges(&layer.ranges, &content_nodes, included_children);
if !ranges.is_empty() { // if !ranges.is_empty() {
injections.push((config, ranges)); // injections.push((config, ranges));
} // }
} // }
} // }
} // }
let depth = layer.depth + 1; // let depth = layer.depth + 1;
// TODO: can't inline this since matches borrows self.layers // // TODO: can't inline this since matches borrows self.layers
for (config, ranges) in injections { // for (config, ranges) in injections {
let parent = Some(layer_id); // let parent = Some(layer_id);
let new_layer = LanguageLayer { // let new_layer = LanguageLayer {
tree: None, // parse_tree: None,
config, // config,
depth, // depth,
ranges, // ranges,
flags: LayerUpdateFlags::empty(), // flags: LayerUpdateFlags::empty(),
parent: None, // parent: None,
}; // };
// Find an identical existing layer // // Find an identical existing layer
let layer = layers_table // let layer = layers_table
.get(layers_hasher.hash_one(&new_layer), |&it| { // .get(layers_hasher.hash_one(&new_layer), |&it| {
self.layers[it] == new_layer // self.layers[it] == new_layer
}) // })
.copied(); // .copied();
// ...or insert a new one. // // ...or insert a new one.
let layer_id = layer.unwrap_or_else(|| self.layers.insert(new_layer)); // let layer_id = layer.unwrap_or_else(|| self.layers.insert(new_layer));
self.layers[layer_id].parent = parent; // self.layers[layer_id].parent = parent;
queue.push_back(layer_id); // queue.push_back(layer_id);
} // }
// TODO: pre-process local scopes at this time, rather than highlight? // // TODO: pre-process local scopes at this time, rather than highlight?
// would solve problems with locals not working across boundaries // // would solve problems with locals not working across boundaries
} // }
// Return the cursor back in the pool. // // Return the cursor back in the pool.
ts_parser.cursors.push(cursor); // ts_parser.cursors.push(cursor);
// Reset all `LayerUpdateFlags` and remove all untouched layers // // Reset all `LayerUpdateFlags` and remove all untouched layers
self.layers.retain(|_, layer| { // self.layers.retain(|_, layer| {
replace(&mut layer.flags, LayerUpdateFlags::empty()) // replace(&mut layer.flags, LayerUpdateFlags::empty())
.contains(LayerUpdateFlags::TOUCHED) // .contains(LayerUpdateFlags::TOUCHED)
}); // });
Ok(()) // Ok(())
}) // })
} // }
} // }
/// Compute the ranges that should be included when parsing an injection. // /// Compute the ranges that should be included when parsing an injection.
/// This takes into account three things: // /// This takes into account three things:
/// * `parent_ranges` - The ranges must all fall within the *current* layer's ranges. // /// * `parent_ranges` - The ranges must all fall within the *current* layer's ranges.
/// * `nodes` - Every injection takes place within a set of nodes. The injection ranges // /// * `nodes` - Every injection takes place within a set of nodes. The injection ranges
/// are the ranges of those nodes. // /// are the ranges of those nodes.
/// * `includes_children` - For some injections, the content nodes' children should be // /// * `includes_children` - For some injections, the content nodes' children should be
/// excluded from the nested document, so that only the content nodes' *own* content // /// excluded from the nested document, so that only the content nodes' *own* content
/// is reparsed. For other injections, the content nodes' entire ranges should be // /// is reparsed. For other injections, the content nodes' entire ranges should be
/// reparsed, including the ranges of their children. // /// reparsed, including the ranges of their children.
fn intersect_ranges( // fn intersect_ranges(
parent_ranges: &[Range], // parent_ranges: &[Range],
nodes: &[Node], // nodes: &[Node],
included_children: IncludedChildren, // included_children: IncludedChildren,
) -> Vec<Range> { // ) -> Vec<Range> {
let mut cursor = nodes[0].walk(); // let mut cursor = nodes[0].walk();
let mut result = Vec::new(); // let mut result = Vec::new();
let mut parent_range_iter = parent_ranges.iter(); // let mut parent_range_iter = parent_ranges.iter();
let mut parent_range = parent_range_iter // let mut parent_range = parent_range_iter
.next() // .next()
.expect("Layers should only be constructed with non-empty ranges vectors"); // .expect("Layers should only be constructed with non-empty ranges vectors");
for node in nodes.iter() { // for node in nodes.iter() {
let mut preceding_range = Range { // let mut preceding_range = Range {
start_byte: 0, // start_byte: 0,
start_point: Point::new(0, 0), // start_point: Point::new(0, 0),
end_byte: node.start_byte(), // end_byte: node.start_byte(),
end_point: node.start_position(), // end_point: node.start_position(),
}; // };
let following_range = Range { // let following_range = Range {
start_byte: node.end_byte(), // start_byte: node.end_byte(),
start_point: node.end_position(), // start_point: node.end_position(),
end_byte: usize::MAX, // end_byte: usize::MAX,
end_point: Point::new(usize::MAX, usize::MAX), // end_point: Point::new(usize::MAX, usize::MAX),
}; // };
for excluded_range in node // for excluded_range in node
.children(&mut cursor) // .children(&mut cursor)
.filter_map(|child| match included_children { // .filter_map(|child| match included_children {
IncludedChildren::None => Some(child.range()), // IncludedChildren::None => Some(child.range()),
IncludedChildren::All => None, // IncludedChildren::All => None,
IncludedChildren::Unnamed => { // IncludedChildren::Unnamed => {
if child.is_named() { // if child.is_named() {
Some(child.range()) // Some(child.range())
} else { // } else {
None // None
} // }
} // }
}) // })
.chain([following_range].iter().cloned()) // .chain([following_range].iter().cloned())
{ // {
let mut range = Range { // let mut range = Range {
start_byte: preceding_range.end_byte, // start_byte: preceding_range.end_byte,
start_point: preceding_range.end_point, // start_point: preceding_range.end_point,
end_byte: excluded_range.start_byte, // end_byte: excluded_range.start_byte,
end_point: excluded_range.start_point, // end_point: excluded_range.start_point,
}; // };
preceding_range = excluded_range; // preceding_range = excluded_range;
if range.end_byte < parent_range.start_byte { // if range.end_byte < parent_range.start_byte {
continue; // continue;
} // }
while parent_range.start_byte <= range.end_byte { // while parent_range.start_byte <= range.end_byte {
if parent_range.end_byte > range.start_byte { // if parent_range.end_byte > range.start_byte {
if range.start_byte < parent_range.start_byte { // if range.start_byte < parent_range.start_byte {
range.start_byte = parent_range.start_byte; // range.start_byte = parent_range.start_byte;
range.start_point = parent_range.start_point; // range.start_point = parent_range.start_point;
} // }
if parent_range.end_byte < range.end_byte { // if parent_range.end_byte < range.end_byte {
if range.start_byte < parent_range.end_byte { // if range.start_byte < parent_range.end_byte {
result.push(Range { // result.push(Range {
start_byte: range.start_byte, // start_byte: range.start_byte,
start_point: range.start_point, // start_point: range.start_point,
end_byte: parent_range.end_byte, // end_byte: parent_range.end_byte,
end_point: parent_range.end_point, // end_point: parent_range.end_point,
}); // });
} // }
range.start_byte = parent_range.end_byte; // range.start_byte = parent_range.end_byte;
range.start_point = parent_range.end_point; // range.start_point = parent_range.end_point;
} else { // } else {
if range.start_byte < range.end_byte { // if range.start_byte < range.end_byte {
result.push(range); // result.push(range);
} // }
break; // break;
} // }
} // }
if let Some(next_range) = parent_range_iter.next() { // if let Some(next_range) = parent_range_iter.next() {
parent_range = next_range; // parent_range = next_range;
} else { // } else {
return result; // return result;
} // }
} // }
} // }
} // }
result // result
} // }
impl LanguageLayer { // impl LanguageLayer {
fn parse(&mut self, parser: &mut Parser, source: RopeSlice) -> Result<(), Error> { // fn parse(&mut self, parser: &mut Parser, source: RopeSlice) -> Result<(), Error> {
parser // parser
.set_included_ranges(&self.ranges) // .set_included_ranges(&self.ranges)
.map_err(|_| Error::InvalidRanges)?; // .map_err(|_| Error::InvalidRanges)?;
parser // parser
.set_language(&self.config.language) // .set_language(&self.config.language)
.map_err(|_| Error::InvalidLanguage)?; // .map_err(|_| Error::InvalidLanguage)?;
// unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) }; // // unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };
let tree = parser // let tree = parser
.parse_with( // .parse_with(
&mut |byte, _| { // &mut |byte, _| {
if byte <= source.len_bytes() { // if byte <= source.len_bytes() {
let (chunk, start_byte, _, _) = source.chunk_at_byte(byte); // let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);
&chunk.as_bytes()[byte - start_byte..] // &chunk.as_bytes()[byte - start_byte..]
} else { // } else {
// out of range // // out of range
&[] // &[]
} // }
}, // },
self.tree.as_ref(), // self.parse_tree.as_ref(),
) // )
.ok_or(Error::Cancelled)?; // .ok_or(Error::Cancelled)?;
// unsafe { ts_parser.parser.set_cancellation_flag(None) }; // // unsafe { ts_parser.parser.set_cancellation_flag(None) };
self.tree = Some(tree); // self.parse_tree = Some(tree);
Ok(()) // Ok(())
} // }
} // }

236
helix-syntax/src/query_iter.rs Normal file
View File

@ -0,0 +1,236 @@
use core::slice;
use std::iter::Peekable;
use std::mem::replace;
use hashbrown::HashMap;
use ropey::RopeSlice;
use crate::tree_sitter::{
Capture, InactiveQueryCursor, Query, QueryCursor, RopeTsInput, SyntaxTreeNode,
};
use crate::{Injection, LayerId, Range, Syntax};
#[derive(Clone)]
pub struct MatchedNode {
pub capture: Capture,
pub byte_range: Range,
}
struct LayerQueryIter<'a> {
cursor: QueryCursor<'a, 'a, RopeTsInput<'a>>,
peeked: Option<MatchedNode>,
}
impl<'a> LayerQueryIter<'a> {
fn peek(&mut self) -> Option<&MatchedNode> {
if self.peeked.is_none() {
let (query_match, node_idx) = self.cursor.next_matched_node()?;
let matched_node = query_match.matched_node(node_idx);
self.peeked = Some(MatchedNode {
capture: matched_node.capture,
byte_range: matched_node.syntax_node.byte_range(),
});
}
self.peeked.as_ref()
}
fn consume(&mut self) -> MatchedNode {
self.peeked.take().unwrap()
}
}
struct ActiveLayer<'a, S> {
state: S,
query_iter: LayerQueryIter<'a>,
injections: Peekable<slice::Iter<'a, Injection>>,
}
// data only needed when entering and exiting injections
// seperate struck to keep the QueryIter reasonably small
struct QueryIterLayerManager<'a, S> {
query: &'a Query,
node: SyntaxTreeNode<'a>,
src: RopeSlice<'a>,
syntax: &'a Syntax,
active_layers: HashMap<LayerId, Box<ActiveLayer<'a, S>>>,
active_injections: Vec<Injection>,
}
impl<'a, S: Default> QueryIterLayerManager<'a, S> {
fn init_layer(&mut self, injection: Injection) -> Box<ActiveLayer<'a, S>> {
self.active_layers
.remove(&injection.layer)
.unwrap_or_else(|| {
let layer = &self.syntax.layers[injection.layer];
let injection_start = layer
.injections
.partition_point(|child| child.byte_range.start < injection.byte_range.start);
let cursor = InactiveQueryCursor::new().execute_query(
self.query,
&self.node,
RopeTsInput::new(self.src),
);
Box::new(ActiveLayer {
state: S::default(),
query_iter: LayerQueryIter {
cursor,
peeked: None,
},
injections: layer.injections[injection_start..].iter().peekable(),
})
})
}
}
pub struct QueryIter<'a, LayerState: Default = ()> {
layer_manager: Box<QueryIterLayerManager<'a, LayerState>>,
current_layer: Box<ActiveLayer<'a, LayerState>>,
current_injection: Injection,
}
impl<'a, LayerState: Default> QueryIter<'a, LayerState> {
pub fn new(syntax: &'a Syntax, src: RopeSlice<'a>, query: &'a Query) -> Self {
Self::at(syntax, src, query, syntax.tree().root_node(), syntax.root)
}
pub fn at(
syntax: &'a Syntax,
src: RopeSlice<'a>,
query: &'a Query,
node: SyntaxTreeNode<'a>,
layer: LayerId,
) -> Self {
// create fake injection for query root
let injection = Injection {
byte_range: node.byte_range(),
layer,
};
let mut layer_manager = Box::new(QueryIterLayerManager {
query,
node,
src,
syntax,
// TODO: reuse allocations with an allocation pool
active_layers: HashMap::with_capacity(8),
active_injections: Vec::with_capacity(8),
});
Self {
current_layer: layer_manager.init_layer(injection),
current_injection: injection,
layer_manager,
}
}
pub fn current_layer_state(&mut self) -> &mut LayerState {
&mut self.current_layer.state
}
pub fn layer_state(&mut self, layer: LayerId) -> &mut LayerState {
if layer == self.current_injection.layer {
self.current_layer_state()
} else {
&mut self
.layer_manager
.active_layers
.get_mut(&layer)
.unwrap()
.state
}
}
fn enter_injection(&mut self, injection: Injection) {
let active_layer = self.layer_manager.init_layer(injection);
let old_injection = replace(&mut self.current_injection, injection);
let old_layer = replace(&mut self.current_layer, active_layer);
self.layer_manager
.active_layers
.insert(old_injection.layer, old_layer);
self.layer_manager.active_injections.push(old_injection);
}
fn exit_injection(&mut self) -> Option<(Injection, Option<LayerState>)> {
let injection = replace(
&mut self.current_injection,
self.layer_manager.active_injections.pop()?,
);
let layer = replace(
&mut self.current_layer,
self.layer_manager
.active_layers
.remove(&self.current_injection.layer)?,
);
let layer_unfinished = layer.query_iter.peeked.is_some();
if layer_unfinished {
self.layer_manager
.active_layers
.insert(injection.layer, layer)
.unwrap();
Some((injection, None))
} else {
Some((injection, Some(layer.state)))
}
}
}
impl<'a, S: Default> Iterator for QueryIter<'a, S> {
type Item = QueryIterEvent<S>;
fn next(&mut self) -> Option<QueryIterEvent<S>> {
loop {
let next_injection = self.current_layer.injections.peek().filter(|injection| {
injection.byte_range.start < self.current_injection.byte_range.end
});
let next_match = self.current_layer.query_iter.peek().filter(|matched_node| {
matched_node.byte_range.start < self.current_injection.byte_range.end
});
match (next_match, next_injection) {
(None, None) => {
return self.exit_injection().map(|(injection, state)| {
QueryIterEvent::ExitInjection { injection, state }
});
}
(Some(_), None) => {
// consume match
let matched_node = self.current_layer.query_iter.consume();
return Some(QueryIterEvent::Match(matched_node));
}
(Some(matched_node), Some(injection))
if matched_node.byte_range.start <= injection.byte_range.end =>
{
// consume match
let matched_node = self.current_layer.query_iter.consume();
// ignore nodes that are overlapped by the injection
if matched_node.byte_range.start <= injection.byte_range.start {
return Some(QueryIterEvent::Match(matched_node));
}
}
(Some(_), Some(_)) | (None, Some(_)) => {
// consume injection
let injection = self.current_layer.injections.next().unwrap();
self.enter_injection(injection.clone());
return Some(QueryIterEvent::EnterInjection(injection.clone()));
}
}
}
}
}
pub enum QueryIterEvent<State = ()> {
EnterInjection(Injection),
Match(MatchedNode),
ExitInjection {
injection: Injection,
state: Option<State>,
},
}
impl<S> QueryIterEvent<S> {
pub fn start(&self) -> u32 {
match self {
QueryIterEvent::EnterInjection(injection) => injection.byte_range.start as u32,
QueryIterEvent::Match(mat) => mat.byte_range.start as u32,
QueryIterEvent::ExitInjection { injection, .. } => injection.byte_range.start as u32,
}
}
}

28
helix-syntax/src/ropey.rs
View File

@ -1,29 +1 @@
// glue code for using TS with ropey, this should be put behind a feature flag
// in the future (and potentially be partially removed)
use ropey::RopeSlice;
use tree_sitter::{Node, TextProvider};
// Adapter to convert rope chunks to bytes
pub struct ChunksBytes<'a> {
chunks: ropey::iter::Chunks<'a>,
}
impl<'a> Iterator for ChunksBytes<'a> {
type Item = &'a [u8];
fn next(&mut self) -> Option<Self::Item> {
self.chunks.next().map(str::as_bytes)
}
}
pub struct RopeProvider<'a>(pub RopeSlice<'a>);
impl<'a> TextProvider<&'a [u8]> for RopeProvider<'a> {
type I = ChunksBytes<'a>;
fn text(&mut self, node: Node) -> Self::I {
let fragment = self.0.byte_slice(node.start_byte()..node.end_byte());
ChunksBytes {
chunks: fragment.chunks(),
}
}
}

93
helix-syntax/src/text_object.rs Normal file
View File

@ -0,0 +1,93 @@
// TODO: rework using query iter
use std::iter;
use ropey::RopeSlice;
use crate::tree_sitter::{InactiveQueryCursor, Query, RopeTsInput, SyntaxTreeNode};
use crate::TREE_SITTER_MATCH_LIMIT;
#[derive(Debug)]
pub enum CapturedNode<'a> {
Single(SyntaxTreeNode<'a>),
/// Guaranteed to be not empty
Grouped(Vec<SyntaxTreeNode<'a>>),
}
impl<'a> CapturedNode<'a> {
pub fn start_byte(&self) -> usize {
match self {
Self::Single(n) => n.start_byte(),
Self::Grouped(ns) => ns[0].start_byte(),
}
}
pub fn end_byte(&self) -> usize {
match self {
Self::Single(n) => n.end_byte(),
Self::Grouped(ns) => ns.last().unwrap().end_byte(),
}
}
}
#[derive(Debug)]
pub struct TextObjectQuery {
pub query: Query,
}
impl TextObjectQuery {
/// Run the query on the given node and return sub nodes which match given
/// capture ("function.inside", "class.around", etc).
///
/// Captures may contain multiple nodes by using quantifiers (+, *, etc),
/// and support for this is partial and could use improvement.
///
/// ```query
/// (comment)+ @capture
///
/// ; OR
/// (
/// (comment)*
/// .
/// (function)
/// ) @capture
/// ```
pub fn capture_nodes<'a>(
&'a self,
capture_name: &str,
node: SyntaxTreeNode<'a>,
slice: RopeSlice<'a>,
cursor: InactiveQueryCursor,
) -> Option<impl Iterator<Item = CapturedNode<'a>>> {
self.capture_nodes_any(&[capture_name], node, slice, cursor)
}
/// Find the first capture that exists out of all given `capture_names`
/// and return sub nodes that match this capture.
pub fn capture_nodes_any<'a>(
&'a self,
capture_names: &[&str],
node: SyntaxTreeNode<'a>,
slice: RopeSlice<'a>,
mut cursor: InactiveQueryCursor,
) -> Option<impl Iterator<Item = CapturedNode<'a>>> {
let capture = capture_names
.iter()
.find_map(|cap| self.query.get_capture(cap))?;
cursor.set_match_limit(TREE_SITTER_MATCH_LIMIT);
let mut cursor = cursor.execute_query(&self.query, &node, RopeTsInput::new(slice));
let capture_node = iter::from_fn(move || {
let (mat, _) = cursor.next_matched_node()?;
Some(mat.nodes_for_capture(capture).cloned().collect())
})
.filter_map(move |nodes: Vec<_>| {
if nodes.len() > 1 {
Some(CapturedNode::Grouped(nodes))
} else {
nodes.into_iter().map(CapturedNode::Single).next()
}
});
Some(capture_node)
}
}

10
helix-syntax/src/tree_sitter.rs
View File

@ -1,6 +1,6 @@
mod grammar; mod grammar;
mod parser; mod parser;
mod query; pub mod query;
mod query_cursor; mod query_cursor;
mod query_match; mod query_match;
mod ropey; mod ropey;
@ -11,21 +11,21 @@
pub use grammar::Grammar; pub use grammar::Grammar;
pub use parser::{Parser, ParserInputRaw}; pub use parser::{Parser, ParserInputRaw};
pub use query::{Capture, ParserErrorLocation, Pattern, Query, QueryStr}; pub use query::{Capture, Pattern, Query, QueryStr};
pub use query_cursor::{InactiveQueryCursor, MatchedNode, MatchedNodeIdx, QueryCursor, QueryMatch}; pub use query_cursor::{InactiveQueryCursor, MatchedNode, MatchedNodeIdx, QueryCursor, QueryMatch};
pub use ropey::RopeTsInput; pub use ropey::RopeTsInput;
pub use syntax_tree::{InputEdit, SyntaxTree}; pub use syntax_tree::{InputEdit, SyntaxTree};
pub use syntax_tree_node::SyntaxTreeNode; pub use syntax_tree_node::SyntaxTreeNode;
#[repr(C)] #[repr(C)]
#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Point { pub struct Point {
pub row: u32, pub row: u32,
pub column: u32, pub col: u32,
} }
#[repr(C)] #[repr(C)]
#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Range { pub struct Range {
pub start_point: Point, pub start_point: Point,
pub end_point: Point, pub end_point: Point,

101
helix-syntax/src/tree_sitter/query.rs
View File

@ -5,28 +5,64 @@
use std::{slice, str}; use std::{slice, str};
use crate::tree_sitter::query::predicate::{InvalidPredicateError, Predicate, TextPredicate}; use crate::tree_sitter::query::predicate::{InvalidPredicateError, Predicate, TextPredicate};
use crate::tree_sitter::query::property::QueryProperty;
use crate::tree_sitter::Grammar; use crate::tree_sitter::Grammar;
mod predicate; mod predicate;
mod property; mod property;
pub enum UserPredicate<'a> {
IsPropertySet {
negate: bool,
key: &'a str,
val: Option<&'a str>,
},
SetProperty {
key: &'a str,
val: Option<&'a str>,
},
Other(Predicate<'a>),
}
impl Display for UserPredicate<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match *self {
UserPredicate::IsPropertySet { negate, key, val } => {
let predicate = if negate { "is-not?" } else { "is?" };
write!(f, " ({predicate} {key} {})", val.unwrap_or(""))
}
UserPredicate::SetProperty { key, val } => {
write!(f, "(set! {key} {})", val.unwrap_or(""))
}
UserPredicate::Other(ref predicate) => {
write!(f, "{}", predicate.name())
}
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Pattern(pub(crate) u32); pub struct Pattern(pub(crate) u32);
pub enum QueryData {} impl Pattern {
pub const SENTINEL: Pattern = Pattern(u32::MAX);
pub(super) struct PatternData { pub fn idx(&self) -> usize {
text_predicates: Range<u32>, self.0 as usize
properties: Range<u32>, }
} }
pub enum QueryData {}
#[derive(Debug)]
pub(super) struct PatternData {
text_predicates: Range<u32>,
}
#[derive(Debug)]
pub struct Query { pub struct Query {
pub(crate) raw: NonNull<QueryData>, pub(crate) raw: NonNull<QueryData>,
num_captures: u32, num_captures: u32,
num_strings: u32, num_strings: u32,
text_predicates: Vec<TextPredicate>, text_predicates: Vec<TextPredicate>,
properties: Vec<QueryProperty>,
patterns: Box<[PatternData]>, patterns: Box<[PatternData]>,
} }
@ -41,7 +77,7 @@ pub fn new(
grammar: Grammar, grammar: Grammar,
source: &str, source: &str,
path: impl AsRef<Path>, path: impl AsRef<Path>,
mut custom_predicate: impl FnMut(Pattern, Predicate) -> Result<(), InvalidPredicateError>, mut custom_predicate: impl FnMut(Pattern, UserPredicate) -> Result<(), InvalidPredicateError>,
) -> Result<Self, ParseError> { ) -> Result<Self, ParseError> {
assert!( assert!(
source.len() <= i32::MAX as usize, source.len() <= i32::MAX as usize,
@ -136,7 +172,6 @@ pub fn new(
num_captures, num_captures,
num_strings, num_strings,
text_predicates: Vec::new(), text_predicates: Vec::new(),
properties: Vec::new(),
patterns: Box::default(), patterns: Box::default(),
}; };
let patterns: Result<_, ParseError> = (0..num_patterns) let patterns: Result<_, ParseError> = (0..num_patterns)
@ -190,15 +225,54 @@ pub fn capture_name(&self, capture_idx: Capture) -> &str {
} }
} }
pub fn pattern_properies(&self, pattern_idx: Pattern) -> &[QueryProperty] { #[inline]
let range = self.patterns[pattern_idx.0 as usize].properties.clone(); pub fn captures(&self) -> impl ExactSizeIterator<Item = (Capture, &str)> {
&self.properties[range.start as usize..range.end as usize] (0..self.num_captures).map(|cap| (Capture(cap), self.capture_name(Capture(cap))))
}
#[inline]
pub fn num_captures(&self) -> u32 {
self.num_captures
}
#[inline]
pub fn get_capture(&self, capture_name: &str) -> Option<Capture> {
for capture in 0..self.num_captures {
if capture_name == self.capture_name(Capture(capture)) {
return Some(Capture(capture));
}
}
None
} }
pub(crate) fn pattern_text_predicates(&self, pattern_idx: u16) -> &[TextPredicate] { pub(crate) fn pattern_text_predicates(&self, pattern_idx: u16) -> &[TextPredicate] {
let range = self.patterns[pattern_idx as usize].text_predicates.clone(); let range = self.patterns[pattern_idx as usize].text_predicates.clone();
&self.text_predicates[range.start as usize..range.end as usize] &self.text_predicates[range.start as usize..range.end as usize]
} }
/// Get the byte offset where the given pattern starts in the query's
/// source.
#[doc(alias = "ts_query_start_byte_for_pattern")]
#[must_use]
pub fn start_byte_for_pattern(&self, pattern: Pattern) -> usize {
assert!(
pattern.0 < self.text_predicates.len() as u32,
"Pattern index is {pattern_index} but the pattern count is {}",
self.text_predicates.len(),
);
unsafe { ts_query_start_byte_for_pattern(self.raw, pattern.0) as usize }
}
/// Get the number of patterns in the query.
#[must_use]
pub fn pattern_count(&self) -> usize {
unsafe { ts_query_pattern_count(self.raw) as usize }
}
/// Get the number of patterns in the query.
#[must_use]
pub fn patterns(&self) -> impl ExactSizeIterator<Item = Pattern> {
(0..self.pattern_count() as u32).map(Pattern)
}
} }
impl Drop for Query { impl Drop for Query {
@ -215,6 +289,9 @@ impl Capture {
pub fn name(self, query: &Query) -> &str { pub fn name(self, query: &Query) -> &str {
query.capture_name(self) query.capture_name(self)
} }
pub fn idx(self) -> usize {
self.0 as usize
}
} }
/// A reference to a string stroed in a query /// A reference to a string stroed in a query

57
helix-syntax/src/tree_sitter/query/predicate.rs
View File

@ -5,7 +5,9 @@
use std::{fmt, slice}; use std::{fmt, slice};
use crate::tree_sitter::query::property::QueryProperty; use crate::tree_sitter::query::property::QueryProperty;
use crate::tree_sitter::query::{Capture, Pattern, PatternData, Query, QueryData, QueryStr}; use crate::tree_sitter::query::{
Capture, Pattern, PatternData, Query, QueryData, QueryStr, UserPredicate,
};
use crate::tree_sitter::query_cursor::MatchedNode; use crate::tree_sitter::query_cursor::MatchedNode;
use crate::tree_sitter::TsInput; use crate::tree_sitter::TsInput;
@ -34,6 +36,7 @@ pub(super) enum TextPredicateKind {
AnyString(Box<[QueryStr]>), AnyString(Box<[QueryStr]>),
} }
#[derive(Debug)]
pub(crate) struct TextPredicate { pub(crate) struct TextPredicate {
capture: Capture, capture: Capture,
kind: TextPredicateKind, kind: TextPredicateKind,
@ -161,10 +164,9 @@ impl Query {
pub(super) fn parse_pattern_predicates( pub(super) fn parse_pattern_predicates(
&mut self, &mut self,
pattern: Pattern, pattern: Pattern,
mut custom_predicate: impl FnMut(Pattern, Predicate) -> Result<(), InvalidPredicateError>, mut custom_predicate: impl FnMut(Pattern, UserPredicate) -> Result<(), InvalidPredicateError>,
) -> Result<PatternData, InvalidPredicateError> { ) -> Result<PatternData, InvalidPredicateError> {
let text_predicate_start = self.text_predicates.len() as u32; let text_predicate_start = self.text_predicates.len() as u32;
let property_start = self.properties.len() as u32;
let predicate_steps = unsafe { let predicate_steps = unsafe {
let mut len = 0u32; let mut len = 0u32;
@ -203,7 +205,7 @@ pub(super) fn parse_pattern_predicates(
"match?" | "not-match?" | "any-match?" | "any-not-match?" => { "match?" | "not-match?" | "any-match?" | "any-not-match?" => {
predicate.check_arg_count(2)?; predicate.check_arg_count(2)?;
let capture_idx = predicate.capture_arg(0)?; let capture_idx = predicate.capture_arg(0)?;
let regex = predicate.str_arg(1)?.get(self); let regex = predicate.query_str_arg(1)?.get(self);
let negated = matches!(predicate.name(), "not-match?" | "any-not-match?"); let negated = matches!(predicate.name(), "not-match?" | "any-not-match?");
let match_all = matches!(predicate.name(), "match?" | "not-match?"); let match_all = matches!(predicate.name(), "match?" | "not-match?");
@ -219,14 +221,34 @@ pub(super) fn parse_pattern_predicates(
}); });
} }
"set!" => self.properties.push(QueryProperty::parse(&predicate)?), "set!" => {
let property = QueryProperty::parse(&predicate)?;
custom_predicate(
pattern,
UserPredicate::SetProperty {
key: property.key.get(&self),
val: property.val.map(|val| val.get(&self)),
},
)?
}
"is-not?" | "is?" => {
let property = QueryProperty::parse(&predicate)?;
custom_predicate(
pattern,
UserPredicate::IsPropertySet {
negate: predicate.name() == "is-not?",
key: property.key.get(&self),
val: property.val.map(|val| val.get(&self)),
},
)?
}
"any-of?" | "not-any-of?" => { "any-of?" | "not-any-of?" => {
predicate.check_min_arg_count(1)?; predicate.check_min_arg_count(1)?;
let capture = predicate.capture_arg(0)?; let capture = predicate.capture_arg(0)?;
let negated = predicate.name() == "not-any-of?"; let negated = predicate.name() == "not-any-of?";
let values: Result<_, InvalidPredicateError> = (1..predicate.num_args()) let values: Result<_, InvalidPredicateError> = (1..predicate.num_args())
.map(|i| predicate.str_arg(i)) .map(|i| predicate.query_str_arg(i))
.collect(); .collect();
self.text_predicates.push(TextPredicate { self.text_predicates.push(TextPredicate {
capture, capture,
@ -239,12 +261,11 @@ pub(super) fn parse_pattern_predicates(
// is and is-not are better handeled as custom predicates since interpreting is context dependent // is and is-not are better handeled as custom predicates since interpreting is context dependent
// "is?" => property_predicates.push((QueryProperty::parse(&predicate), false)), // "is?" => property_predicates.push((QueryProperty::parse(&predicate), false)),
// "is-not?" => property_predicates.push((QueryProperty::parse(&predicate), true)), // "is-not?" => property_predicates.push((QueryProperty::parse(&predicate), true)),
_ => custom_predicate(pattern, predicate)?, _ => custom_predicate(pattern, UserPredicate::Other(predicate))?,
} }
} }
Ok(PatternData { Ok(PatternData {
text_predicates: text_predicate_start..self.text_predicates.len() as u32, text_predicates: text_predicate_start..self.text_predicates.len() as u32,
properties: property_start..self.properties.len() as u32,
}) })
} }
} }
@ -312,7 +333,7 @@ pub fn check_max_arg_count(&self, n: usize) -> Result<(), InvalidPredicateError>
Ok(()) Ok(())
} }
pub fn str_arg(&self, i: usize) -> Result<QueryStr, InvalidPredicateError> { pub fn query_str_arg(&self, i: usize) -> Result<QueryStr, InvalidPredicateError> {
match self.arg(i) { match self.arg(i) {
PredicateArg::String(str) => Ok(str), PredicateArg::String(str) => Ok(str),
PredicateArg::Capture(capture) => bail!( PredicateArg::Capture(capture) => bail!(
@ -323,6 +344,10 @@ pub fn str_arg(&self, i: usize) -> Result<QueryStr, InvalidPredicateError> {
} }
} }
pub fn str_arg(&self, i: usize) -> Result<&str, InvalidPredicateError> {
Ok(self.query_str_arg(i)?.get(self.query))
}
pub fn num_args(&self) -> usize { pub fn num_args(&self) -> usize {
self.args.len() self.args.len()
} }
@ -352,6 +377,20 @@ pub struct InvalidPredicateError {
pub(super) msg: Box<str>, pub(super) msg: Box<str>,
} }
impl From<String> for InvalidPredicateError {
fn from(value: String) -> Self {
InvalidPredicateError {
msg: value.into_boxed_str(),
}
}
}
impl<'a> From<&'a str> for InvalidPredicateError {
fn from(value: &'a str) -> Self {
InvalidPredicateError { msg: value.into() }
}
}
impl fmt::Display for InvalidPredicateError { impl fmt::Display for InvalidPredicateError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.msg) f.write_str(&self.msg)

5
helix-syntax/src/tree_sitter/query/property.rs
View File

@ -1,6 +1,7 @@
use crate::tree_sitter::query::predicate::{InvalidPredicateError, Predicate}; use crate::tree_sitter::query::predicate::{InvalidPredicateError, Predicate};
use crate::tree_sitter::query::QueryStr; use crate::tree_sitter::query::QueryStr;
#[derive(Debug)]
pub struct QueryProperty { pub struct QueryProperty {
pub key: QueryStr, pub key: QueryStr,
pub val: Option<QueryStr>, pub val: Option<QueryStr>,
@ -10,9 +11,9 @@ impl QueryProperty {
pub fn parse(predicate: &Predicate) -> Result<Self, InvalidPredicateError> { pub fn parse(predicate: &Predicate) -> Result<Self, InvalidPredicateError> {
predicate.check_min_arg_count(1)?; predicate.check_min_arg_count(1)?;
predicate.check_max_arg_count(2)?; predicate.check_max_arg_count(2)?;
let key = predicate.str_arg(0)?; let key = predicate.query_str_arg(0)?;
let val = (predicate.num_args() == 1) let val = (predicate.num_args() == 1)
.then(|| predicate.str_arg(1)) .then(|| predicate.query_str_arg(1))
.transpose()?; .transpose()?;
Ok(QueryProperty { key, val }) Ok(QueryProperty { key, val })
} }

32
helix-syntax/src/tree_sitter/query_cursor.rs
View File

@ -1,4 +1,5 @@
use core::slice; use core::slice;
use std::cell::UnsafeCell;
use std::marker::PhantomData; use std::marker::PhantomData;
use std::mem::replace; use std::mem::replace;
use std::ops::Range; use std::ops::Range;
@ -10,6 +11,15 @@
enum QueryCursorData {} enum QueryCursorData {}
thread_local! {
static CURSOR_CACHE: UnsafeCell<Vec<InactiveQueryCursor>> = UnsafeCell::new(Vec::with_capacity(8));
}
/// SAFETY: must not call itself recuresively
unsafe fn with_cache<T>(f: impl FnOnce(&mut Vec<InactiveQueryCursor>) -> T) -> T {
CURSOR_CACHE.with(|cache| f(&mut *cache.get()))
}
pub struct QueryCursor<'a, 'tree, I: TsInput> { pub struct QueryCursor<'a, 'tree, I: TsInput> {
query: &'a Query, query: &'a Query,
ptr: *mut QueryCursorData, ptr: *mut QueryCursorData,
@ -115,8 +125,8 @@ impl<I: TsInput> Drop for QueryCursor<'_, '_, I> {
fn drop(&mut self) { fn drop(&mut self) {
// we allow moving the cursor data out so we need the null check here // we allow moving the cursor data out so we need the null check here
// would be cleaner with a subtype but doesn't really matter at the end of the day // would be cleaner with a subtype but doesn't really matter at the end of the day
if !self.ptr.is_null() { if let Some(ptr) = NonNull::new(self.ptr) {
unsafe { ts_query_cursor_delete(self.ptr) } unsafe { with_cache(|cache| cache.push(InactiveQueryCursor { ptr })) }
} }
} }
} }
@ -128,8 +138,12 @@ pub struct InactiveQueryCursor {
impl InactiveQueryCursor { impl InactiveQueryCursor {
pub fn new() -> Self { pub fn new() -> Self {
InactiveQueryCursor { unsafe {
ptr: unsafe { NonNull::new_unchecked(ts_query_cursor_new()) }, with_cache(|cache| {
cache.pop().unwrap_or_else(|| InactiveQueryCursor {
ptr: NonNull::new_unchecked(ts_query_cursor_new()),
})
})
} }
} }
@ -208,6 +222,16 @@ pub fn matched_nodes(&self) -> impl Iterator<Item = &MatchedNode<'tree>> {
self.matched_nodes.iter() self.matched_nodes.iter()
} }
pub fn nodes_for_capture(
&self,
capture: Capture,
) -> impl Iterator<Item = &SyntaxTreeNode<'tree>> {
self.matched_nodes
.iter()
.filter(move |mat| mat.capture == capture)
.map(|mat| &mat.syntax_node)
}
pub fn matched_node(&self, i: MatchedNodeIdx) -> &MatchedNode { pub fn matched_node(&self, i: MatchedNodeIdx) -> &MatchedNode {
&self.matched_nodes[i as usize] &self.matched_nodes[i as usize]
} }