Handle single-line comment prefixes in :reflow.

I changed :reflow to use the DocumentFormatter object instead of the
textwrap crate. This allows using the same logic for soft wrap as
for :reflow. Because the logic is the same as for soft wrap, we end up
preserving all existing newlines, so it's more like "wrap" than reflow,
but I think this behavior makes sense anyway to avoid extraneous diffs.

Fixes #3332, #3622

Cargo.lock

@@ -1226,7 +1226,6 @@ dependencies = [
  "slotmap",
  "smallvec",
  "smartstring",
- "textwrap",
  "toml",
  "tree-sitter",
  "unicode-general-category",
@@ -2151,12 +2150,6 @@ dependencies = [
  "version_check",
 ]
 
-[[package]]
-name = "smawk"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"
-
 [[package]]
 name = "socket2"
 version = "0.5.7"
@@ -2212,17 +2205,6 @@ dependencies = [
  "home",
 ]
 
-[[package]]
-name = "textwrap"
-version = "0.16.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9"
-dependencies = [
- "smawk",
- "unicode-linebreak",
- "unicode-width",
-]
-
 [[package]]
 name = "thiserror"
 version = "1.0.63"
@@ -2384,12 +2366,6 @@ version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
 
-[[package]]
-name = "unicode-linebreak"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f"
-
 [[package]]
 name = "unicode-normalization"
 version = "0.1.23"

helix-core/Cargo.toml

@@ -53,7 +53,6 @@ encoding_rs = "0.8"
 chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }
 
 etcetera = "0.8"
-textwrap = "0.16.1"
 
 nucleo.workspace = true
 parking_lot = "0.12"

helix-core/src/doc_formatter.rs

@@ -22,7 +22,8 @@
 use crate::graphemes::{Grapheme, GraphemeStr};
 use crate::syntax::Highlight;
 use crate::text_annotations::TextAnnotations;
-use crate::{Position, RopeGraphemes, RopeSlice};
+use crate::{movement, Change, LineEnding, Position, Rope, RopeGraphemes, RopeSlice, Tendril};
+use helix_stdx::rope::RopeSliceExt;
 
 /// TODO make Highlight a u32 to reduce the size of this enum to a single word.
 #[derive(Debug, Clone, Copy)]
@@ -150,6 +151,7 @@ pub struct TextFormat {
     pub wrap_indicator_highlight: Option<Highlight>,
     pub viewport_width: u16,
     pub soft_wrap_at_text_width: bool,
+    pub continue_comments: Vec<String>,
 }
 
 // test implementation is basically only used for testing or when softwrap is always disabled
@@ -164,6 +166,7 @@ fn default() -> Self {
             viewport_width: 17,
             wrap_indicator_highlight: None,
             soft_wrap_at_text_width: false,
+            continue_comments: Vec::new(),
         }
     }
 }
@@ -425,6 +428,51 @@ pub fn next_char_pos(&self) -> usize {
     pub fn next_visual_pos(&self) -> Position {
         self.visual_pos
     }
+
+    fn find_indent<'a>(&self, line: usize, doc: RopeSlice<'a>) -> RopeSlice<'a> {
+        let line_start = doc.line_to_char(line);
+        let mut indent_end = movement::skip_while(doc, line_start, |ch| matches!(ch, ' ' | '\t'))
+            .unwrap_or(line_start);
+        let slice = doc.slice(indent_end..);
+        if let Some(token) = self
+            .text_fmt
+            .continue_comments
+            .iter()
+            .filter(|token| slice.starts_with(token))
+            .max_by_key(|x| x.len())
+        {
+            indent_end += token.chars().count();
+        }
+        let indent_end = movement::skip_while(doc, indent_end, |ch| matches!(ch, ' ' | '\t'))
+            .unwrap_or(indent_end);
+        return doc.slice(line_start..indent_end);
+    }
+
+    /// consumes the iterator and hard-wraps the input where soft wraps would
+    /// have been applied. It probably only makes sense to call this method if
+    /// soft_wrap is true.
+    pub fn reflow(&mut self, doc: &Rope, line_ending: LineEnding) -> Vec<Change> {
+        let slice = doc.slice(..);
+        let mut last_char_start = self.char_pos;
+        let mut current_line = self.visual_pos.row;
+        let mut changes = Vec::new();
+        while let Some(grapheme) = self.next() {
+            if grapheme.visual_pos.row != current_line {
+                let indent = Tendril::from(format!(
+                    "{}{}",
+                    line_ending.as_str(),
+                    self.find_indent(doc.char_to_line(last_char_start), slice)
+                ));
+                changes.push((last_char_start, grapheme.char_idx, Some(indent)));
+                current_line = grapheme.visual_pos.row;
+            }
+            if grapheme.raw == Grapheme::Newline {
+                current_line += 1;
+            }
+            last_char_start = grapheme.char_idx;
+        }
+        changes
+    }
 }
 
 impl<'t> Iterator for DocumentFormatter<'t> {

helix-core/src/doc_formatter/test.rs

@@ -13,6 +13,7 @@ fn new_test(softwrap: bool) -> Self {
             // use a prime number to allow lining up too often with repeat
             viewport_width: 17,
             soft_wrap_at_text_width: false,
+            continue_comments: Vec::new(),
         }
     }
 }

helix-core/src/lib.rs

@@ -28,7 +28,6 @@
 pub mod textobject;
 mod transaction;
 pub mod uri;
-pub mod wrap;
 
 pub mod unicode {
     pub use unicode_general_category as category;

helix-core/src/wrap.rs

@@ -1,9 +0,0 @@
-use smartstring::{LazyCompact, SmartString};
-use textwrap::{Options, WordSplitter::NoHyphenation};
-
-/// Given a slice of text, return the text re-wrapped to fit it
-/// within the given width.
-pub fn reflow_hard_wrap(text: &str, text_width: usize) -> SmartString<LazyCompact> {
-    let options = Options::new(text_width).word_splitter(NoHyphenation);
-    textwrap::refill(text, options).into()
-}

helix-term/src/commands/typed.rs

@@ -6,6 +6,7 @@
 
 use super::*;
 
+use helix_core::doc_formatter::DocumentFormatter;
 use helix_core::fuzzy::fuzzy_match;
 use helix_core::indent::MAX_INDENT;
 use helix_core::{line_ending, shellwords::Shellwords};
@@ -2118,14 +2119,35 @@ fn reflow(
         .unwrap_or(cfg_text_width);
 
     let rope = doc.text();
+    let slice = rope.slice(..);
+    let format = TextFormat {
+        soft_wrap: true,
+        tab_width: 8,
+        max_wrap: u16::try_from(text_width).unwrap_or(u16::MAX),
+        max_indent_retain: u16::try_from(text_width).unwrap_or(u16::MAX),
+        wrap_indicator: Box::from(""),
+        wrap_indicator_highlight: None,
+        viewport_width: u16::try_from(text_width).unwrap_or(u16::MAX),
+        soft_wrap_at_text_width: true,
+        continue_comments: Vec::from(
+            doc.language_config()
+                .and_then(|config| config.comment_tokens.as_deref())
+                .unwrap_or(&[]),
+        ),
+    };
+    let annotations = TextAnnotations::default();
 
-    let selection = doc.selection(view.id);
-    let transaction = Transaction::change_by_selection(rope, selection, |range| {
-        let fragment = range.fragment(rope.slice(..));
-        let reflowed_text = helix_core::wrap::reflow_hard_wrap(&fragment, text_width);
-
-        (range.from(), range.to(), Some(reflowed_text))
-    });
+    let mut changes = Vec::new();
+    for selection in doc.selection(view.id) {
+        let mut formatter = DocumentFormatter::new_at_prev_checkpoint(
+            slice.slice(..selection.to()),
+            &format,
+            &annotations,
+            selection.from(),
+        );
+        changes.append(&mut formatter.reflow(rope, doc.line_ending));
+    }
+    let transaction = Transaction::change(rope, changes.into_iter());
 
     doc.apply(&transaction, view.id);
     doc.append_changes_to_history(view);

helix-term/tests/test/commands.rs

@@ -729,3 +729,74 @@ fn foo() {
 
     Ok(())
 }
+
+#[tokio::test(flavor = "multi_thread")]
+async fn test_reflow() -> anyhow::Result<()> {
+    test((
+        "#[|This is a long line bla bla bla]#",
+        ":reflow 5<ret>",
+        "#[|This
+is a
+long
+line
+bla
+bla
+bla]#",
+    ))
+    .await?;
+
+    test((
+        "// #[|This is a really long comment that we want to break onto multiple lines.]#",
+        ":lang rust<ret>:reflow 13<ret>",
+        "// #[|This is a
+// really long
+// comment that
+// we want to
+// break onto
+// multiple
+// lines.]#",
+    ))
+    .await?;
+
+    test((
+        "#[\t// Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod
+\t// tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim
+\t// veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea
+\t// commodo consequat. Duis aute irure dolor in reprehenderit in voluptate
+\t// velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint
+\t// occaecat cupidatat non proident, sunt in culpa qui officia deserunt
+\t// mollit anim id est laborum.
+|]#",
+        ":lang go<ret>:reflow 50<ret>",
+        "#[\t// Lorem ipsum dolor sit amet,
+\t// consectetur adipiscing elit, sed do
+\t// eiusmod
+\t// tempor incididunt ut labore et dolore
+\t// magna aliqua. Ut enim ad minim
+\t// veniam, quis nostrud exercitation
+\t// ullamco laboris nisi ut aliquip ex ea
+\t// commodo consequat. Duis aute irure
+\t// dolor in reprehenderit in voluptate
+\t// velit esse cillum dolore eu fugiat
+\t// nulla pariatur. Excepteur sint
+\t// occaecat cupidatat non proident, sunt
+\t// in culpa qui officia deserunt
+\t// mollit anim id est laborum.
+|]#",
+    ))
+    .await?;
+
+    test((
+        "    // #[|This document has multiple lines that each need wrapping
+    /// currently we wrap each line completely separately in order to preserve existing newlines.]#",
+        ":lang rust<ret>:reflow 40<ret>",
+        "    // #[|This document has multiple lines
+    // that each need wrapping
+    /// currently we wrap each line
+    /// completely separately in order to
+    /// preserve existing newlines.]#"
+    ))
+    .await?;
+
+    Ok(())
+}

helix-view/src/annotations/diagnostics.rs

@@ -102,6 +102,7 @@ pub fn text_fmt(&self, anchor_col: u16, width: u16) -> TextFormat {
             wrap_indicator_highlight: None,
             viewport_width: width,
             soft_wrap_at_text_width: true,
+            continue_comments: Vec::new(),
         }
     }
 }

helix-view/src/document.rs

@@ -2106,6 +2106,7 @@ pub fn text_format(&self, mut viewport_width: u16, theme: Option<&Theme>) -> Tex
                 .and_then(|theme| theme.find_scope_index("ui.virtual.wrap"))
                 .map(Highlight),
             soft_wrap_at_text_width,
+            continue_comments: Vec::new(),
         }
     }