Handle Multibyte Unicode

src/visitor.rs

@@ -91,11 +91,6 @@ pub enum ParseError {
         #[backtrace]
         source: std::io::Error,
     },
-    #[error("Multibyte characters found in the file {file:?}")]
-    MultibyteError {
-        file: PathBuf,
-        backtrace: backtrace::Backtrace,
-    },
     #[error("Error parsing the source code for file {file:?} using tree-sitter")]
     TreeSitter {
         file: PathBuf,
@@ -123,14 +118,6 @@ pub fn parse(path: &PathBuf, visitors: Vec<Rc<RefCell<dyn Visitor>>>) -> Result<
         source,
     })?;
 
-    // Check for multibyte characters
-    if source.chars().count() != source.len() {
-        return Err(ParseError::MultibyteError {
-            file: path.clone(),
-            backtrace: backtrace::Backtrace::force_capture(),
-        });
-    }
-
     // Parse the source code
     let arena = Arena::new();
     let options = ExtensionOptions::builder()

tests/logseq/broken_wikilink/assets/pages/foo.md

@@ -2,5 +2,5 @@
 ugds: xlvf, zptqgmbety
 ---
 
-- lorem lhet qfee trbiuhd bz #dolor
+- ’ #dolor
 -

tests/logseq/broken_wikilink/tests.rs

@@ -175,6 +175,6 @@ fn dolor_does_not_exist_and_is_wikilink_in_foo_span() {
         &format!("{}::foo::dolor", broken_wikilink::CODE).into(),
     );
     let err = err_list.iter().exactly_one().unwrap();
-    assert_eq!(err.wikilink.offset(), 62);
-    assert_eq!(err.wikilink.len(), 5);
+    assert_eq!(err.wikilink.offset(), 38);
+    assert_eq!(err.wikilink.len(), 6);
 }

tests/logseq/unlinked_text/assets/pages/foo.md

@@ -3,4 +3,5 @@ ugds: xlvf, zptqgmbety
 ---
 
 - #lorem lhet qfee trbiuhd bz dolors
+- ’ [[lorem]] lorem
 -

tests/logseq/unlinked_text/tests.rs

@@ -25,7 +25,7 @@ fn number_of_unlinked_texts() {
     for unlinked_texts in &report.unlinked_texts() {
         debug!("{unlinked_texts:#?}");
     }
-    assert_eq!(report.unlinked_texts().len(), 3);
+    assert_eq!(report.unlinked_texts().len(), 4);
 }
 
 /// This passes because the link is valid
@@ -101,3 +101,17 @@ fn icazyvey_exists_and_is_not_wikilink_in_journal() {
     assert_eq!(err.span.offset(), offset.offset());
     assert_eq!(err.span.len(), 8);
 }
+
+/// Tests that linking is right after a non-standard character like "right parentheses" U+2019
+#[test]
+fn lorem_exists_and_is_not_wikilink_in_journal() {
+    info!("lorem_exists_and_is_not_wikilink_in_journal");
+    let report = get_report(PATHS.as_slice());
+    let err_list = filter_code(
+        report.unlinked_texts(),
+        &format!("{}::foo::lorem", unlinked_text::CODE).into(),
+    );
+    let err = err_list.iter().exactly_one().unwrap();
+    assert_eq!(err.span.offset(), 85);
+    assert_eq!(err.span.len(), 5);
+}