chore(core): move some types to their own files

harper-core/src/document.rs

@@ -7,10 +7,9 @@ use paste::paste;
 use crate::parsers::{Markdown, Parser, PlainEnglish};
 use crate::patterns::{PatternExt, RepeatingPattern, SequencePattern};
 use crate::punctuation::Punctuation;
-use crate::token::NumberSuffix;
 use crate::vec_ext::VecExt;
-use crate::Span;
 use crate::{Dictionary, FatToken, FstDictionary, Lrc, Token, TokenKind, TokenStringExt};
+use crate::{NumberSuffix, Span};
 
 /// A document containing some amount of lexed and parsed English text.
 #[derive(Debug, Clone)]

harper-core/src/fat_token.rs

@@ -0,0 +1,11 @@
+use serde::{Deserialize, Serialize};
+
+use crate::TokenKind;
+
+/// A [`Token`] that holds its content as a fat [`Vec<char>`] rather than as a
+/// [`Span`].
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd)]
+pub struct FatToken {
+    pub content: Vec<char>,
+    pub kind: TokenKind,
+}

harper-core/src/lexing/mod.rs

@@ -8,8 +8,7 @@ use url::lex_url;
 use self::email_address::lex_email_address;
 use crate::char_ext::CharExt;
 use crate::punctuation::{Punctuation, Quote};
-use crate::token::TokenKind;
-use crate::WordMetadata;
+use crate::{TokenKind, WordMetadata};
 
 #[derive(Debug)]
 pub struct FoundToken {

harper-core/src/lib.rs

@@ -4,6 +4,7 @@
 mod char_ext;
 mod char_string;
 mod document;
+mod fat_token;
 pub mod language_detection;
 mod lexing;
 pub mod linting;
@@ -16,21 +17,24 @@ mod spell;
 mod sync;
 mod title_case;
 mod token;
+mod token_kind;
 mod vec_ext;
 mod word_metadata;
 
 use std::collections::VecDeque;
 
 pub use char_string::{CharString, CharStringExt};
 pub use document::Document;
+pub use fat_token::FatToken;
 use linting::Lint;
 pub use mask::{Mask, Masker};
 pub use punctuation::{Punctuation, Quote};
 pub use span::Span;
 pub use spell::{Dictionary, FstDictionary, FullDictionary, MergedDictionary};
 pub use sync::Lrc;
 pub use title_case::{make_title_case, make_title_case_str};
-pub use token::{FatToken, Token, TokenKind, TokenStringExt};
+pub use token::{Token, TokenStringExt};
+pub use token_kind::{NumberSuffix, TokenKind};
 pub use vec_ext::VecExt;
 pub use word_metadata::{AdverbData, ConjunctionData, NounData, Tense, VerbData, WordMetadata};
 

harper-core/src/linting/compound_words.rs

@@ -84,9 +84,7 @@ impl Linter for CompoundWords {
 
 #[cfg(test)]
 mod tests {
-    use crate::linting::tests::{
-        assert_lint_count, assert_suggestion_count, assert_suggestion_result,
-    };
+    use crate::linting::tests::{assert_lint_count, assert_suggestion_count};
 
     use super::CompoundWords;
 

harper-core/src/linting/correct_number_suffix.rs

@@ -1,6 +1,6 @@
 use super::{Lint, LintKind, Linter, Suggestion};
-use crate::token::{NumberSuffix, TokenStringExt};
-use crate::{Document, Span, TokenKind};
+use crate::token::TokenStringExt;
+use crate::{Document, NumberSuffix, Span, TokenKind};
 
 /// Detect and warn that the sentence is too long.
 #[derive(Debug, Clone, Copy, Default)]

harper-core/src/linting/mod.rs

@@ -74,7 +74,7 @@ pub trait Linter: Send + Sync {
 #[cfg(test)]
 mod tests {
     use super::Linter;
-    use crate::{remove_overlaps, Document};
+    use crate::Document;
 
     pub fn assert_lint_count(text: &str, mut linter: impl Linter, count: usize) {
         let test = Document::new_markdown_curated(text);

harper-core/src/parsers/collapse_identifiers.rs

@@ -3,9 +3,9 @@ use std::sync::Arc;
 
 use itertools::Itertools;
 
-use super::{Parser, TokenKind};
+use super::Parser;
 use crate::patterns::{PatternExt, SequencePattern};
-use crate::{Dictionary, Lrc, Span, Token, VecExt};
+use crate::{Dictionary, Lrc, Span, Token, TokenKind, VecExt};
 
 /// A parser that wraps any other parser to collapse token strings that match
 /// the pattern `word_word` or `word-word`.

harper-core/src/parsers/mod.rs

@@ -11,7 +11,7 @@ pub use markdown::Markdown;
 pub use mask::Mask;
 pub use plain_english::PlainEnglish;
 
-pub use crate::token::{Token, TokenKind, TokenStringExt};
+use crate::{Token, TokenStringExt};
 
 #[cfg(not(feature = "concurrent"))]
 #[blanket(derive(Box))]