diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs index 1c34ccea..61b75f4d 100644 --- a/harper-core/src/document.rs +++ b/harper-core/src/document.rs @@ -7,10 +7,9 @@ use paste::paste; use crate::parsers::{Markdown, Parser, PlainEnglish}; use crate::patterns::{PatternExt, RepeatingPattern, SequencePattern}; use crate::punctuation::Punctuation; -use crate::token::NumberSuffix; use crate::vec_ext::VecExt; -use crate::Span; use crate::{Dictionary, FatToken, FstDictionary, Lrc, Token, TokenKind, TokenStringExt}; +use crate::{NumberSuffix, Span}; /// A document containing some amount of lexed and parsed English text. #[derive(Debug, Clone)] diff --git a/harper-core/src/fat_token.rs b/harper-core/src/fat_token.rs new file mode 100644 index 00000000..7db6c609 --- /dev/null +++ b/harper-core/src/fat_token.rs @@ -0,0 +1,11 @@ +use serde::{Deserialize, Serialize}; + +use crate::TokenKind; + +/// A [`Token`] that holds its content as a fat [`Vec`] rather than as a +/// [`Span`]. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd)] +pub struct FatToken { + pub content: Vec, + pub kind: TokenKind, +} diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rs index a2d03ece..e451d311 100644 --- a/harper-core/src/lexing/mod.rs +++ b/harper-core/src/lexing/mod.rs @@ -8,8 +8,7 @@ use url::lex_url; use self::email_address::lex_email_address; use crate::char_ext::CharExt; use crate::punctuation::{Punctuation, Quote}; -use crate::token::TokenKind; -use crate::WordMetadata; +use crate::{TokenKind, WordMetadata}; #[derive(Debug)] pub struct FoundToken { diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rs index aa5c16b1..abaefe6d 100644 --- a/harper-core/src/lib.rs +++ b/harper-core/src/lib.rs @@ -4,6 +4,7 @@ mod char_ext; mod char_string; mod document; +mod fat_token; pub mod language_detection; mod lexing; pub mod linting; @@ -16,6 +17,7 @@ mod spell; mod sync; mod title_case; mod token; +mod token_kind; mod vec_ext; mod word_metadata; @@ -23,6 +25,7 @@ use std::collections::VecDeque; pub use char_string::{CharString, CharStringExt}; pub use document::Document; +pub use fat_token::FatToken; use linting::Lint; pub use mask::{Mask, Masker}; pub use punctuation::{Punctuation, Quote}; @@ -30,7 +33,8 @@ pub use span::Span; pub use spell::{Dictionary, FstDictionary, FullDictionary, MergedDictionary}; pub use sync::Lrc; pub use title_case::{make_title_case, make_title_case_str}; -pub use token::{FatToken, Token, TokenKind, TokenStringExt}; +pub use token::{Token, TokenStringExt}; +pub use token_kind::{NumberSuffix, TokenKind}; pub use vec_ext::VecExt; pub use word_metadata::{AdverbData, ConjunctionData, NounData, Tense, VerbData, WordMetadata}; diff --git a/harper-core/src/linting/compound_words.rs b/harper-core/src/linting/compound_words.rs index c644f5f6..1d50f60a 100644 --- a/harper-core/src/linting/compound_words.rs +++ b/harper-core/src/linting/compound_words.rs @@ -84,9 +84,7 @@ impl Linter for CompoundWords { #[cfg(test)] mod tests { - use crate::linting::tests::{ - assert_lint_count, assert_suggestion_count, assert_suggestion_result, - }; + use crate::linting::tests::{assert_lint_count, assert_suggestion_count}; use super::CompoundWords; diff --git a/harper-core/src/linting/correct_number_suffix.rs b/harper-core/src/linting/correct_number_suffix.rs index 8d095156..f5f7d5f0 100644 --- a/harper-core/src/linting/correct_number_suffix.rs +++ b/harper-core/src/linting/correct_number_suffix.rs @@ -1,6 +1,6 @@ use super::{Lint, LintKind, Linter, Suggestion}; -use crate::token::{NumberSuffix, TokenStringExt}; -use crate::{Document, Span, TokenKind}; +use crate::token::TokenStringExt; +use crate::{Document, NumberSuffix, Span, TokenKind}; /// Detect and warn that the sentence is too long. #[derive(Debug, Clone, Copy, Default)] diff --git a/harper-core/src/linting/mod.rs b/harper-core/src/linting/mod.rs index bbd650b6..87e26c91 100644 --- a/harper-core/src/linting/mod.rs +++ b/harper-core/src/linting/mod.rs @@ -74,7 +74,7 @@ pub trait Linter: Send + Sync { #[cfg(test)] mod tests { use super::Linter; - use crate::{remove_overlaps, Document}; + use crate::Document; pub fn assert_lint_count(text: &str, mut linter: impl Linter, count: usize) { let test = Document::new_markdown_curated(text); diff --git a/harper-core/src/parsers/collapse_identifiers.rs b/harper-core/src/parsers/collapse_identifiers.rs index 59fd81a1..e98f2c11 100644 --- a/harper-core/src/parsers/collapse_identifiers.rs +++ b/harper-core/src/parsers/collapse_identifiers.rs @@ -3,9 +3,9 @@ use std::sync::Arc; use itertools::Itertools; -use super::{Parser, TokenKind}; +use super::Parser; use crate::patterns::{PatternExt, SequencePattern}; -use crate::{Dictionary, Lrc, Span, Token, VecExt}; +use crate::{Dictionary, Lrc, Span, Token, TokenKind, VecExt}; /// A parser that wraps any other parser to collapse token strings that match /// the pattern `word_word` or `word-word`. diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rs index f35f209b..55900667 100644 --- a/harper-core/src/parsers/mod.rs +++ b/harper-core/src/parsers/mod.rs @@ -11,7 +11,7 @@ pub use markdown::Markdown; pub use mask::Mask; pub use plain_english::PlainEnglish; -pub use crate::token::{Token, TokenKind, TokenStringExt}; +use crate::{Token, TokenStringExt}; #[cfg(not(feature = "concurrent"))] #[blanket(derive(Box))] diff --git a/harper-core/src/token.rs b/harper-core/src/token.rs index b8ef580e..5eaa36c7 100644 --- a/harper-core/src/token.rs +++ b/harper-core/src/token.rs @@ -1,13 +1,8 @@ -use is_macro::Is; use itertools::Itertools; -use ordered_float::OrderedFloat; use paste::paste; use serde::{Deserialize, Serialize}; -use crate::punctuation::Punctuation; -use crate::Span; -use crate::{ConjunctionData, NounData}; -use crate::{Quote, WordMetadata}; +use crate::{FatToken, Span, TokenKind}; #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Default)] pub struct Token { @@ -31,344 +26,6 @@ impl Token { } } -/// A [`Token`] that holds its content as a fat [`Vec`] rather than as a -/// [`Span`]. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd)] -pub struct FatToken { - pub content: Vec, - pub kind: TokenKind, -} - -#[derive( - Debug, Is, Clone, Copy, Serialize, Deserialize, Default, PartialOrd, Hash, Eq, PartialEq, -)] -#[serde(tag = "kind", content = "value")] -pub enum TokenKind { - Word(WordMetadata), - Punctuation(Punctuation), - Number(OrderedFloat, Option), - /// A sequence of " " spaces. - Space(usize), - /// A sequence of "\n" newlines - Newline(usize), - EmailAddress, - Url, - Hostname, - /// A special token used for things like inline code blocks that should be - /// ignored by all linters. - #[default] - Unlintable, - ParagraphBreak, -} - -impl TokenKind { - pub fn is_open_square(&self) -> bool { - matches!(self, TokenKind::Punctuation(Punctuation::OpenSquare)) - } - - pub fn is_close_square(&self) -> bool { - matches!(self, TokenKind::Punctuation(Punctuation::CloseSquare)) - } - - pub fn is_pipe(&self) -> bool { - matches!(self, TokenKind::Punctuation(Punctuation::Pipe)) - } - - pub fn is_pronoun(&self) -> bool { - matches!( - self, - TokenKind::Word(WordMetadata { - noun: Some(NounData { - is_pronoun: Some(true), - .. - }), - .. - }) - ) - } - - pub fn is_conjunction(&self) -> bool { - matches!( - self, - TokenKind::Word(WordMetadata { - conjunction: Some(ConjunctionData {}), - .. - }) - ) - } - - fn is_chunk_terminator(&self) -> bool { - if self.is_sentence_terminator() { - return true; - } - - match self { - TokenKind::Punctuation(punct) => { - matches!( - punct, - Punctuation::Comma | Punctuation::Quote { .. } | Punctuation::Colon - ) - } - _ => false, - } - } - - fn is_sentence_terminator(&self) -> bool { - match self { - TokenKind::Punctuation(punct) => [ - Punctuation::Period, - Punctuation::Bang, - Punctuation::Question, - ] - .contains(punct), - TokenKind::ParagraphBreak => true, - _ => false, - } - } - - pub fn is_ellipsis(&self) -> bool { - matches!(self, TokenKind::Punctuation(Punctuation::Ellipsis)) - } - - pub fn is_hyphen(&self) -> bool { - matches!(self, TokenKind::Punctuation(Punctuation::Hyphen)) - } - - pub fn is_adjective(&self) -> bool { - matches!( - self, - TokenKind::Word(WordMetadata { - adjective: Some(_), - .. - }) - ) - } - - pub fn is_adverb(&self) -> bool { - matches!( - self, - TokenKind::Word(WordMetadata { - adverb: Some(_), - .. - }) - ) - } - - pub fn is_swear(&self) -> bool { - matches!( - self, - TokenKind::Word(WordMetadata { - swear: Some(true), - .. - }) - ) - } - - /// Checks that `self` is the same enum variant as `other`, regardless of - /// whether the inner metadata is also equal. - pub fn matches_variant_of(&self, other: &Self) -> bool { - self.with_default_data() == other.with_default_data() - } - - /// Produces a copy of `self` with any inner data replaced with it's default - /// value. Useful for making comparisons on just the variant of the - /// enum. - pub fn with_default_data(&self) -> Self { - match self { - TokenKind::Word(_) => TokenKind::Word(Default::default()), - TokenKind::Punctuation(_) => TokenKind::Punctuation(Default::default()), - TokenKind::Number(..) => TokenKind::Number(Default::default(), Default::default()), - TokenKind::Space(_) => TokenKind::Space(Default::default()), - TokenKind::Newline(_) => TokenKind::Newline(Default::default()), - _ => *self, - } - } -} - -impl TokenKind { - /// Construct a [`TokenKind::Word`] with no (default) metadata. - pub fn blank_word() -> Self { - Self::Word(WordMetadata::default()) - } -} - -#[derive( - Debug, Serialize, Deserialize, Default, PartialEq, PartialOrd, Clone, Copy, Is, Hash, Eq, -)] -pub enum NumberSuffix { - #[default] - Th, - St, - Nd, - Rd, -} - -impl NumberSuffix { - pub fn correct_suffix_for(number: impl Into) -> Option { - let number = number.into(); - - if number < 0.0 || number - number.floor() > f64::EPSILON || number > u64::MAX as f64 { - return None; - } - - let integer = number as u64; - - if let 11..=13 = integer % 100 { - return Some(Self::Th); - }; - - match integer % 10 { - 0 => Some(Self::Th), - 1 => Some(Self::St), - 2 => Some(Self::Nd), - 3 => Some(Self::Rd), - 4 => Some(Self::Th), - 5 => Some(Self::Th), - 6 => Some(Self::Th), - 7 => Some(Self::Th), - 8 => Some(Self::Th), - 9 => Some(Self::Th), - _ => None, - } - } - - pub fn to_chars(self) -> Vec { - match self { - NumberSuffix::Th => vec!['t', 'h'], - NumberSuffix::St => vec!['s', 't'], - NumberSuffix::Nd => vec!['n', 'd'], - NumberSuffix::Rd => vec!['r', 'd'], - } - } - - /// Check the first several characters in a buffer to see if it matches a - /// number suffix. - pub fn from_chars(chars: &[char]) -> Option { - if chars.len() < 2 { - return None; - } - - match (chars[0], chars[1]) { - ('t', 'h') => Some(NumberSuffix::Th), - ('T', 'h') => Some(NumberSuffix::Th), - ('t', 'H') => Some(NumberSuffix::Th), - ('T', 'H') => Some(NumberSuffix::Th), - ('s', 't') => Some(NumberSuffix::St), - ('S', 't') => Some(NumberSuffix::St), - ('s', 'T') => Some(NumberSuffix::St), - ('S', 'T') => Some(NumberSuffix::St), - ('n', 'd') => Some(NumberSuffix::Nd), - ('N', 'd') => Some(NumberSuffix::Nd), - ('n', 'D') => Some(NumberSuffix::Nd), - ('N', 'D') => Some(NumberSuffix::Nd), - ('r', 'd') => Some(NumberSuffix::Rd), - ('R', 'd') => Some(NumberSuffix::Rd), - ('r', 'D') => Some(NumberSuffix::Rd), - ('R', 'D') => Some(NumberSuffix::Rd), - _ => None, - } - } -} - -impl TokenKind { - pub fn as_mut_quote(&mut self) -> Option<&mut Quote> { - self.as_mut_punctuation()?.as_mut_quote() - } - - pub fn as_quote(&self) -> Option<&Quote> { - self.as_punctuation()?.as_quote() - } - - pub fn is_quote(&self) -> bool { - matches!(self, TokenKind::Punctuation(Punctuation::Quote(_))) - } - - pub fn is_apostrophe(&self) -> bool { - matches!(self, TokenKind::Punctuation(Punctuation::Apostrophe)) - } - - pub fn is_period(&self) -> bool { - matches!(self, TokenKind::Punctuation(Punctuation::Period)) - } - - pub fn is_at(&self) -> bool { - matches!(self, TokenKind::Punctuation(Punctuation::At)) - } - - /// Used by `crate::parsers::CollapseIdentifiers` - /// TODO: Separate this into two functions and add OR functionality to - /// pattern matching - pub fn is_case_separator(&self) -> bool { - matches!(self, TokenKind::Punctuation(Punctuation::Underscore)) - || matches!(self, TokenKind::Punctuation(Punctuation::Hyphen)) - } - - pub fn is_verb(&self) -> bool { - let TokenKind::Word(metadata) = self else { - return false; - }; - - metadata.is_verb() - } - - pub fn is_linking_verb(&self) -> bool { - let TokenKind::Word(metadata) = self else { - return false; - }; - - metadata.is_linking_verb() - } - - pub fn is_not_plural_noun(&self) -> bool { - let TokenKind::Word(metadata) = self else { - return true; - }; - - metadata.is_not_plural_noun() - } - - pub fn is_common_word(&self) -> bool { - let TokenKind::Word(metadata) = self else { - return true; - }; - - metadata.common - } - - pub fn is_plural_noun(&self) -> bool { - let TokenKind::Word(metadata) = self else { - return false; - }; - - metadata.is_plural_noun() - } - - pub fn is_noun(&self) -> bool { - let TokenKind::Word(metadata) = self else { - return false; - }; - - metadata.is_noun() - } - - pub fn is_likely_homograph(&self) -> bool { - let TokenKind::Word(metadata) = self else { - return false; - }; - - metadata.is_likely_homograph() - } - - pub fn is_comma(&self) -> bool { - matches!(self, TokenKind::Punctuation(Punctuation::Comma)) - } - - /// Checks whether the token is whitespace. - pub fn is_whitespace(&self) -> bool { - matches!(self, TokenKind::Space(_) | TokenKind::Newline(_)) - } -} - macro_rules! create_decl_for { ($thing:ident) => { paste! { diff --git a/harper-core/src/token_kind.rs b/harper-core/src/token_kind.rs new file mode 100644 index 00000000..b669c040 --- /dev/null +++ b/harper-core/src/token_kind.rs @@ -0,0 +1,335 @@ +use is_macro::Is; +use ordered_float::OrderedFloat; +use serde::{Deserialize, Serialize}; + +use crate::{ConjunctionData, NounData, Punctuation, Quote, WordMetadata}; + +#[derive( + Debug, Is, Clone, Copy, Serialize, Deserialize, Default, PartialOrd, Hash, Eq, PartialEq, +)] +#[serde(tag = "kind", content = "value")] +pub enum TokenKind { + Word(WordMetadata), + Punctuation(Punctuation), + Number(OrderedFloat, Option), + /// A sequence of " " spaces. + Space(usize), + /// A sequence of "\n" newlines + Newline(usize), + EmailAddress, + Url, + Hostname, + /// A special token used for things like inline code blocks that should be + /// ignored by all linters. + #[default] + Unlintable, + ParagraphBreak, +} + +impl TokenKind { + pub fn is_open_square(&self) -> bool { + matches!(self, TokenKind::Punctuation(Punctuation::OpenSquare)) + } + + pub fn is_close_square(&self) -> bool { + matches!(self, TokenKind::Punctuation(Punctuation::CloseSquare)) + } + + pub fn is_pipe(&self) -> bool { + matches!(self, TokenKind::Punctuation(Punctuation::Pipe)) + } + + pub fn is_pronoun(&self) -> bool { + matches!( + self, + TokenKind::Word(WordMetadata { + noun: Some(NounData { + is_pronoun: Some(true), + .. + }), + .. + }) + ) + } + + pub fn is_conjunction(&self) -> bool { + matches!( + self, + TokenKind::Word(WordMetadata { + conjunction: Some(ConjunctionData {}), + .. + }) + ) + } + + pub(crate) fn is_chunk_terminator(&self) -> bool { + if self.is_sentence_terminator() { + return true; + } + + match self { + TokenKind::Punctuation(punct) => { + matches!( + punct, + Punctuation::Comma | Punctuation::Quote { .. } | Punctuation::Colon + ) + } + _ => false, + } + } + + pub(crate) fn is_sentence_terminator(&self) -> bool { + match self { + TokenKind::Punctuation(punct) => [ + Punctuation::Period, + Punctuation::Bang, + Punctuation::Question, + ] + .contains(punct), + TokenKind::ParagraphBreak => true, + _ => false, + } + } + + pub fn is_ellipsis(&self) -> bool { + matches!(self, TokenKind::Punctuation(Punctuation::Ellipsis)) + } + + pub fn is_hyphen(&self) -> bool { + matches!(self, TokenKind::Punctuation(Punctuation::Hyphen)) + } + + pub fn is_adjective(&self) -> bool { + matches!( + self, + TokenKind::Word(WordMetadata { + adjective: Some(_), + .. + }) + ) + } + + pub fn is_adverb(&self) -> bool { + matches!( + self, + TokenKind::Word(WordMetadata { + adverb: Some(_), + .. + }) + ) + } + + pub fn is_swear(&self) -> bool { + matches!( + self, + TokenKind::Word(WordMetadata { + swear: Some(true), + .. + }) + ) + } + + /// Checks that `self` is the same enum variant as `other`, regardless of + /// whether the inner metadata is also equal. + pub fn matches_variant_of(&self, other: &Self) -> bool { + self.with_default_data() == other.with_default_data() + } + + /// Produces a copy of `self` with any inner data replaced with it's default + /// value. Useful for making comparisons on just the variant of the + /// enum. + pub fn with_default_data(&self) -> Self { + match self { + TokenKind::Word(_) => TokenKind::Word(Default::default()), + TokenKind::Punctuation(_) => TokenKind::Punctuation(Default::default()), + TokenKind::Number(..) => TokenKind::Number(Default::default(), Default::default()), + TokenKind::Space(_) => TokenKind::Space(Default::default()), + TokenKind::Newline(_) => TokenKind::Newline(Default::default()), + _ => *self, + } + } +} + +impl TokenKind { + /// Construct a [`TokenKind::Word`] with no (default) metadata. + pub fn blank_word() -> Self { + Self::Word(WordMetadata::default()) + } +} + +#[derive( + Debug, Serialize, Deserialize, Default, PartialEq, PartialOrd, Clone, Copy, Is, Hash, Eq, +)] +pub enum NumberSuffix { + #[default] + Th, + St, + Nd, + Rd, +} + +impl NumberSuffix { + pub fn correct_suffix_for(number: impl Into) -> Option { + let number = number.into(); + + if number < 0.0 || number - number.floor() > f64::EPSILON || number > u64::MAX as f64 { + return None; + } + + let integer = number as u64; + + if let 11..=13 = integer % 100 { + return Some(Self::Th); + }; + + match integer % 10 { + 0 => Some(Self::Th), + 1 => Some(Self::St), + 2 => Some(Self::Nd), + 3 => Some(Self::Rd), + 4 => Some(Self::Th), + 5 => Some(Self::Th), + 6 => Some(Self::Th), + 7 => Some(Self::Th), + 8 => Some(Self::Th), + 9 => Some(Self::Th), + _ => None, + } + } + + pub fn to_chars(self) -> Vec { + match self { + NumberSuffix::Th => vec!['t', 'h'], + NumberSuffix::St => vec!['s', 't'], + NumberSuffix::Nd => vec!['n', 'd'], + NumberSuffix::Rd => vec!['r', 'd'], + } + } + + /// Check the first several characters in a buffer to see if it matches a + /// number suffix. + pub fn from_chars(chars: &[char]) -> Option { + if chars.len() < 2 { + return None; + } + + match (chars[0], chars[1]) { + ('t', 'h') => Some(NumberSuffix::Th), + ('T', 'h') => Some(NumberSuffix::Th), + ('t', 'H') => Some(NumberSuffix::Th), + ('T', 'H') => Some(NumberSuffix::Th), + ('s', 't') => Some(NumberSuffix::St), + ('S', 't') => Some(NumberSuffix::St), + ('s', 'T') => Some(NumberSuffix::St), + ('S', 'T') => Some(NumberSuffix::St), + ('n', 'd') => Some(NumberSuffix::Nd), + ('N', 'd') => Some(NumberSuffix::Nd), + ('n', 'D') => Some(NumberSuffix::Nd), + ('N', 'D') => Some(NumberSuffix::Nd), + ('r', 'd') => Some(NumberSuffix::Rd), + ('R', 'd') => Some(NumberSuffix::Rd), + ('r', 'D') => Some(NumberSuffix::Rd), + ('R', 'D') => Some(NumberSuffix::Rd), + _ => None, + } + } +} + +impl TokenKind { + pub fn as_mut_quote(&mut self) -> Option<&mut Quote> { + self.as_mut_punctuation()?.as_mut_quote() + } + + pub fn as_quote(&self) -> Option<&Quote> { + self.as_punctuation()?.as_quote() + } + + pub fn is_quote(&self) -> bool { + matches!(self, TokenKind::Punctuation(Punctuation::Quote(_))) + } + + pub fn is_apostrophe(&self) -> bool { + matches!(self, TokenKind::Punctuation(Punctuation::Apostrophe)) + } + + pub fn is_period(&self) -> bool { + matches!(self, TokenKind::Punctuation(Punctuation::Period)) + } + + pub fn is_at(&self) -> bool { + matches!(self, TokenKind::Punctuation(Punctuation::At)) + } + + /// Used by `crate::parsers::CollapseIdentifiers` + /// TODO: Separate this into two functions and add OR functionality to + /// pattern matching + pub fn is_case_separator(&self) -> bool { + matches!(self, TokenKind::Punctuation(Punctuation::Underscore)) + || matches!(self, TokenKind::Punctuation(Punctuation::Hyphen)) + } + + pub fn is_verb(&self) -> bool { + let TokenKind::Word(metadata) = self else { + return false; + }; + + metadata.is_verb() + } + + pub fn is_linking_verb(&self) -> bool { + let TokenKind::Word(metadata) = self else { + return false; + }; + + metadata.is_linking_verb() + } + + pub fn is_not_plural_noun(&self) -> bool { + let TokenKind::Word(metadata) = self else { + return true; + }; + + metadata.is_not_plural_noun() + } + + pub fn is_common_word(&self) -> bool { + let TokenKind::Word(metadata) = self else { + return true; + }; + + metadata.common + } + + pub fn is_plural_noun(&self) -> bool { + let TokenKind::Word(metadata) = self else { + return false; + }; + + metadata.is_plural_noun() + } + + pub fn is_noun(&self) -> bool { + let TokenKind::Word(metadata) = self else { + return false; + }; + + metadata.is_noun() + } + + pub fn is_likely_homograph(&self) -> bool { + let TokenKind::Word(metadata) = self else { + return false; + }; + + metadata.is_likely_homograph() + } + + pub fn is_comma(&self) -> bool { + matches!(self, TokenKind::Punctuation(Punctuation::Comma)) + } + + /// Checks whether the token is whitespace. + pub fn is_whitespace(&self) -> bool { + matches!(self, TokenKind::Space(_) | TokenKind::Newline(_)) + } +}