Skip to content

Commit

Permalink
Revert "Stop using String token in pygments lexers"
Browse files Browse the repository at this point in the history
  • Loading branch information
@denismerigoux
denismerigoux authored Jan 15, 2025
1 parent d4ee59c commit 517325c
Show file tree
Hide file tree
Showing 3 changed files with 34 additions and 34 deletions.
28 changes: 14 additions & 14 deletions syntax_highlighting/en/pygments/catala_en_lexer/lexer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,17 @@ class CustomLexer(RegexLexer):
'root' : [
(u'(^\\s*[\\#]+.*)', bygroups(Generic.Heading)),
(u'(^\\s*[\\#]+\\s*\\[[^\\]\\n\\r]\\s*].*)', bygroups(Generic.Heading)),
(u'([^`\\n\\r])', bygroups(Text)),
(u'(^```catala$)', bygroups(Text), 'code'),
(u'(^```catala-metadata$)', bygroups(Text), 'code'),
(u'(^```catala-test-inline$)', bygroups(Text), 'test'),
('(\n|\r|\r\n)', Whitespace),
('.', Text),
(u'([^`\\n\\r])', bygroups(String)),
(u'(^```catala$)', bygroups(String), 'code'),
(u'(^```catala-metadata$)', bygroups(String), 'code'),
(u'(^```catala-test-inline$)', bygroups(String), 'test'),
('(\n|\r|\r\n)', String),
('.', String),
],
'code' : [
(u'(^```$)', bygroups(Text), 'root'),
(u'(^```$)', bygroups(String), 'root'),
(u'(\\s*\\#.*$)', bygroups(Comment.Single)),
(u'(context|input|output|internal)(\\s*)(|output)(\\s+)([a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xca\u0152\xc70-9_\\\']*)', bygroups(Keyword.Declaration, Whitespace, Keyword.Declaration, Whitespace, Name.Variable)),
(u'(context|input|output|internal)(\\s*)(|output)(\\s+)([a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xca\u0152\xc70-9_\\\']*)', bygroups(Keyword.Declaration, String, Keyword.Declaration, String, Name.Variable)),
(u'\\b(match|with\\s+pattern|but\\s+replace|fixed|by|decreasing|increasing|varies|with|we\\s+have|let|in|scope|depends\\s+on|declaration|includes|content|rule|under\\s+condition|condition|data|consequence|fulfilled|equals|assertion|definition|state|label|exception)\\b', bygroups(Keyword.Reserved)),
(u'\\b(contains|number|sum|such\\s+that|exists|for|all|of|if|then|else|is|list\\s+empty|among|maximum|minimum|round|combine|initially)\\b', bygroups(Keyword.Declaration)),
(u'(\\|[0-9]+\\-[0-9]+\\-[0-9]+\\|)', bygroups(Number.Integer)),
Expand All @@ -35,16 +35,16 @@ class CustomLexer(RegexLexer):
(u'(\\-\\>|\\+\\.|\\+\\@|\\+\\^|\\+\\$|\\+|\\-\\.|\\-\\@|\\-\\^|\\-\\$|\\-|\\*\\.|\\*\\@|\\*\\^|\\*\\$|\\*|/\\.|/\\@|/\\$|/|\\!|>\\.|>=\\.|<=\\.|<\\.|>\\@|>=\\@|<=\\@|<\\@|>\\$|>=\\$|<=\\$|<\\$|>\\^|>=\\^|<=\\^|<\\^|>|>=|<=|<|=|not|or|xor|and|\\$|\u20ac|%|year|month|day)', bygroups(Operator)),
(u'\\b(structure|enumeration|list\\s+of|integer|boolean|date|duration|money|text|decimal)\\b', bygroups(Keyword.Type)),
(u'\\b([A-Z\xc9\xc8\xc0\xc2\xd9\xce\xca\u0152\xc7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xca\u0152\xc70-9_\\\']*)(\\.)([a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xca\u0152\xc70-9_\\\']*)\\b', bygroups(Name.Class, Operator, Name.Variable)),
(u'\\b([a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xca\u0152\xc70-9_\\\']*)(\\.)([a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xca\u0152\xc70-9_\\\'\\.]*)\\b', bygroups(Name.Variable, Operator, Text)),
(u'\\b([a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xca\u0152\xc70-9_\\\']*)(\\.)([a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xca\u0152\xc70-9_\\\'\\.]*)\\b', bygroups(Name.Variable, Operator, String)),
(u'\\b([a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xca\u0152\xc70-9_\\\']*)\\b', bygroups(Name.Variable)),
(u'\\b([A-Z\xc9\xc8\xc0\xc2\xd9\xce\xca\u0152\xc7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xca\u0152\xc70-9_\\\']*)\\b', bygroups(Name.Class)),
('(\n|\r|\r\n)', Whitespace),
('.', Text),
('(\n|\r|\r\n)', String),
('.', String),
],
'test' : [
(u'(^```$)', bygroups(Text), 'root'),
(u'(^```$)', bygroups(String), 'root'),
(u'(^[$] catala \\S*)', bygroups(Keyword.Constant)),
('(\n|\r|\r\n)', Whitespace),
('.', Text),
('(\n|\r|\r\n)', String),
('.', String),
]
}
28 changes: 14 additions & 14 deletions syntax_highlighting/fr/pygments/catala_fr_lexer/lexer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,17 @@ class CustomLexer(RegexLexer):
'root' : [
(u'(^\\s*[\\#]+.*)', bygroups(Generic.Heading)),
(u'(^\\s*[\\#]+\\s*\\[[^\\]\\n\\r]\\s*].*)', bygroups(Generic.Heading)),
(u'([^`\\n\\r])', bygroups(Text)),
(u'(^```catala$)', bygroups(Text), 'code'),
(u'(^```catala-metadata$)', bygroups(Text), 'code'),
(u'(^```catala-test-inline$)', bygroups(Text), 'test'),
('(\n|\r|\r\n)', Whitespace),
('.', Text),
(u'([^`\\n\\r])', bygroups(String)),
(u'(^```catala$)', bygroups(String), 'code'),
(u'(^```catala-metadata$)', bygroups(String), 'code'),
(u'(^```catala-test-inline$)', bygroups(String), 'test'),
('(\n|\r|\r\n)', String),
('.', String),
],
'code' : [
(u'(^```$)', bygroups(Text), 'root'),
(u'(^```$)', bygroups(String), 'root'),
(u'(\\s*\\#.*$)', bygroups(Comment.Single)),
(u'(contexte|entr\xe9e|r\xe9sultat|interne)(\\s*)(|r\xe9sultat)(\\s+)([a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xd4\xca\u0152\xc70-9_\\\']*)', bygroups(Keyword.Declaration, Whitespace, Keyword.Declaration, Whitespace, Name.Variable)),
(u'(contexte|entr\xe9e|r\xe9sultat|interne)(\\s*)(|r\xe9sultat)(\\s+)([a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xd4\xca\u0152\xc70-9_\\\']*)', bygroups(Keyword.Declaration, String, Keyword.Declaration, String, Name.Variable)),
(u'\\b(selon|sous\\s+forme|mais\\s+en\\s+rempla\xe7ant|fix\xe9|par|d\xe9croissant|croissant|varie|avec|on\\s+a|soit|dans|champ\\s+d\'application|d\xe9pend\\s+de|d\xe9claration|inclusion|contenu|r\xe8gle|sous\\s+condition|condition|donn\xe9e|cons\xe9quence|rempli|\xe9gal\\s+\xe0|assertion|d\xe9finition|\xe9tat|\xe9tiquette|exception)\\b', bygroups(Keyword.Reserved)),
(u'\\b(contient|nombre|somme|tel\\s+que|existe|pour|tout|de|si|alors|sinon|est|liste\\s+vide|parmi|maximum|minimum|arrondi|combinaison\\s+de|initialement)\\b', bygroups(Keyword.Declaration)),
(u'(\\|[0-9]+\\-[0-9]+\\-[0-9]+\\|)', bygroups(Number.Integer)),
Expand All @@ -36,16 +36,16 @@ class CustomLexer(RegexLexer):
(u'\\b(non|ou\\s+bien|ou|et|an|mois|jour)\\b', bygroups(Operator)),
(u'\\b(structure|\xe9num\xe9ration|liste\\s+de|entier|bool\xe9en|date|dur\xe9e|argent|texte|d\xe9cimal|d\xe9cret|loi|nombre|somme)\\b', bygroups(Keyword.Type)),
(u'\\b([A-Z\xc9\xc8\xc0\xc2\xd9\xce\xd4\xca\u0152\xc7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xd4\xca\u0152\xc70-9_\\\']*)(\\.)([a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xd4\xca\u0152\xc70-9_\\\']*)\\b', bygroups(Name.Class, Operator, Name.Variable)),
(u'\\b([a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xd4\xca\u0152\xc70-9_\\\']*)(\\.)([a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xd4\xca\u0152\xc70-9_\\\'\\.]*)\\b', bygroups(Name.Variable, Operator, Text)),
(u'\\b([a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xd4\xca\u0152\xc70-9_\\\']*)(\\.)([a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xd4\xca\u0152\xc70-9_\\\'\\.]*)\\b', bygroups(Name.Variable, Operator, String)),
(u'\\b([a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xd4\xca\u0152\xc70-9_\\\']*)\\b', bygroups(Name.Variable)),
(u'\\b([A-Z\xc9\xc8\xc0\xc2\xd9\xce\xd4\xca\u0152\xc7][a-z\xe9\xe8\xe0\xe2\xf9\xee\xf4\xea\u0153\xe7A-Z\xc9\xc8\xc0\xc2\xd9\xce\xd4\xca\u0152\xc70-9_\\\']*)\\b', bygroups(Name.Class)),
('(\n|\r|\r\n)', Whitespace),
('.', Text),
('(\n|\r|\r\n)', String),
('.', String),
],
'test' : [
(u'(^```$)', bygroups(Text), 'root'),
(u'(^```$)', bygroups(String), 'root'),
(u'(^[$] catala \\S*)', bygroups(Keyword.Constant)),
('(\n|\r|\r\n)', Whitespace),
('.', Text),
('(\n|\r|\r\n)', String),
('.', String),
]
}
12 changes: 6 additions & 6 deletions tests/literate/good/test_grave_char_en.catala_en
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,13 +80,13 @@ let () = print_endline "Hello world!"
\end{verbatim}

\begin{Verbatim}[commandchars=\\\{\},numbers=left,firstnumber=19,stepnumber=1,label={\hspace*{\fill}\texttt{test\_grave\_char\_en.catala\_en}}]
\```catala
\PY{k+kr}{declaration} \PY{k+kr}{scope} \PY{n+nc}{A}\PY{o}{:}
\PY{k+kd}{context}\PY{+w}{ }\PY{k+kd}{output}\PY{+w}{ }\PY{n+nv}{literate\PYZus{}parsing\PYZus{}is\PYZus{}ok} \PY{k+kr}{content} \PY{k+kt}{boolean}
\PY{l+s}{```catala}
\PY{k+kr}{declaration}\PY{l+s}{ }\PY{k+kr}{scope}\PY{l+s}{ }\PY{n+nc}{A}\PY{o}{:}
\PY{l+s}{ }\PY{l+s}{ }\PY{k+kd}{context}\PY{l+s}{ }\PY{k+kd}{output}\PY{l+s}{ }\PY{n+nv}{literate\PYZus{}parsing\PYZus{}is\PYZus{}ok}\PY{l+s}{ }\PY{k+kr}{content}\PY{l+s}{ }\PY{k+kt}{boolean}

\PY{k+kr}{scope} \PY{n+nc}{A}\PY{o}{:}
\PY{k+kr}{definition} \PY{n+nv}{literate\PYZus{}parsing\PYZus{}is\PYZus{}ok} \PY{k+kr}{equals} \PY{k+kc}{true}
\```
\PY{k+kr}{scope}\PY{l+s}{ }\PY{n+nc}{A}\PY{o}{:}
\PY{l+s}{ }\PY{l+s}{ }\PY{k+kr}{definition}\PY{l+s}{ }\PY{n+nv}{literate\PYZus{}parsing\PYZus{}is\PYZus{}ok}\PY{l+s}{ }\PY{k+kr}{equals}\PY{l+s}{ }\PY{k+kc}{true}
\PY{l+s}{```}
\end{Verbatim}


Expand Down

0 comments on commit 517325c

Please sign in to comment.