1+ import ace.Tokenizer
2+
3+ /* *
4+ * A simple stateless tokenizer which can parse the tokens specified in [DemoToken].
5+ *
6+ * Lexing essentially tests for presence of all possible tokens and then greedily
7+ * reads as much as possible at once, since all the tokens are completely distinct.
8+ */
9+ object DemoTokenizer : Tokenizer<DemoToken, Unit>(null ) {
10+
11+ private val operators = charArrayOf(' +' , ' -' , ' *' , ' /' )
12+
13+ override fun getLineTokens (line : String , startState : Unit? ): Tokens <DemoToken , Unit > {
14+ // override default getLineTokens functionality with our custom parsing rules:
15+
16+ val tokens = ArrayList <DemoToken >()
17+ var text = line
18+ while (text.isNotEmpty()) {
19+ val token = when {
20+ text[0 ] in operators -> {
21+ // Token is an operator:
22+ DemoToken .Operator (text[0 ].toString())
23+ }
24+ text[0 ] == ' (' || text[0 ] == ' )' -> {
25+ // Token is parenthesis:
26+ DemoToken .Parenthesis (text[0 ].toString())
27+ }
28+ text[0 ].isWhitespace() -> {
29+ // Token is whitespace:
30+ val token = text.takeWhile { it.isWhitespace() }
31+ DemoToken .Whitespace (token)
32+ }
33+ text[0 ] in ' 0' .. ' 9' -> {
34+ // Token is a number:
35+ val token = text.takeWhile { it in ' 0' .. ' 9' }
36+ DemoToken .Number (token)
37+ }
38+ else -> {
39+ // We don't know what this is:
40+ val token = text.takeWhile {
41+ it !in ' 0' .. ' 9'
42+ && it !in operators
43+ && it != ' (' && it != ' )'
44+ && ! it.isWhitespace()
45+ }
46+ DemoToken .Unknown (token)
47+ }
48+ }
49+ tokens.add(token)
50+ text = text.drop(token.value.length)
51+ }
52+
53+ return DemoTokens (null , tokens.toTypedArray())
54+ }
55+
56+ private class DemoTokens (
57+ override val state : Unit? ,
58+ override val tokens : Array <DemoToken >
59+ ) : Tokens<DemoToken, Unit>
60+
61+ }
0 commit comments