Collapse value and scope checks to single line and remove need to escape quotes

Author: Miles Richardson

spec/grammar-spec.coffee

@@ -14,44 +14,31 @@ describe "Clojure grammar", ->
 
   it "tokenizes semi-colon comments", ->
     {tokens} = grammar.tokenizeLine "; clojure"
-    expect(tokens[0].value).toEqual ";"
-    expect(tokens[0].scopes).toEqual ["source.clojure", "comment.line.semicolon.clojure", "punctuation.definition.comment.clojure"]
-    expect(tokens[1].value).toEqual " clojure"
-    expect(tokens[1].scopes).toEqual ["source.clojure", "comment.line.semicolon.clojure"]
+    expect(tokens[0]).toEqual value: ";", scopes: ["source.clojure", "comment.line.semicolon.clojure", "punctuation.definition.comment.clojure"]
+    expect(tokens[1]).toEqual value: " clojure", scopes: ["source.clojure", "comment.line.semicolon.clojure"]
 
   it "tokenizes shebang comments", ->
     {tokens} = grammar.tokenizeLine "#!/usr/bin/env clojure"
-    expect(tokens[0].value).toEqual "#!"
-    expect(tokens[0].scopes).toEqual ["source.clojure", "comment.line.semicolon.clojure", "punctuation.definition.comment.shebang.clojure"]
-    expect(tokens[1].value).toEqual "/usr/bin/env clojure"
-    expect(tokens[1].scopes).toEqual ["source.clojure", "comment.line.semicolon.clojure"]
+    expect(tokens[0]).toEqual value: "#!", scopes: ["source.clojure", "comment.line.semicolon.clojure", "punctuation.definition.comment.shebang.clojure"]
+    expect(tokens[1]).toEqual value: "/usr/bin/env clojure", scopes: ["source.clojure", "comment.line.semicolon.clojure"]
 
   it "tokenizes strings", ->
-    {tokens} = grammar.tokenizeLine "\"foo bar\""
-    expect(tokens[0].value).toEqual "\""
-    expect(tokens[0].scopes).toEqual ["source.clojure", "string.quoted.double.clojure", "punctuation.definition.string.begin.clojure"]
-    expect(tokens[1].value).toEqual "foo bar"
-    expect(tokens[1].scopes).toEqual ["source.clojure", "string.quoted.double.clojure"]
-    expect(tokens[2].value).toEqual "\""
-    expect(tokens[2].scopes).toEqual ["source.clojure", "string.quoted.double.clojure", "punctuation.definition.string.end.clojure"]
+    {tokens} = grammar.tokenizeLine '"foo bar"'
+    expect(tokens[0]).toEqual value: '"', scopes: ["source.clojure", "string.quoted.double.clojure", "punctuation.definition.string.begin.clojure"]
+    expect(tokens[1]).toEqual value: 'foo bar', scopes: ["source.clojure", "string.quoted.double.clojure"]
+    expect(tokens[2]).toEqual value: '"', scopes: ["source.clojure", "string.quoted.double.clojure", "punctuation.definition.string.end.clojure"]
 
   it "tokenizes character escape sequences", ->
-    {tokens} = grammar.tokenizeLine "\"\\n\""
-    expect(tokens[0].value).toEqual "\""
-    expect(tokens[0].scopes).toEqual ["source.clojure", "string.quoted.double.clojure", "punctuation.definition.string.begin.clojure"]
-    expect(tokens[1].value).toEqual "\\n"
-    expect(tokens[1].scopes).toEqual ["source.clojure", "string.quoted.double.clojure", "constant.character.escape.clojure"]
-    expect(tokens[2].value).toEqual "\""
-    expect(tokens[2].scopes).toEqual ["source.clojure", "string.quoted.double.clojure", "punctuation.definition.string.end.clojure"]
+    {tokens} = grammar.tokenizeLine '"\\n"'
+    expect(tokens[0]).toEqual value: '"', scopes: ["source.clojure", "string.quoted.double.clojure", "punctuation.definition.string.begin.clojure"]
+    expect(tokens[1]).toEqual value: '\\n', scopes: ["source.clojure", "string.quoted.double.clojure", "constant.character.escape.clojure"]
+    expect(tokens[2]).toEqual value: '"', scopes: ["source.clojure", "string.quoted.double.clojure", "punctuation.definition.string.end.clojure"]
 
   it "tokenizes regexes", ->
-    {tokens} = grammar.tokenizeLine "#\"foo\""
-    expect(tokens[0].value).toEqual "#\""
-    expect(tokens[0].scopes).toEqual ["source.clojure", "string.regexp.clojure"]
-    expect(tokens[1].value).toEqual "foo"
-    expect(tokens[1].scopes).toEqual ["source.clojure", "string.regexp.clojure"]
-    expect(tokens[2].value).toEqual "\""
-    expect(tokens[2].scopes).toEqual ["source.clojure", "string.regexp.clojure"]
+    {tokens} = grammar.tokenizeLine '#"foo"'
+    expect(tokens[0]).toEqual value: '#"', scopes: ["source.clojure", "string.regexp.clojure"]
+    expect(tokens[1]).toEqual value: 'foo', scopes: ["source.clojure", "string.regexp.clojure"]
+    expect(tokens[2]).toEqual value: '"', scopes: ["source.clojure", "string.regexp.clojure"]
 
   it "tokenizes numerics", ->
     numbers =
@@ -67,8 +54,7 @@ describe "Clojure grammar", ->
     for scope, nums of numbers
       for num in nums
         {tokens} = grammar.tokenizeLine num
-        expect(tokens[0].value).toEqual num
-        expect(tokens[0].scopes).toEqual ["source.clojure", scope]
+        expect(tokens[0]).toEqual value: num, scopes: ["source.clojure", scope]
 
   it "tokenizes booleans", ->
     booleans =
@@ -77,13 +63,11 @@ describe "Clojure grammar", ->
     for scope, bools of booleans
       for bool in bools
         {tokens} = grammar.tokenizeLine bool
-        expect(tokens[0].value).toEqual bool
-        expect(tokens[0].scopes).toEqual ["source.clojure", scope]
+        expect(tokens[0]).toEqual value: bool, scopes: ["source.clojure", scope]
 
   it "tokenizes nil", ->
     {tokens} = grammar.tokenizeLine "nil"
-    expect(tokens[0].value).toEqual "nil"
-    expect(tokens[0].scopes).toEqual ["source.clojure", "constant.language.nil.clojure"]
+    expect(tokens[0]).toEqual value: "nil", scopes: ["source.clojure", "constant.language.nil.clojure"]
 
   it "tokenizes keywords", ->
     tests =
@@ -95,87 +79,67 @@ describe "Clojure grammar", ->
     for metaScope, lines of tests
       for line in lines
         {tokens} = grammar.tokenizeLine line
-        expect(tokens[1].value).toEqual ":foo"
-        expect(tokens[1].scopes).toEqual ["source.clojure", metaScope, "constant.keyword.clojure"]
+        expect(tokens[1]).toEqual value: ":foo", scopes: ["source.clojure", metaScope, "constant.keyword.clojure"]
 
   it "tokenizes keyfns (keyword control)", ->
     keyfns = ["declare", "declare-", "ns", "in-ns", "import", "use", "require", "load", "compile", "def", "defn", "defn-", "defmacro"]
 
     for keyfn in keyfns
       {tokens} = grammar.tokenizeLine "(#{keyfn})"
-      expect(tokens[1].value).toEqual keyfn
-      expect(tokens[1].scopes).toEqual ["source.clojure", "meta.expression.clojure", "keyword.control.clojure"]
+      expect(tokens[1]).toEqual value: keyfn, scopes: ["source.clojure", "meta.expression.clojure", "keyword.control.clojure"]
 
   it "tokenizes keyfns (storage control)", ->
     keyfns = ["if", "when", "for", "cond", "do", "let", "binding", "loop", "recur", "fn", "throw", "try", "catch", "finally", "case"]
 
     for keyfn in keyfns
       {tokens} = grammar.tokenizeLine "(#{keyfn})"
-      expect(tokens[1].value).toEqual keyfn
-      expect(tokens[1].scopes).toEqual ["source.clojure", "meta.expression.clojure", "storage.control.clojure"]
+      expect(tokens[1]).toEqual value: keyfn, scopes: ["source.clojure", "meta.expression.clojure", "storage.control.clojure"]
 
   it "tokenizes global definitions", ->
     {tokens} = grammar.tokenizeLine "(def foo 'bar)"
-    expect(tokens[1].value).toEqual "def"
-    expect(tokens[1].scopes).toEqual ["source.clojure", "meta.expression.clojure", "meta.definition.global.clojure", "keyword.control.clojure"]
-    expect(tokens[3].value).toEqual "foo"
-    expect(tokens[3].scopes).toEqual ["source.clojure", "meta.expression.clojure", "meta.definition.global.clojure", "entity.global.clojure"]
+    expect(tokens[1]).toEqual value: "def", scopes: ["source.clojure", "meta.expression.clojure", "meta.definition.global.clojure", "keyword.control.clojure"]
+    expect(tokens[3]).toEqual value: "foo", scopes: ["source.clojure", "meta.expression.clojure", "meta.definition.global.clojure", "entity.global.clojure"]
 
   it "tokenizes dynamic variables", ->
     mutables = ["*ns*", "*foo-bar*"]
 
     for mutable in mutables
       {tokens} = grammar.tokenizeLine mutable
-      expect(tokens[0].value).toEqual mutable
-      expect(tokens[0].scopes).toEqual ["source.clojure", "meta.symbol.dynamic.clojure"]
+      expect(tokens[0]).toEqual value: mutable, scopes: ["source.clojure", "meta.symbol.dynamic.clojure"]
 
   it "tokenizes metadata", ->
     {tokens} = grammar.tokenizeLine "^Foo"
-    expect(tokens[0].value).toEqual "^"
-    expect(tokens[0].scopes).toEqual ["source.clojure", "meta.metadata.simple.clojure"]
-    expect(tokens[1].value).toEqual "Foo"
-    expect(tokens[1].scopes).toEqual ["source.clojure", "meta.metadata.simple.clojure", "meta.symbol.clojure"]
+    expect(tokens[0]).toEqual value: "^", scopes: ["source.clojure", "meta.metadata.simple.clojure"]
+    expect(tokens[1]).toEqual value: "Foo", scopes: ["source.clojure", "meta.metadata.simple.clojure", "meta.symbol.clojure"]
 
     {tokens} = grammar.tokenizeLine "^{:foo true}"
-    expect(tokens[0].value).toEqual "^{"
-    expect(tokens[0].scopes).toEqual ["source.clojure", "meta.metadata.map.clojure", "punctuation.section.metadata.map.begin.clojure"]
-    expect(tokens[1].value).toEqual ":foo"
-    expect(tokens[1].scopes).toEqual ["source.clojure", "meta.metadata.map.clojure", "constant.keyword.clojure"]
-    expect(tokens[2].value).toEqual " "
-    expect(tokens[2].scopes).toEqual ["source.clojure", "meta.metadata.map.clojure"]
-    expect(tokens[3].value).toEqual "true"
-    expect(tokens[3].scopes).toEqual ["source.clojure", "meta.metadata.map.clojure", "constant.language.boolean.clojure"]
-    expect(tokens[4].value).toEqual "}"
-    expect(tokens[4].scopes).toEqual ["source.clojure", "meta.metadata.map.clojure", "punctuation.section.metadata.map.end.clojure"]
+    expect(tokens[0]).toEqual value: "^{", scopes: ["source.clojure", "meta.metadata.map.clojure", "punctuation.section.metadata.map.begin.clojure"]
+    expect(tokens[1]).toEqual value: ":foo", scopes: ["source.clojure", "meta.metadata.map.clojure", "constant.keyword.clojure"]
+    expect(tokens[2]).toEqual value: " ", scopes: ["source.clojure", "meta.metadata.map.clojure"]
+    expect(tokens[3]).toEqual value: "true", scopes: ["source.clojure", "meta.metadata.map.clojure", "constant.language.boolean.clojure"]
+    expect(tokens[4]).toEqual value: "}", scopes: ["source.clojure", "meta.metadata.map.clojure", "punctuation.section.metadata.map.end.clojure"]
 
   it "tokenizes functions", ->
     expressions = ["(foo)", "(foo 1 10)"]
 
     for expr in expressions
       {tokens} = grammar.tokenizeLine expr
-      expect(tokens[1].value).toEqual "foo"
-      expect(tokens[1].scopes).toEqual ["source.clojure", "meta.expression.clojure", "entity.name.function.clojure"]
+      expect(tokens[1]).toEqual value: "foo", scopes: ["source.clojure", "meta.expression.clojure", "entity.name.function.clojure"]
 
   it "tokenizes vars", ->
     {tokens} = grammar.tokenizeLine "(func #'foo)"
-    expect(tokens[2].value).toEqual " #"
-    expect(tokens[2].scopes).toEqual ["source.clojure", "meta.expression.clojure"]
-    expect(tokens[3].value).toEqual "'foo"
-    expect(tokens[3].scopes).toEqual ["source.clojure", "meta.expression.clojure", "meta.var.clojure"]
+    expect(tokens[2]).toEqual value: " #", scopes: ["source.clojure", "meta.expression.clojure"]
+    expect(tokens[3]).toEqual value: "'foo", scopes: ["source.clojure", "meta.expression.clojure", "meta.var.clojure"]
 
   it "tokenizes symbols", ->
     {tokens} = grammar.tokenizeLine "foo/bar"
-    expect(tokens[0].value).toEqual "foo"
-    expect(tokens[0].scopes).toEqual ["source.clojure", "meta.symbol.namespace.clojure"]
-    expect(tokens[1].value).toEqual "/"
-    expect(tokens[1].scopes).toEqual ["source.clojure"]
-    expect(tokens[2].value).toEqual "bar"
-    expect(tokens[2].scopes).toEqual ["source.clojure", "meta.symbol.clojure"]
+    expect(tokens[0]).toEqual value: "foo", scopes: ["source.clojure", "meta.symbol.namespace.clojure"]
+    expect(tokens[1]).toEqual value: "/", scopes: ["source.clojure"]
+    expect(tokens[2]).toEqual value: "bar", scopes: ["source.clojure", "meta.symbol.clojure"]
 
   it "tokenizes trailing whitespace", ->
     {tokens} = grammar.tokenizeLine "   \n"
-    expect(tokens[0].value).toEqual "   \n"
-    expect(tokens[0].scopes).toEqual ["source.clojure", "invalid.trailing-whitespace"]
+    expect(tokens[0]).toEqual value: "   \n", scopes: ["source.clojure", "invalid.trailing-whitespace"]
 
   # TODO: Test 'after' tokens?
   testMetaSection = (metaScope, puncScope, startsWith, endsWith) ->
@@ -184,11 +148,8 @@ describe "Clojure grammar", ->
 
     [start, mid..., end, after] = tokens
 
-    expect(start.value).toEqual startsWith
-    expect(start.scopes).toEqual ["source.clojure", "meta.#{metaScope}.clojure", "punctuation.section.#{puncScope}.begin.clojure"]
-
-    expect(end.value).toEqual endsWith
-    expect(end.scopes).toEqual ["source.clojure", "meta.#{metaScope}.clojure", "punctuation.section.#{puncScope}.end.clojure"]
+    expect(start).toEqual value: startsWith, scopes: ["source.clojure", "meta.#{metaScope}.clojure", "punctuation.section.#{puncScope}.begin.clojure"]
+    expect(end).toEqual value: endsWith, scopes: ["source.clojure", "meta.#{metaScope}.clojure", "punctuation.section.#{puncScope}.end.clojure"]
 
     for token in mid
       expect(token.scopes.slice(0, 2)).toEqual ["source.clojure", "meta.#{metaScope}.clojure"]
@@ -198,16 +159,14 @@ describe "Clojure grammar", ->
 
     [start, mid..., after] = tokens[0]
 
-    expect(start.value).toEqual startsWith
-    expect(start.scopes).toEqual ["source.clojure", "meta.#{metaScope}.clojure", "punctuation.section.#{puncScope}.begin.clojure"]
+    expect(start).toEqual value: startsWith, scopes: ["source.clojure", "meta.#{metaScope}.clojure", "punctuation.section.#{puncScope}.begin.clojure"]
 
     for token in mid
       expect(token.scopes.slice(0, 2)).toEqual ["source.clojure", "meta.#{metaScope}.clojure"]
 
     [mid..., end, after] = tokens[1]
 
-    expect(end.value).toEqual endsWith
-    expect(end.scopes).toEqual ["source.clojure", "meta.#{metaScope}.clojure", "punctuation.section.#{puncScope}.end.clojure"]
+    expect(end).toEqual value: endsWith, scopes: ["source.clojure", "meta.#{metaScope}.clojure", "punctuation.section.#{puncScope}.end.clojure"]
 
     for token in mid
       expect(token.scopes.slice(0, 2)).toEqual ["source.clojure", "meta.#{metaScope}.clojure"]