add python-style # comments to remespath

CHANGELOG.md

@@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
     * maybe only try to do this for files with the `.csv` and `.tsv` extensions
 	* only test the `,` and `\t` delimiters, and only the `"` or `'` quote characters
 	* test only the first 10KB of the file, or first 25 lines, whichever comes first.
+8. Unit tests that randomly generate text with JSON chars to make sure JSON parser never throws for any reason, since errors aren't caught.
 
 ### To Be Changed
 
@@ -39,6 +40,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### To Be Fixed
 
+- Fix issue where pretty-printing or compressing causes tree view position tracking to be out of sync with the document until a query is issued or the `Refresh` button is hit.
 - Improve Alt-key accelerators *in forms*. They don't seem to work right for some reason.
 - When a tree viewer is refreshed using JSON from a file with a different name, the title of the docking form that the user sees doesn't change to reflect the new file. For example, a tree viewer is opened up for `foo.json` and then refreshed with a buffer named `bar.json`, and the title of the docking form still reads `Json Tree View for foo.json`.
 	- This is also true if a file with a tree viewer is renamed, e.g., the file `foo.json` is renamed to `bar.json`, but the tree viewer still says `Json Tree View for foo.json`.
@@ -56,6 +58,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 1. A [RemesPath user-defined language (UDL) file](/RemesPath%20UDL.xml), providing some very basic syntax highlighting. It is buggy, but that is because the UDL system is inherently buggy, not because I did anything wrong (as far as I know).
 2. Add [`and` and `or` non-vectorized functions in RemesPath](/docs/RemesPath.md#non-vectorized-functions), which both use conditional excution.
 3. [PPrint-style pretty-printing that remembers comments](/docs/README.md#remember_comments)
+4. [Python-style single-line comments in RemesPath](/docs/RemesPath.md#comments-added-in-v62)
 
 ### Changed
 

JsonToolsNppPlugin/JSONTools/RemesPathLexer.cs

@@ -79,6 +79,7 @@ public RemesPathLexer()
             @"([,\[\]\(\)\{\}\.:=!;])|" + // delimiters
             @"([gjf]?(?<!\\)`(?:\\\\|\\`|[^`\r\n])*`)|" + // backtick strings
             $@"({JsonParser.UNQUOTED_START}(?:[\p{{Mn}}\p{{Mc}}\p{{Nd}}\p{{Pc}}\u200c\u200d]|{JsonParser.UNQUOTED_START})*)|" + // unquoted strings
+            @"\#([^\n]*)(?:\n|\z)|" + // comments (Python-style, single-line)
             @"(\S+)", // anything non-whitespace non-token stuff (will cause error)
             RegexOptions.Compiled
         );
@@ -174,8 +175,9 @@ public List<object> Tokenize(string q)
                         toks.Add(new JNode(enquoted));
                     break;
                 case 11: toks.Add(ParseUnquotedString(m.Value)); break;
+                case 12: /* toks.Add(new Comment(m.Value, false, m.Index)); */ break; // comments; currently we won't add them to the tokens, but maybe later?
                 default:
-                    throw new RemesLexerException($"Invalid token \"{m.Value}\" at position {m.Index}");
+                    throw new RemesLexerException(m.Index, q, $"Invalid token \"{m.Value}\"");
                 }
             }
             BraceMatchCheck(q, regtoks, toks, '(', ')');
@@ -254,7 +256,7 @@ public static void BraceMatchCheck(string q, MatchCollection regtoks, List<objec
                     if (c == open)
                     {
                         if (++unclosedCount >= MAX_RECURSION_DEPTH)
-                            throw new RemesLexerException($"Maximum recuresion depth ({MAX_RECURSION_DEPTH}) in a RemesPath query reached");
+                            throw new RemesLexerException(regtoks[regTokIndex].Index, q, $"Maximum recursion depth ({MAX_RECURSION_DEPTH}) in a RemesPath query reached");
                         else if (unclosedCount == 1)
                             lastUnclosed = regtoks[regTokIndex].Index;
                     }

JsonToolsNppPlugin/Properties/AssemblyInfo.cs

@@ -28,5 +28,5 @@
 //      Build Number
 //      Revision
 //
-[assembly: AssemblyVersion("6.1.1.12")]
-[assembly: AssemblyFileVersion("6.1.1.12")]
+[assembly: AssemblyVersion("6.1.1.13")]
+[assembly: AssemblyFileVersion("6.1.1.13")]

JsonToolsNppPlugin/Tests/RemesPathTests.cs

@@ -457,8 +457,15 @@ public static bool Test()
                 // parens tests
                 new Query_DesiredResult("(@.foo[:2])", "[[0, 1, 2], [3.0, 4.0, 5.0]]"),
                 new Query_DesiredResult("(@.foo)[0]", "[0, 1, 2]"),
+                new Query_DesiredResult("(@.foo # comment\n)[0]", "[0, 1, 2]"),
+                new Query_DesiredResult("(@.foo)[0] # comment at end", "[0, 1, 2]"),
+                new Query_DesiredResult("(@.foo)[0] # comment at end\n", "[0, 1, 2]"),
+                new Query_DesiredResult("(@.foo)[0] #", "[0, 1, 2]"), // empty comment at end
+                new Query_DesiredResult("# comment at start\n(@.foo)[0]", "[0, 1, 2]"),
+                new Query_DesiredResult("# #s in comment #\r\n(@.foo)[0]", "[0, 1, 2]"),
+                new Query_DesiredResult("# comment at start\r\n# another comment\n(@.foo)[0]# moar comments!\n\n#yeah!!", "[0, 1, 2]"),
                 // projection tests
-                new Query_DesiredResult("@{@.jub, @.quz}", "[[], {}]"),
+                new Query_DesiredResult("@    #\r\n{@.jub, @.quz}", "[[], {}]"), // comment is ignored
                 new Query_DesiredResult("@.foo{foo: @[0], bar: @[1][:2]}", "{\"foo\": [0, 1, 2], \"bar\": [3.0, 4.0]}"),
                 new Query_DesiredResult("sorted(flatten(@.guzo, 2)){`min`: @[0], `max`: @[-1], `tot`: sum(@)}", "{\"min\": 1, \"max\": 3, \"tot\": 6}"),
                 new Query_DesiredResult("(@.foo[:]{`max`: max(@), `min`: min(@)})[0]", "{\"max\": 2.0, \"min\": 0.0}"),
@@ -496,7 +503,7 @@ public static bool Test()
                 // making sure various escapes work in backtickstrings
                 new Query_DesiredResult("`\\t\\r\\n`", "\"\t\\r\\n\""),
                 // "->" (map) operator
-                new Query_DesiredResult("@ -> len(@)", fooLen.ToString()),
+                new Query_DesiredResult("@ -> #cmnt\n len(@)", fooLen.ToString()),
                 new Query_DesiredResult("@.foo[:] -> stringify(@)", "[\"[0,1,2]\", \"[3.0,4.0,5.0]\", \"[6.0,7.0,8.0]\"]"),
                 new Query_DesiredResult("@.* -> 3", "{\"foo\": 3, \"bar\": 3, \"baz\": 3, \"quz\": 3, \"jub\": 3, \"guzo\": 3, \"7\": 3, \"_\": 3}"),
                 new Query_DesiredResult("@.bar.* -> type(@)", "{\"a\": \"boolean\", \"b\": \"array\"}"),
@@ -1004,6 +1011,8 @@ public static bool Test()
                 new []{"var zuten = f`foo {@[0] = 3} bar`; @", "[1]"}, // mutation expression ('=' char) inside f-string interpolation
                 new []{"hundenheim + f`{@[0]; @[1]}` ", "[0, 1]"}, // multiple statements (';' char) inside an f-string interpolation
                 new []{"f`foo {} bar`", "[]"}, // empty interpolated section in f-string
+                new []{"# commment and nothing else", "[]" },
+                new []{"# commment and nothing else\r\n# and another comment", "[]"},
             };
             // test issue where sometimes a binop does not raise an error when it operates on two invalid types
             string[] invalidOthers = new string[] { "{}", "[]", "\"1\"" };
@@ -1175,10 +1184,10 @@ public static bool Test()
             var testcases = new Query_DesiredResult[]
             {
                 new Query_DesiredResult("var a = 1; " +
-                                        "var b = @.foo[0]; " +
+                                        "var b = @.foo # comment\r\n[0]; " +
                                         "var c = a + 2; " +
                                         "b = @ * c; " + // not reassigning b (use "var b = @ * c" for that), but rather mutating it
-                                        "@.foo[:][1]",
+                                        "@ #another comment\n.foo[:][ # more comments\n1]",
                     "[3, 4.0, 7.0]"),
                 new Query_DesiredResult("var two = @.foo[1]; " + // [3.0, 4.0, 5.0]
                                         "var one = @.foo[0]; " + // [0, 1, 2]
@@ -1187,6 +1196,7 @@ public static bool Test()
                                         "two",
                     "[8.0, 9.0, 10.0]"),
                 new Query_DesiredResult("var two = @.foo[1]; " + // [3.0, 4.0, 5.0]
+                                        "  # a comment;\r\n" +
                                         "var one = @.foo[0]; " + // [0, 1, 2]
                                         "var mintwo = min(two); " + // 3.0
                                         "var z = one[:]{@, @ - mintwo};", // [[0, -3.0], [1, -2.0], [2, -1.0]]

docs/RemesPath.md

@@ -1399,4 +1399,17 @@ __Notes:__
 Beginning in [v6.0](/CHANGELOG.md#600---2023-12-13), if a function has multiple optional arguments, you can leave any number of optional arguments (including the last) empty, rather than writing `null`.
 For example, if the function `foo` has two optional arguments:
 * `foo(1, , 2)` would be equivalent to `foo(1, null, 2)`
-* `foo(1, 2, )` would be equivalent to `foo(1, 2, null)` or `foo(1, 2)`.
+* `foo(1, 2, )` would be equivalent to `foo(1, 2, null)` or `foo(1, 2)`.
+
+## Comments (added in [v6.2](/CHANGELOG.md#620---unreleased-yyyy-mm-dd)) ##
+
+Beginning in [v6.2](/CHANGELOG.md#620---unreleased-yyyy-mm-dd), queries can include any number of Python-style single-line comments.
+
+Thus the query
+```
+foo # comment1
++ #comment2
+# comment3
+bar #comment4
+```
+would simply be parsed as `foo + bar`