Merge pull request #60 from jansorg/jansorg/path-antiquotations

Support antiquotations in paths

Author: JojOatXGME

src/main/java/org/nixos/idea/lang/NixParserDefinition.java

@@ -77,6 +77,14 @@ public SpaceRequirements spaceExistenceTypeBetweenTokens(ASTNode left, ASTNode r
         if (leftType == NixTypes.DOLLAR && rightType == NixTypes.LCURLY) {
             return SpaceRequirements.MUST_NOT;
         }
+        else if (leftType == NixTypes.PATH_SEGMENT) {
+            // path segment, antiquotation or PATH_END on the right
+            return SpaceRequirements.MUST_NOT;
+        }
+        else if (rightType == NixTypes.PATH_END) {
+            // path segment or antiquotation on the left
+            return SpaceRequirements.MUST_NOT;
+        }
         else if (NixTypeUtil.MIGHT_COLLAPSE_WITH_ID.contains(leftType) &&
                  NixTypeUtil.MIGHT_COLLAPSE_WITH_ID.contains(rightType)) {
             return SpaceRequirements.MUST;

src/main/java/org/nixos/idea/lang/highlighter/NixSyntaxHighlighter.java

@@ -66,8 +66,7 @@ public class NixSyntaxHighlighter extends SyntaxHighlighterBase {
             // Literals
             entry(NixTypes.INT, NixTextAttributes.NUMBER),
             entry(NixTypes.FLOAT, NixTextAttributes.NUMBER),
-            entry(NixTypes.PATH, NixTextAttributes.PATH),
-            entry(NixTypes.HPATH, NixTextAttributes.PATH),
+            entry(NixTypes.PATH_SEGMENT, NixTextAttributes.PATH),
             entry(NixTypes.SPATH, NixTextAttributes.PATH),
             entry(NixTypes.URI, NixTextAttributes.URI),
             // String literals

src/main/java/org/nixos/idea/psi/NixTypeUtil.java

@@ -24,9 +24,9 @@ public final class NixTypeUtil {
                     NixTypes.ID,
                     NixTypes.INT,
                     NixTypes.FLOAT,
-                    NixTypes.PATH,
-                    NixTypes.HPATH,
                     NixTypes.SPATH,
+                    NixTypes.PATH_SEGMENT,
+                    NixTypes.PATH_END,
                     NixTypes.URI));
 
     private NixTypeUtil() {} // Cannot be instantiated

src/main/java/org/nixos/idea/psi/impl/NixParserUtil.java

@@ -3,6 +3,7 @@
 import com.intellij.lang.PsiBuilder;
 import com.intellij.lang.parser.GeneratedParserUtilBase;
 import com.intellij.openapi.util.Key;
+import com.intellij.psi.TokenType;
 import org.jetbrains.annotations.NotNull;
 
 public final class NixParserUtil extends GeneratedParserUtilBase {

src/main/lang/Nix.bnf

@@ -169,7 +169,9 @@ expr_simple ::=
   | list
   | legacy_let
 identifier ::= ID
-literal ::= INT | FLOAT | PATH | HPATH | SPATH | URI
+literal ::= INT | FLOAT | URI | path
+;{ extends("path")=literal }
+path ::= SPATH | PATH_SEGMENT (PATH_SEGMENT | antiquotation)* PATH_END
 parens ::= LPAREN expr recover_parens RPAREN { pin=1 }
 set ::= [ REC ] LCURLY recover_set (bind recover_set)* RCURLY { pin=2 }
 list ::= LBRAC recover_list (expr_select recover_list)* RBRAC { pin=1 }

src/main/lang/Nix.flex

@@ -43,15 +43,17 @@ import static org.nixos.idea.psi.NixTypes.*;
 %function advance
 %type IElementType
 %unicode
-%state BLOCK STRING IND_STRING ANTIQUOTATION_START ANTIQUOTATION
+%state BLOCK STRING IND_STRING ANTIQUOTATION_START ANTIQUOTATION PATH
 
 ANY=[^]
 ID=[a-zA-Z_][a-zA-Z0-9_'-]*
 INT=[0-9]+
 FLOAT=(([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?
-PATH=[a-zA-Z0-9._+-]*(\/[a-zA-Z0-9._+-]+)+\/?
-HPATH=\~(\/[a-zA-Z0-9._+-]+)+\/?
-SPATH=\<[a-zA-Z0-9._+-]+(\/[a-zA-Z0-9._+-]+)*\>
+PATH_CHAR=[a-zA-Z0-9\.\_\-\+]
+PATH={PATH_CHAR}*(\/{PATH_CHAR}+)+\/?
+PATH_SEG={PATH_CHAR}*\/
+HPATH_START=\~\/
+SPATH=\<{PATH_CHAR}+(\/{PATH_CHAR}+)*\>
 URI=[a-zA-Z][a-zA-Z0-9.+-]*\:[a-zA-Z0-9%/?:@&=+$,\-_.!~*']+
 
 WHITE_SPACE=[\ \t\r\n]+
@@ -91,6 +93,16 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/
   "}"                   { popState(BLOCK); return RCURLY; }
 }
 
+<PATH> {
+  "$"/"{"               { pushState(ANTIQUOTATION_START); return DOLLAR; }
+  {PATH_SEG}            { return PATH_SEGMENT; }
+  {PATH_CHAR}+          { return PATH_SEGMENT; }
+  // anything else, e.g. whitespace, stops lexing of a PATH
+  // we're delegating back to the parent state
+  // PATH_END is an empty-length token to signal the end of the path
+  [^]                   { popState(PATH); yypushback(yylength()); return PATH_END; }
+}
+
 <YYINITIAL, BLOCK, ANTIQUOTATION> {
   "if"                  { return IF; }
   "then"                { return THEN; }
@@ -146,8 +158,10 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/
   {ID}                  { return ID; }
   {INT}                 { return INT; }
   {FLOAT}               { return FLOAT; }
-  {PATH}                { return PATH; }
-  {HPATH}               { return HPATH; }
+  "/" / "${"            { pushState(PATH); return PATH_SEGMENT; }
+  {PATH}
+  | {PATH_SEG}
+  | {HPATH_START}       { pushState(PATH); return PATH_SEGMENT; }
   {SPATH}               { return SPATH; }
   {URI}                 { return URI; }
 

src/test/java/org/nixos/idea/lang/ParsingTest.java

@@ -145,10 +145,13 @@ public void testWith() {
 
   @Override
   protected void tearDown() throws Exception {
-    // Ensure that the parser does not generate errors even when the errors have
-    // accidentally been added to the expected result.
-    ensureNoErrorElements();
-    super.tearDown();
+    try {
+      // Ensure that the parser does not generate errors even when the errors have
+      // accidentally been added to the expected result.
+      ensureNoErrorElements();
+    } finally {
+      super.tearDown();
+    }
   }
 
   @Override

src/test/java/org/nixos/idea/lang/highlighter/NixSyntaxHighlighterTest.java

@@ -3,6 +3,7 @@
 import com.intellij.openapi.editor.colors.TextAttributesKey;
 import com.intellij.psi.TokenType;
 import com.intellij.psi.tree.IElementType;
+import com.intellij.psi.tree.TokenSet;
 import org.jetbrains.annotations.NotNull;
 import org.junit.jupiter.api.Named;
 import org.junit.jupiter.api.Test;
@@ -15,12 +16,12 @@
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
-import static org.junit.jupiter.api.Assertions.assertAll;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static java.util.function.Predicate.not;
+import static org.junit.jupiter.api.Assertions.*;
 
 final class NixSyntaxHighlighterTest {
+    private static final TokenSet EMPTY_LENGTH_TOKENS = TokenSet.create(NixTypes.PATH_END);
+
     @Test
     void testAttributesKeysForUnknownTokenType() {
         TextAttributesKey[] tokenHighlights = new NixSyntaxHighlighter().getTokenHighlights(TokenType.CODE_FRAGMENT);
@@ -38,9 +39,10 @@ void testAttributesKeysForKnownTokenTypes(@NotNull IElementType tokenType) {
                 () -> assertNotNull(tokenHighlights[index], String.format("tokenHighlights[%d]", index))));
     }
 
-    static @NotNull Stream<Named<IElementType>> testAttributesKeysForKnownTokenTypes() {
+    static @NotNull Stream<Named<? extends IElementType>> testAttributesKeysForKnownTokenTypes() {
         return Stream.concat(
                 Stream.of(Named.of("TokenType.BAD_CHARACTER", TokenType.BAD_CHARACTER)),
-                ReflectionUtils.getPublicStaticFieldValues(NixTypes.class, NixTokenType.class));
+                ReflectionUtils.getPublicStaticFieldValues(NixTypes.class, NixTokenType.class)
+        ).filter(not(named -> EMPTY_LENGTH_TOKENS.contains(named.getPayload())));
     }
 }

src/test/testData/ParsingTest/List.lexer.txt

@@ -2,7 +2,8 @@
 WHITE_SPACE (' ')
 INT ('123')
 WHITE_SPACE (' ')
-PATH ('./foo.nix')
+PATH_SEGMENT ('./foo.nix')
+PATH_END ('')
 WHITE_SPACE (' ')
 STRING_OPEN ('"')
 STR ('abc')

src/test/testData/ParsingTest/List.txt

@@ -5,8 +5,8 @@ Nix File(0,36)
     NixLiteralImpl(LITERAL)(2,5)
       PsiElement(INT)('123')(2,5)
     PsiWhiteSpace(' ')(5,6)
-    NixLiteralImpl(LITERAL)(6,15)
-      PsiElement(PATH)('./foo.nix')(6,15)
+    NixPathImpl(PATH)(6,15)
+      PsiElement(PATH_SEGMENT)('./foo.nix')(6,15)
     PsiWhiteSpace(' ')(15,16)
     NixStdStringImpl(STD_STRING)(16,21)
       PsiElement(STRING_OPEN)('"')(16,17)
@@ -33,4 +33,4 @@ Nix File(0,36)
       PsiWhiteSpace(' ')(32,33)
       PsiElement(})('}')(33,34)
     PsiWhiteSpace(' ')(34,35)
-    PsiElement(])(']')(35,36)
+    PsiElement(])(']')(35,36)