Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 9edeed9

Browse files
gregshgregsh
committedMay 29, 2016
JFlex: 1.6 syntax JetBrains#81 
1 parent 864cdaf commit 9edeed9

37 files changed

+2888
-3154
lines changed
 

‎gen/org/intellij/jflex/parser/JFlexParser.java

Lines changed: 195 additions & 265 deletions
Large diffs are not rendered by default.

‎gen/org/intellij/jflex/parser/_JFlexLexer.java

Lines changed: 1063 additions & 576 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎gen/org/intellij/jflex/psi/JFlexClassExpression.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,7 @@
2121

2222
public interface JFlexClassExpression extends JFlexExpression {
2323

24+
@NotNull
25+
List<JFlexClassExpression> getClassExpressionList();
26+
2427
}

‎gen/org/intellij/jflex/psi/JFlexDeclarationsSection.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
import org.jetbrains.annotations.*;
2020
import com.intellij.psi.PsiElement;
2121

22-
public interface JFlexDeclarationsSection extends JFlexCompositeElement {
22+
public interface JFlexDeclarationsSection extends JFlexFileSection {
2323

2424
@NotNull
2525
List<JFlexMacroDefinition> getMacroDefinitionList();

‎gen/org/intellij/jflex/psi/JFlexFileSection.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Copyright 2011-present Greg Shrago
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.intellij.jflex.psi;
17+
18+
import java.util.List;
19+
import org.jetbrains.annotations.*;
20+
import com.intellij.psi.PsiElement;
21+
22+
public interface JFlexFileSection extends JFlexCompositeElement {
23+
24+
}

‎gen/org/intellij/jflex/psi/JFlexLexicalRulesSection.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
import org.jetbrains.annotations.*;
2020
import com.intellij.psi.PsiElement;
2121

22-
public interface JFlexLexicalRulesSection extends JFlexCompositeElement {
22+
public interface JFlexLexicalRulesSection extends JFlexFileSection {
2323

2424
@NotNull
2525
List<JFlexOption> getOptionList();

‎gen/org/intellij/jflex/psi/JFlexLiteralExpression.java

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -21,19 +21,4 @@
2121

2222
public interface JFlexLiteralExpression extends JFlexExpression {
2323

24-
@Nullable
25-
PsiElement getChar();
26-
27-
@Nullable
28-
PsiElement getEscapedChar();
29-
30-
@Nullable
31-
PsiElement getId();
32-
33-
@Nullable
34-
PsiElement getNumber();
35-
36-
@Nullable
37-
PsiElement getString();
38-
3924
}

‎gen/org/intellij/jflex/psi/JFlexRule.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ public interface JFlexRule extends JFlexCompositeElement {
3636
@NotNull
3737
List<JFlexRule> getRuleList();
3838

39-
@NotNull
40-
List<JFlexStateReference> getStateReferenceList();
39+
@Nullable
40+
JFlexStateList getStateList();
4141

4242
}

‎gen/org/intellij/jflex/psi/JFlexStateList.java

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* Copyright 2011-present Greg Shrago
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.intellij.jflex.psi;
17+
18+
import java.util.List;
19+
import org.jetbrains.annotations.*;
20+
import com.intellij.psi.PsiElement;
21+
22+
public interface JFlexStateList extends JFlexCompositeElement {
23+
24+
@NotNull
25+
List<JFlexStateReference> getStateReferenceList();
26+
27+
}

‎gen/org/intellij/jflex/psi/JFlexTypes.java

Lines changed: 18 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -43,44 +43,43 @@ public interface JFlexTypes {
4343
IElementType FLEX_SEQUENCE_EXPRESSION = new JFlexCompositeElementType("FLEX_SEQUENCE_EXPRESSION");
4444
IElementType FLEX_STATE_DECLARATION = new JFlexCompositeElementType("FLEX_STATE_DECLARATION");
4545
IElementType FLEX_STATE_DEFINITION = new JFlexCompositeElementType("FLEX_STATE_DEFINITION");
46+
IElementType FLEX_STATE_LIST = new JFlexCompositeElementType("FLEX_STATE_LIST");
4647
IElementType FLEX_STATE_REFERENCE = new JFlexCompositeElementType("FLEX_STATE_REFERENCE");
4748
IElementType FLEX_USER_CODE_SECTION = new JFlexCompositeElementType("FLEX_USER_CODE_SECTION");
4849
IElementType FLEX_USER_VALUE = new JFlexCompositeElementType("FLEX_USER_VALUE");
4950

51+
IElementType FLEX_AMPAMP = new JFlexTokenType("&&");
5052
IElementType FLEX_ANGLE1 = new JFlexTokenType("<");
5153
IElementType FLEX_ANGLE2 = new JFlexTokenType(">");
54+
IElementType FLEX_BANG = new JFlexTokenType("!");
55+
IElementType FLEX_BAR = new JFlexTokenType("|");
56+
IElementType FLEX_BARBAR = new JFlexTokenType("||");
5257
IElementType FLEX_BLOCK_COMMENT = new JFlexTokenType("block_comment");
5358
IElementType FLEX_BRACE1 = new JFlexTokenType("{");
5459
IElementType FLEX_BRACE2 = new JFlexTokenType("}");
5560
IElementType FLEX_BRACK1 = new JFlexTokenType("[");
5661
IElementType FLEX_BRACK2 = new JFlexTokenType("]");
62+
IElementType FLEX_BSLASH = new JFlexTokenType("\\");
5763
IElementType FLEX_CHAR = new JFlexTokenType("char");
58-
IElementType FLEX_CLASS_D = new JFlexTokenType("[:digit:]");
59-
IElementType FLEX_CLASS_JL = new JFlexTokenType("[:jletter:]");
60-
IElementType FLEX_CLASS_JLD = new JFlexTokenType("[:jletterdigit:]");
61-
IElementType FLEX_CLASS_L = new JFlexTokenType("[:letter:]");
62-
IElementType FLEX_CLASS_LL = new JFlexTokenType("[:lowercase:]");
63-
IElementType FLEX_CLASS_LU = new JFlexTokenType("[:uppercase:]");
64+
IElementType FLEX_CHAR_CLASS = new JFlexTokenType("char_class");
6465
IElementType FLEX_COMMA = new JFlexTokenType(",");
6566
IElementType FLEX_DASH = new JFlexTokenType("-");
67+
IElementType FLEX_DASHDASH = new JFlexTokenType("--");
6668
IElementType FLEX_DOLLAR = new JFlexTokenType("$");
6769
IElementType FLEX_DOT = new JFlexTokenType(".");
6870
IElementType FLEX_EOF = new JFlexTokenType("<<EOF>>");
6971
IElementType FLEX_EQ = new JFlexTokenType("=");
70-
IElementType FLEX_ESCAPED_CHAR = new JFlexTokenType("escaped_char");
72+
IElementType FLEX_FSLASH = new JFlexTokenType("/");
73+
IElementType FLEX_HAT = new JFlexTokenType("^");
7174
IElementType FLEX_ID = new JFlexTokenType("id");
72-
IElementType FLEX_JAVA = new JFlexTokenType("java");
7375
IElementType FLEX_LINE_COMMENT = new JFlexTokenType("line_comment");
74-
IElementType FLEX_NOT = new JFlexTokenType("!");
75-
IElementType FLEX_NOT2 = new JFlexTokenType("~");
7676
IElementType FLEX_NUMBER = new JFlexTokenType("number");
7777
IElementType FLEX_OPT16BIT = new JFlexTokenType("%16bit");
7878
IElementType FLEX_OPT_7BIT = new JFlexTokenType("%7bit");
7979
IElementType FLEX_OPT_8BIT = new JFlexTokenType("%8bit");
8080
IElementType FLEX_OPT_ABSTRACT = new JFlexTokenType("%abstract");
8181
IElementType FLEX_OPT_APIPRIVATE = new JFlexTokenType("%apiprivate");
8282
IElementType FLEX_OPT_BUFFER = new JFlexTokenType("%buffer");
83-
IElementType FLEX_OPT_BYACC = new JFlexTokenType("%byacc");
8483
IElementType FLEX_OPT_CASELESS = new JFlexTokenType("%caseless");
8584
IElementType FLEX_OPT_CHAR = new JFlexTokenType("%char");
8685
IElementType FLEX_OPT_CLASS = new JFlexTokenType("%class");
@@ -117,33 +116,29 @@ public interface JFlexTypes {
117116
IElementType FLEX_OPT_INTWRAP = new JFlexTokenType("%intwrap");
118117
IElementType FLEX_OPT_LINE = new JFlexTokenType("%line");
119118
IElementType FLEX_OPT_NOTUNIX = new JFlexTokenType("%notunix");
120-
IElementType FLEX_OPT_PACK = new JFlexTokenType("%pack");
121119
IElementType FLEX_OPT_PUBLIC = new JFlexTokenType("%public");
122-
IElementType FLEX_OPT_S = new JFlexTokenType("%s");
123120
IElementType FLEX_OPT_SCANERROR = new JFlexTokenType("%scanerror");
124121
IElementType FLEX_OPT_STANDALONE = new JFlexTokenType("%standalone");
125122
IElementType FLEX_OPT_STATE = new JFlexTokenType("%state");
126-
IElementType FLEX_OPT_SWITCH = new JFlexTokenType("%switch");
127-
IElementType FLEX_OPT_TABLE = new JFlexTokenType("%table");
128123
IElementType FLEX_OPT_TYPE = new JFlexTokenType("%type");
129124
IElementType FLEX_OPT_UNICODE = new JFlexTokenType("%unicode");
130-
IElementType FLEX_OPT_X = new JFlexTokenType("%x");
131125
IElementType FLEX_OPT_XSTATE = new JFlexTokenType("%xstate");
132126
IElementType FLEX_OPT_YYEOF = new JFlexTokenType("%yyeof");
133127
IElementType FLEX_OPT_YYLEXTHROW = new JFlexTokenType("%yylexthrow");
134128
IElementType FLEX_OPT_YYLEXTHROW1 = new JFlexTokenType("%yylexthrow{");
135129
IElementType FLEX_OPT_YYLEXTHROW2 = new JFlexTokenType("%yylexthrow}");
136-
IElementType FLEX_OR = new JFlexTokenType("|");
137130
IElementType FLEX_PAREN1 = new JFlexTokenType("(");
138131
IElementType FLEX_PAREN2 = new JFlexTokenType(")");
139132
IElementType FLEX_PLUS = new JFlexTokenType("+");
140133
IElementType FLEX_QUESTION = new JFlexTokenType("?");
141-
IElementType FLEX_ROOF = new JFlexTokenType("^");
142-
IElementType FLEX_SLASH = new JFlexTokenType("\\");
143-
IElementType FLEX_SLASH2 = new JFlexTokenType("/");
134+
IElementType FLEX_RAW = new JFlexTokenType("code block");
144135
IElementType FLEX_STAR = new JFlexTokenType("*");
145136
IElementType FLEX_STRING = new JFlexTokenType("string");
137+
IElementType FLEX_TILDE = new JFlexTokenType("~");
138+
IElementType FLEX_TILDETILDE = new JFlexTokenType("~~");
146139
IElementType FLEX_TWO_PERCS = new JFlexTokenType("%%");
140+
IElementType FLEX_UNCLOSED = new JFlexTokenType("unclosed");
141+
IElementType FLEX_VERSION = new JFlexTokenType("version");
147142

148143
class Factory {
149144
public static PsiElement createElement(ASTNode node) {
@@ -211,6 +206,9 @@ else if (type == FLEX_STATE_DECLARATION) {
211206
else if (type == FLEX_STATE_DEFINITION) {
212207
return new JFlexStateDefinitionImpl(node);
213208
}
209+
else if (type == FLEX_STATE_LIST) {
210+
return new JFlexStateListImpl(node);
211+
}
214212
else if (type == FLEX_STATE_REFERENCE) {
215213
return new JFlexStateReferenceImpl(node);
216214
}

‎gen/org/intellij/jflex/psi/JFlexUserCodeSection.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
import org.jetbrains.annotations.*;
2020
import com.intellij.psi.PsiElement;
2121

22-
public interface JFlexUserCodeSection extends JFlexCompositeElement {
22+
public interface JFlexUserCodeSection extends JFlexFileSection {
2323

2424
@Nullable
2525
JFlexJavaCode getJavaCode();

‎gen/org/intellij/jflex/psi/impl/JFlexClassExpressionImpl.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,4 +39,10 @@ public void accept(@NotNull PsiElementVisitor visitor) {
3939
else super.accept(visitor);
4040
}
4141

42+
@Override
43+
@NotNull
44+
public List<JFlexClassExpression> getClassExpressionList() {
45+
return PsiTreeUtil.getChildrenOfTypeAsList(this, JFlexClassExpression.class);
46+
}
47+
4248
}

‎gen/org/intellij/jflex/psi/impl/JFlexDeclarationsSectionImpl.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
import static org.intellij.jflex.psi.JFlexTypes.*;
2525
import org.intellij.jflex.psi.*;
2626

27-
public class JFlexDeclarationsSectionImpl extends JFlexCompositeElementImpl implements JFlexDeclarationsSection {
27+
public class JFlexDeclarationsSectionImpl extends JFlexFileSectionImpl implements JFlexDeclarationsSection {
2828

2929
public JFlexDeclarationsSectionImpl(ASTNode node) {
3030
super(node);

‎gen/org/intellij/jflex/psi/impl/JFlexFileSectionImpl.java

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Copyright 2011-present Greg Shrago
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.intellij.jflex.psi.impl;
17+
18+
import java.util.List;
19+
import org.jetbrains.annotations.*;
20+
import com.intellij.lang.ASTNode;
21+
import com.intellij.psi.PsiElement;
22+
import com.intellij.psi.PsiElementVisitor;
23+
import com.intellij.psi.util.PsiTreeUtil;
24+
import static org.intellij.jflex.psi.JFlexTypes.*;
25+
import org.intellij.jflex.psi.*;
26+
27+
public class JFlexFileSectionImpl extends JFlexCompositeElementImpl implements JFlexFileSection {
28+
29+
public JFlexFileSectionImpl(ASTNode node) {
30+
super(node);
31+
}
32+
33+
public void accept(@NotNull JFlexVisitor visitor) {
34+
visitor.visitFileSection(this);
35+
}
36+
37+
public void accept(@NotNull PsiElementVisitor visitor) {
38+
if (visitor instanceof JFlexVisitor) accept((JFlexVisitor)visitor);
39+
else super.accept(visitor);
40+
}
41+
42+
}

‎gen/org/intellij/jflex/psi/impl/JFlexLexicalRulesSectionImpl.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
import static org.intellij.jflex.psi.JFlexTypes.*;
2525
import org.intellij.jflex.psi.*;
2626

27-
public class JFlexLexicalRulesSectionImpl extends JFlexCompositeElementImpl implements JFlexLexicalRulesSection {
27+
public class JFlexLexicalRulesSectionImpl extends JFlexFileSectionImpl implements JFlexLexicalRulesSection {
2828

2929
public JFlexLexicalRulesSectionImpl(ASTNode node) {
3030
super(node);

‎gen/org/intellij/jflex/psi/impl/JFlexLiteralExpressionImpl.java

Lines changed: 0 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -39,34 +39,4 @@ public void accept(@NotNull PsiElementVisitor visitor) {
3939
else super.accept(visitor);
4040
}
4141

42-
@Override
43-
@Nullable
44-
public PsiElement getChar() {
45-
return findChildByType(FLEX_CHAR);
46-
}
47-
48-
@Override
49-
@Nullable
50-
public PsiElement getEscapedChar() {
51-
return findChildByType(FLEX_ESCAPED_CHAR);
52-
}
53-
54-
@Override
55-
@Nullable
56-
public PsiElement getId() {
57-
return findChildByType(FLEX_ID);
58-
}
59-
60-
@Override
61-
@Nullable
62-
public PsiElement getNumber() {
63-
return findChildByType(FLEX_NUMBER);
64-
}
65-
66-
@Override
67-
@Nullable
68-
public PsiElement getString() {
69-
return findChildByType(FLEX_STRING);
70-
}
71-
7242
}

‎gen/org/intellij/jflex/psi/impl/JFlexRuleImpl.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,9 @@ public List<JFlexRule> getRuleList() {
7070
}
7171

7272
@Override
73-
@NotNull
74-
public List<JFlexStateReference> getStateReferenceList() {
75-
return PsiTreeUtil.getChildrenOfTypeAsList(this, JFlexStateReference.class);
73+
@Nullable
74+
public JFlexStateList getStateList() {
75+
return findChildByClass(JFlexStateList.class);
7676
}
7777

7878
}

‎gen/org/intellij/jflex/psi/impl/JFlexStateListImpl.java

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* Copyright 2011-present Greg Shrago
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.intellij.jflex.psi.impl;
17+
18+
import java.util.List;
19+
import org.jetbrains.annotations.*;
20+
import com.intellij.lang.ASTNode;
21+
import com.intellij.psi.PsiElement;
22+
import com.intellij.psi.PsiElementVisitor;
23+
import com.intellij.psi.util.PsiTreeUtil;
24+
import static org.intellij.jflex.psi.JFlexTypes.*;
25+
import org.intellij.jflex.psi.*;
26+
27+
public class JFlexStateListImpl extends JFlexCompositeElementImpl implements JFlexStateList {
28+
29+
public JFlexStateListImpl(ASTNode node) {
30+
super(node);
31+
}
32+
33+
public void accept(@NotNull JFlexVisitor visitor) {
34+
visitor.visitStateList(this);
35+
}
36+
37+
public void accept(@NotNull PsiElementVisitor visitor) {
38+
if (visitor instanceof JFlexVisitor) accept((JFlexVisitor)visitor);
39+
else super.accept(visitor);
40+
}
41+
42+
@Override
43+
@NotNull
44+
public List<JFlexStateReference> getStateReferenceList() {
45+
return PsiTreeUtil.getChildrenOfTypeAsList(this, JFlexStateReference.class);
46+
}
47+
48+
}

‎gen/org/intellij/jflex/psi/impl/JFlexUserCodeSectionImpl.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
import static org.intellij.jflex.psi.JFlexTypes.*;
2525
import org.intellij.jflex.psi.*;
2626

27-
public class JFlexUserCodeSectionImpl extends JFlexCompositeElementImpl implements JFlexUserCodeSection {
27+
public class JFlexUserCodeSectionImpl extends JFlexFileSectionImpl implements JFlexUserCodeSection {
2828

2929
public JFlexUserCodeSectionImpl(ASTNode node) {
3030
super(node);

‎grammars/JFlex.bnf

Lines changed: 34 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,11 @@
2020
tokenTypeClass="org.intellij.jflex.psi.JFlexTokenType"
2121

2222
tokens = [
23-
java = 'regexp:.*'
24-
newline = 'regexp:\n'
2523
id = 'regexp:(_|\p{Alpha})\w*'
26-
string = 'regexp:"([^"\\]|\\.)*"'
27-
number = 'regexp:\p{Digit}+'
28-
line_comment='regexp://.*'
29-
block_comment='regexp:/\*((?<!\*/)(.|\n))*'
30-
escaped_char='regexp:\\.'
24+
line_comment='regexp://..'
25+
block_comment='regexp:/*..*/'
26+
raw='code block'
27+
unclosed='unclosed'
3128

3229
two_percs = '%%'
3330
star = '*'
@@ -40,30 +37,28 @@
4037
question = '?'
4138
dash = '-'
4239
plus = '+'
43-
roof = '^'
44-
slash2 = '/'
40+
hat = '^'
41+
fslash = '/'
42+
43+
dashdash = '--'
44+
ampamp = '&&'
45+
barbar = '||'
46+
tildetilde = '~~'
4547

4648
dot = '.'
4749
comma = ','
4850

4951
angle1 = '<'
5052
angle2 = '>'
51-
or = '|'
52-
slash = '\'
53+
bar = '|'
54+
bslash = '\'
5355
dollar = '$'
5456
eq = '='
55-
not = '!'
56-
not2 = '~'
57+
bang = '!'
58+
tilde = '~'
5759

5860
eof = '<<EOF>>'
5961

60-
class_jl='[:jletter:]'
61-
class_jld='[:jletterdigit:]'
62-
class_l='[:letter:]'
63-
class_d='[:digit:]'
64-
class_lu='[:uppercase:]'
65-
class_ll='[:lowercase:]'
66-
6762
opt_class='%class'
6863
opt_implements='%implements'
6964
opt_extends='%extends'
@@ -103,10 +98,6 @@
10398
opt_cup='%cup'
10499
opt_cupsym='%cupsym'
105100
opt_cupdebug='%cupdebug'
106-
opt_byacc='%byacc'
107-
opt_switch='%switch'
108-
opt_table='%table'
109-
opt_pack='%pack'
110101
opt_7bit='%7bit'
111102
opt_full='%full'
112103
opt_8bit='%8bit'
@@ -120,12 +111,10 @@
120111
opt_notunix='%notunix'
121112
opt_yyeof='%yyeof'
122113
opt_state='%state'
123-
opt_s='%s'
124114
opt_xstate='%xstate'
125-
opt_x='%x'
126-
127-
char='regexp:[^\{\}\[\]\(\)\+\*\?\\/]'
128115
]
116+
117+
extends(".*_section")=file_section
129118
recoverWhile(".*_section")=section_recover
130119
pin(".*_section")=1
131120

@@ -135,7 +124,6 @@
135124
extends(".*_expression")=expression
136125
name(".*_expression")="expression"
137126
name("option.*")="option"
138-
name(".*class")="char class"
139127

140128
//consumeTokenMethod("character|macro_reference|.*_(recover|expression|op)")="consumeTokenFast"
141129
//consumeTokenMethod="consumeTokenFast"
@@ -148,6 +136,7 @@ flex_file ::= []
148136
section_div
149137
lexical_rules_section
150138
{pin=1}
139+
fake file_section ::= '%%'
151140

152141
private meta line ::= new_line <<p>> new_line {pin=2}
153142
private new_line ::= &<<is_new_line>>
@@ -157,8 +146,7 @@ private section_recover ::= !'%%'
157146

158147
user_code_section ::= [] java_code {pin=1}
159148
user_value ::= <<anything2 !new_line>>
160-
java_code ::= <<anything2 (&java | !<<is_percent>>)>>
161-
{methods=[getReferences]}
149+
java_code ::= raw? { methods=[getReferences] }
162150

163151
java_type ::= id ( safe_dot id ) * {pin(".*")=1 methods=[getReferences]}
164152
private safe_dot ::= '.' !'*'
@@ -206,8 +194,8 @@ option ::= option_class
206194

207195
| option_switch
208196
| option_table
209-
| option_pack
210197
| option_7bit
198+
| option_16bit
211199
| option_full
212200
| option_unicode
213201
| option_ignore_case
@@ -255,18 +243,18 @@ option_byacc ::= '%byacc'
255243

256244
option_switch ::= '%switch'
257245
option_table ::= '%table'
258-
option_pack ::= '%pack'
259246
option_7bit ::= '%7bit'
260247
option_full ::= '%full' | '%8bit'
261-
option_unicode ::= '%unicode' | '%16bit'
248+
option_16bit ::= '%16bit'
249+
option_unicode ::= '%unicode' [number | version]
262250
option_ignore_case ::= '%caseless' | '%ignorecase'
263251
option_count_char ::= '%char'
264252
option_count_line ::= '%line'
265253
option_count_column ::= '%column'
266254

267255
option_obsolete ::= '%notunix' | '%yyeof'
268256

269-
state_declaration ::= ('%state' | '%s' | '%xstate' | '%x') state_definition ((','? !(id '=')) state_definition) * ','? {pin=1}
257+
state_declaration ::= ('%state' | '%xstate') state_definition ((','? !(id '=')) state_definition) * ','? {pin=1}
270258
state_definition ::= id {
271259
implements="com.intellij.psi.PsiNameIdentifierOwner"
272260
methods=[getName setName getNameIdentifier]
@@ -292,11 +280,10 @@ private rule_tail1 ::= '<<EOF>>' action {pin=1}
292280
private rule_tail2 ::= ['^'] expression look_ahead? action {pin=2}
293281
private rule_group ::= !('{' id '}') '{' rule_group_item + '}' {pin=2} // empty block forbidden
294282

295-
private state_list ::= '<' state_reference (',' state_reference) * '>' {pin(".*")=1}
283+
state_list ::= '<' state_reference (',' state_reference) * '>' {pin(".*")=1}
296284
state_reference ::= id { methods=[getReference] }
297285
look_ahead ::= '$' | '/' expression
298-
action ::= '{' java_class_statements '}' | '|' {pin(".*")=1 elementType=java_code}
299-
private java_class_statements ::= <<anything !('}' | '<' id '>')>>
286+
action ::= '{' raw? '}' | '|' {pin(".*")=1 elementType=java_code}
300287

301288
expression ::= choice_expression
302289
| sequence_expression
@@ -314,23 +301,17 @@ sequence_expression ::= expression (sequence_op expression) +
314301
paren_expression ::= '(' expression ')' {pin=1}
315302
not_expression ::= ('!'|'~') expression
316303
quantifier_expression ::= expression ( '*' | '+' | '?' | '{' number [ ',' number] '}' )
317-
class_expression ::= '[' '^'? '-'? (string | predefined_class | class_char [ '-' class_char]) * '-'? ']' {pin=1}
318-
private class_char ::= char | escaped_char
304+
class_expression ::= '[' '^'? (char_class_item (char_class_op char_class_item)* )* ']' {pin(".*")=1}
305+
private char_class_item ::= char_class_atom | class_expression
306+
char_class_atom ::= string | char_class | char [ '-' char] | '-' {elementType="" name="char"}
307+
char_class_op ::= '&&' | '||' | '~~' | '--' {elementType="" name="char"}
319308
macro_ref_expression ::= '{' macro_reference '}'
320309
macro_reference ::= id { methods=[getReference] }
321-
literal_expression ::= string | id | allowed_chars
322-
private allowed_chars ::= escaped_char | char | number | '-' | '=' | '<' | '>' | ','
323-
| ']' /*| '}' -- allowed after <STATE> } { } and in / part */
310+
literal_expression ::= string | id | char | number { methods=[id=""] }
324311

325-
sequence_op ::= &( '(' | '!' | '~' | '[' | ']'
326-
| string | predefined_class | '.' | '{' id '}'
327-
| !new_line (id | allowed_chars) )
312+
sequence_op ::= &('!' | '(' | '.' | '[' | '~'
313+
| char | char_class | number | string
314+
| '{' id | !new_line id )
328315
{elementType="" name="expression"}
329316

330-
predefined_class_expression ::= predefined_class | '.'
331-
private predefined_class ::= '[:jletter:]'
332-
| '[:jletterdigit:]'
333-
| '[:letter:]'
334-
| '[:digit:]'
335-
| '[:uppercase:]'
336-
| '[:lowercase:]'
317+
predefined_class_expression ::= char_class | '.'

‎support/org/intellij/jflex/editor/JFlexCompletionContributor.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,8 @@ private static Collection<String> suggestKeywords(PsiElement position) {
101101
public String convertItem(Object o) {
102102
if (o == null) return null;
103103
if (o instanceof IElementType[]) return super.convertItem(o);
104-
if (o == FLEX_ID || o == FLEX_CHAR || o == FLEX_ESCAPED_CHAR || o == FLEX_STRING || o == FLEX_NUMBER || o == FLEX_JAVA) return null;
104+
if (o == FLEX_ID || o == FLEX_CHAR || o == FLEX_STRING ||
105+
o == FLEX_NUMBER || o == FLEX_RAW || o == FLEX_VERSION) return null;
105106
String text = o.toString();
106107
return text.length() == 1 || inMacro && text.startsWith("%") || !inMacro && text.startsWith("[")? null : text;
107108
}

‎support/org/intellij/jflex/editor/JFlexSyntaxHighlighterFactory.java

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,9 +80,6 @@ else if (tokenType == FLEX_NUMBER) {
8080
else if (tokenType == FLEX_STRING) {
8181
return pack(STRING);
8282
}
83-
else if (tokenType == FLEX_ESCAPED_CHAR) {
84-
return pack(ESCAPED_CHAR);
85-
}
8683
else if (tokenType == FLEX_CHAR) {
8784
return pack(CHAR);
8885
}

‎support/org/intellij/jflex/parser/JFlexLexer.java

Lines changed: 87 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,97 @@
1616

1717
package org.intellij.jflex.parser;
1818

19-
import com.intellij.lexer.FlexAdapter;
19+
import com.intellij.lexer.FlexLexer;
20+
import com.intellij.lexer.LexerBase;
21+
import com.intellij.psi.tree.IElementType;
22+
import org.jetbrains.annotations.NotNull;
23+
24+
import java.io.IOException;
2025

2126
/**
27+
* Workaround com.intellij.lexer.FlexAdapter#getTokenStart()
28+
*
2229
* @author gregsh
2330
*/
24-
public class JFlexLexer extends FlexAdapter{
31+
public class JFlexLexer extends LexerBase {
32+
private final FlexLexer myFlex;
33+
private IElementType myTokenType;
34+
private CharSequence myText;
35+
36+
private int myStart;
37+
private int myEnd;
38+
private int myState;
39+
2540
public JFlexLexer() {
26-
super(new _JFlexLexer());
41+
myFlex = new _JFlexLexer();
42+
}
43+
44+
public FlexLexer getFlex() {
45+
return myFlex;
46+
}
47+
48+
@Override
49+
public void start(@NotNull final CharSequence buffer, int startOffset, int endOffset, final int initialState) {
50+
myText = buffer;
51+
myEnd = endOffset;
52+
myFlex.reset(myText, startOffset, endOffset, initialState);
53+
myTokenType = null;
54+
}
55+
56+
@Override
57+
public int getState() {
58+
if (myTokenType == null) locateToken();
59+
return myState;
60+
}
61+
62+
@Override
63+
public IElementType getTokenType() {
64+
if (myTokenType == null) locateToken();
65+
return myTokenType;
66+
}
67+
68+
@Override
69+
public int getTokenStart() {
70+
if (myTokenType == null) locateToken();
71+
return myStart;
72+
}
73+
74+
@Override
75+
public int getTokenEnd() {
76+
if (myTokenType == null) locateToken();
77+
return myFlex.getTokenEnd();
78+
}
79+
80+
@Override
81+
public void advance() {
82+
if (myTokenType == null) locateToken();
83+
myTokenType = null;
84+
}
85+
86+
@NotNull
87+
@Override
88+
public CharSequence getBufferSequence() {
89+
return myText;
90+
}
91+
92+
@Override
93+
public int getBufferEnd() {
94+
return myEnd;
95+
}
96+
97+
protected void locateToken() {
98+
if (myTokenType != null) return;
99+
try {
100+
myState = myFlex.yystate();
101+
myStart = myFlex.getTokenEnd();
102+
myTokenType = myFlex.advance();
103+
}
104+
catch (IOException e) { /*Can't happen*/ }
105+
catch (Error e) {
106+
// add lexer class name to the error
107+
final Error error = new Error(myFlex.getClass().getName() + ": " + e.getMessage());
108+
error.setStackTrace(e.getStackTrace());
109+
throw error;
110+
}
27111
}
28112
}

‎support/org/intellij/jflex/parser/JFlexParserUtil.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,9 @@ public static boolean is_new_line(PsiBuilder builder, int level) {
5959
IElementType type = builder.rawLookup(i);
6060
if (type == TokenType.WHITE_SPACE) continue;
6161
if (type == JFlexTypes.FLEX_LINE_COMMENT || type == JFlexTypes.FLEX_BLOCK_COMMENT) continue;
62-
return type == JFlexParserDefinition.FLEX_NEWLINE || type == null;
62+
if (type == JFlexParserDefinition.FLEX_NEWLINE || type == null) return true;
63+
if (builder.getOriginalText().charAt(builder.rawTokenTypeStart(i+1)-1) == '\n') return true;
64+
return false;
6365
}
6466
}
6567

‎support/org/intellij/jflex/parser/_JFlexLexerNew.flex

Lines changed: 391 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,391 @@
1+
/*
2+
* Copyright 2011-present Greg Shrago
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.intellij.jflex.parser;
17+
18+
import com.intellij.lexer.*;
19+
import com.intellij.psi.tree.IElementType;
20+
import static com.intellij.psi.TokenType.WHITE_SPACE;
21+
import static org.intellij.jflex.parser.JFlexParserDefinition.FLEX_NEWLINE;
22+
import static org.intellij.jflex.psi.JFlexTypes.*;
23+
import static com.intellij.psi.TokenType.BAD_CHARACTER;
24+
25+
%%
26+
27+
// The outline, states and macros are borrowed from the original JFlex lexer.
28+
// See https://github.com/jflex-de/jflex/blob/master/jflex/src/main/jflex/LexScan.flex
29+
30+
%public
31+
%class _JFlexLexer
32+
%implements FlexLexer
33+
%function advance
34+
%type IElementType
35+
%unicode
36+
37+
%state COMMENT, MACROS, REGEXPSTART, EXPRBAR
38+
%state REGEXP, STATES, JAVA_CODE, STRING_CONTENT
39+
%state CHARCLASS, COPY, MACROUSE, REPEATEXP
40+
%state REGEXP_CODEPOINT_SEQUENCE
41+
%state STRING_CODEPOINT_SEQUENCE
42+
%state SKIP_TOEOL, SKIP_WSNL, REPORT_UNCLOSED, REGEXPSTART0
43+
44+
%{
45+
int nextState;
46+
boolean macroDefinition;
47+
48+
int braceCount;
49+
int bracketCount;
50+
int commentCount;
51+
52+
public _JFlexLexer() {
53+
this((java.io.Reader)null);
54+
}
55+
%}
56+
57+
Digit = [0-9]
58+
HexDigit = [0-9a-fA-F]
59+
OctDigit = [0-7]
60+
61+
Number = {Digit}+
62+
HexNumber = \\ x {HexDigit} {2}
63+
OctNumber = \\ [0-3]? {OctDigit} {1, 2}
64+
65+
DottedVersion = [1-9][0-9]*(\.[0-9]+){0,2}
66+
67+
Unicode4 = \\ u {HexDigit} {4}
68+
Unicode6 = \\ U {HexDigit} {6}
69+
70+
WSP = [ \t\b]
71+
WSPNL = [\u2028\u2029\u000A\u000B\u000C\u000D\u0085\t\b\ ]
72+
NWSPNL = [^\u2028\u2029\u000A\u000B\u000C\u000D\u0085\t\b\ ]
73+
NL = [\u2028\u2029\u000A\u000B\u000C\u000D\u0085] | \u000D\u000A
74+
NNL = [^\u2028\u2029\u000A\u000B\u000C\u000D\u0085]
75+
76+
Ident = {IdentStart} {IdentPart}*
77+
78+
IdentStart = [:jletter:]
79+
IdentPart = [:jletterdigit:]
80+
81+
JavaComment = {TraditionalComment}|{EndOfLineComment}
82+
TraditionalComment = "/*"{CommentContent}\*+"/"
83+
EndOfLineComment = "//" {NNL}*
84+
CommentContent = ([^*]|\*+[^*/])*
85+
StringCharacter = [^\u2028\u2029\u000A\u000B\u000C\u000D\u0085\"\\]
86+
CharLiteral = \'([^\u2028\u2029\u000A\u000B\u000C\u000D\u0085\'\\]|{EscapeSequence})\'
87+
StringLiteral = \"({StringCharacter}|{EscapeSequence})*\"
88+
EscapeSequence = \\[^\u2028\u2029\u000A\u000B\u000C\u000D\u0085]|\\+u{HexDigit}{4}|\\[0-3]?{OctDigit}{1,2}
89+
JavaRest = [^\{\}\"\'/]|"/"[^*/]
90+
JavaCode = ({JavaRest}|{StringLiteral}|{CharLiteral}|{JavaComment})+
91+
92+
93+
%%
94+
<YYINITIAL> {
95+
"%%" { macroDefinition = true; nextState = MACROS; yybegin(SKIP_TOEOL); return FLEX_TWO_PERCS; }
96+
([^\%]|\%[^\%])* { return FLEX_RAW; }
97+
}
98+
99+
<SKIP_WSNL> {
100+
{WSP}+ { return WHITE_SPACE; }
101+
{NL}+ { return FLEX_NEWLINE; }
102+
. { yypushback(yylength()); yybegin(nextState); }
103+
}
104+
105+
<SKIP_TOEOL> {
106+
.* { return FLEX_LINE_COMMENT; }
107+
{NL}+ { yybegin(nextState); return FLEX_NEWLINE; }
108+
}
109+
110+
<REPORT_UNCLOSED> {
111+
[^] { yypushback(yylength()); yybegin(nextState); return FLEX_UNCLOSED; }
112+
}
113+
114+
<COMMENT> {
115+
"/"+ "*" { commentCount++; }
116+
"*"+ "/" { if (commentCount > 0) commentCount--; else { yybegin(nextState); return FLEX_BLOCK_COMMENT; } }
117+
118+
[^] { }
119+
<<EOF>> { yybegin(REPORT_UNCLOSED); return FLEX_BLOCK_COMMENT; }
120+
}
121+
122+
<MACROS> {
123+
"%{" { nextState=COPY; yybegin(SKIP_TOEOL); return FLEX_OPT_CODE1; }
124+
"%init{" { nextState=COPY; yybegin(SKIP_TOEOL); return FLEX_OPT_INIT1; }
125+
"%eofval{" { nextState=COPY; yybegin(SKIP_TOEOL); return FLEX_OPT_EOFVAL1; }
126+
"%eof{" { nextState=COPY; yybegin(SKIP_TOEOL); return FLEX_OPT_EOF1; }
127+
"%initthrow{" { nextState=COPY; yybegin(SKIP_TOEOL); return FLEX_OPT_INITTHROW1; }
128+
"%yylexthrow{" { nextState=COPY; yybegin(SKIP_TOEOL); return FLEX_OPT_YYLEXTHROW1; }
129+
"%eofthrow{" { nextState=COPY; yybegin(SKIP_TOEOL); return FLEX_OPT_EOFTHROW1; }
130+
}
131+
132+
<COPY> {
133+
"%}" { nextState=MACROS; yybegin(SKIP_TOEOL); return FLEX_OPT_CODE2; }
134+
"%init}" { nextState=MACROS; yybegin(SKIP_TOEOL); return FLEX_OPT_INIT2; }
135+
"%eofval}" { nextState=MACROS; yybegin(SKIP_TOEOL); return FLEX_OPT_EOFVAL2; }
136+
"%eof}" { nextState=MACROS; yybegin(SKIP_TOEOL); return FLEX_OPT_EOF2; }
137+
"%initthrow}" { nextState=MACROS; yybegin(SKIP_TOEOL); return FLEX_OPT_INITTHROW2; }
138+
"%yylexthrow}" { nextState=MACROS; yybegin(SKIP_TOEOL); return FLEX_OPT_YYLEXTHROW2; }
139+
"%eofthrow}" { nextState=MACROS; yybegin(SKIP_TOEOL); return FLEX_OPT_EOFTHROW2; }
140+
141+
([^\%]|\%[^\}iey])* { return FLEX_RAW; }
142+
<<EOF>> { yybegin(REPORT_UNCLOSED); return FLEX_RAW; }
143+
}
144+
145+
<MACROS> {
146+
"%class" { return FLEX_OPT_CLASS; }
147+
"%implements" { return FLEX_OPT_IMPLEMENTS; }
148+
"%extends" { return FLEX_OPT_EXTENDS; }
149+
"%public" { return FLEX_OPT_PUBLIC; }
150+
"%final" { return FLEX_OPT_FINAL; }
151+
"%abstract" { return FLEX_OPT_ABSTRACT; }
152+
"%apiprivate" { return FLEX_OPT_APIPRIVATE; }
153+
"%initthrow" { return FLEX_OPT_INITTHROW; }
154+
"%ctorarg" { return FLEX_OPT_CTORARG; }
155+
"%scanerror" { return FLEX_OPT_SCANERROR; }
156+
"%buffer" { return FLEX_OPT_BUFFER; }
157+
"%function" { return FLEX_OPT_FUNCTION; }
158+
"%include" { return FLEX_OPT_INCLUDE; }
159+
"%integer" { return FLEX_OPT_INTEGER; }
160+
"%int" { return FLEX_OPT_INT; }
161+
"%intwrap" { return FLEX_OPT_INTWRAP; }
162+
"%type" { return FLEX_OPT_TYPE; }
163+
"%yylexthrow" { return FLEX_OPT_YYLEXTHROW; }
164+
"%eofthrow" { return FLEX_OPT_EOFTHROW; }
165+
"%eofclose" { return FLEX_OPT_EOFCLOSE; }
166+
"%debug" { return FLEX_OPT_DEBUG; }
167+
"%standalone" { return FLEX_OPT_STANDALONE; }
168+
"%cup" { return FLEX_OPT_CUP; }
169+
"%cupsym" { return FLEX_OPT_CUPSYM; }
170+
"%cupdebug" { return FLEX_OPT_CUPDEBUG; }
171+
"%7bit" { return FLEX_OPT_7BIT; }
172+
"%full" { return FLEX_OPT_FULL; }
173+
"%8bit" { return FLEX_OPT_8BIT; }
174+
"%unicode" { return FLEX_OPT_UNICODE; }
175+
"%16bit" { return FLEX_OPT16BIT; }
176+
"%caseless" { return FLEX_OPT_CASELESS; }
177+
"%ignorecase" { return FLEX_OPT_IGNORECASE; }
178+
"%char" { return FLEX_OPT_CHAR; }
179+
"%line" { return FLEX_OPT_LINE; }
180+
"%column" { return FLEX_OPT_COLUMN; }
181+
"%notunix" { return FLEX_OPT_NOTUNIX; }
182+
"%yyeof" { return FLEX_OPT_YYEOF; }
183+
^"%s" ("tate" "s"?)? { return FLEX_OPT_STATE; }
184+
^"%x" ("state" "s"?)? { return FLEX_OPT_XSTATE; }
185+
186+
// noop/deprecated
187+
//"%pack" { return FLEX_OPT_NOOP; }
188+
//"%inputstreamctor" { return FLEX_OPT_NOOP; }
189+
//"%byacc" { return FLEX_OPT_BYACC; }
190+
//"%switch" { return FLEX_OPT_SWITCH; }
191+
//"%table" { return FLEX_OPT_TABLE; }
192+
}
193+
194+
<MACROS> {
195+
^ "%%" { macroDefinition=false; nextState=REGEXPSTART; yybegin(SKIP_TOEOL); return FLEX_TWO_PERCS; }
196+
{EndOfLineComment} { return FLEX_LINE_COMMENT; }
197+
"/*" { nextState=MACROS; yybegin(COMMENT); }
198+
"." { return FLEX_DOT; }
199+
"," { return FLEX_COMMA; }
200+
201+
"=" { yybegin(REGEXP); return FLEX_EQ; }
202+
{Ident} { return FLEX_ID; }
203+
{StringLiteral} { return FLEX_STRING; }
204+
{Number} { return FLEX_NUMBER; }
205+
{DottedVersion} { return FLEX_VERSION; }
206+
207+
{WSP}+ { return WHITE_SPACE; }
208+
{NL}+ { return FLEX_NEWLINE; }
209+
}
210+
211+
<REGEXPSTART> {
212+
{EndOfLineComment} { return FLEX_LINE_COMMENT; }
213+
"/*" { nextState = REGEXPSTART; yybegin(COMMENT); }
214+
215+
^ {WSP}* "%include" { return FLEX_OPT_INCLUDE; }
216+
217+
"<<EOF>>" { yypushback(yylength()); yybegin(REGEXP); }
218+
"<" { yybegin(STATES); return FLEX_ANGLE1; }
219+
"}" { return FLEX_BRACE2; }
220+
221+
{WSP}+ { return WHITE_SPACE; }
222+
{NL}+ { return FLEX_NEWLINE; }
223+
. { yypushback(yylength()); yybegin(REGEXP); }
224+
}
225+
226+
<STATES> {
227+
">" / {WSPNL}* "{" { nextState=REGEXPSTART0; yybegin(SKIP_WSNL); return FLEX_ANGLE2; }
228+
">" { nextState=REGEXP; yybegin(SKIP_WSNL); return FLEX_ANGLE2; }
229+
230+
"," { return FLEX_COMMA; }
231+
{Ident} { return FLEX_ID; }
232+
233+
{WSP}+ { return WHITE_SPACE; }
234+
{NL}+ { return FLEX_NEWLINE; }
235+
<<EOF>> { nextState=REGEXPSTART; yybegin(REPORT_UNCLOSED); }
236+
}
237+
238+
<REGEXPSTART0> {
239+
"{" / {WSP}* {Ident} {WSP}* "}" { yybegin(MACROUSE); return FLEX_BRACE1; }
240+
"{" / "{" {WSP}* {Number} { yybegin(REPEATEXP); return FLEX_BRACE1; }
241+
242+
"{" { yybegin(REGEXPSTART); return FLEX_BRACE1; }
243+
}
244+
245+
<REGEXP> {
246+
"<<EOF>>" { return FLEX_EOF; }
247+
{WSP}+ { return WHITE_SPACE; }
248+
{EndOfLineComment} { return FLEX_LINE_COMMENT; }
249+
"/*" { nextState=REGEXP; yybegin(COMMENT); }
250+
251+
"|" { if (macroDefinition) yybegin(EXPRBAR); return FLEX_BAR; }
252+
253+
"{" / {WSP}* {Ident} {WSP}* "}" { yybegin(MACROUSE); return FLEX_BRACE1; }
254+
"{" / {WSP}* {Number} { yybegin(REPEATEXP); return FLEX_BRACE1; }
255+
"{" { if (macroDefinition) return FLEX_CHAR; else yybegin(JAVA_CODE); return FLEX_BRACE1; }
256+
257+
{NL} / {WSPNL}* [\|\[\{\"\\|\!\~\(\)\*\+\?\$\^\.\/]
258+
{ yypushback(1); nextState=REGEXP; yybegin(SKIP_WSNL); }
259+
260+
\" { nextState = REGEXP; yybegin(STRING_CONTENT); }
261+
"\\u{" { yybegin(REGEXP_CODEPOINT_SEQUENCE); }
262+
263+
"!" { return FLEX_BANG; }
264+
"~" { return FLEX_TILDE; }
265+
"(" { return FLEX_PAREN1; }
266+
")" { return FLEX_PAREN2; }
267+
"*" { return FLEX_STAR; }
268+
"+" { return FLEX_PLUS; }
269+
"?" { return FLEX_QUESTION; }
270+
"$" { return FLEX_DOLLAR; }
271+
"^" { return FLEX_HAT; }
272+
"." { return FLEX_DOT; }
273+
"\\R" { return FLEX_CHAR; }
274+
"[" { yybegin(CHARCLASS); return FLEX_BRACK1; }
275+
"/" { return FLEX_FSLASH; }
276+
277+
{NL} { yypushback(yylength()); if (macroDefinition) yybegin(MACROS); else yybegin(REGEXPSTART); }
278+
{Ident} { return FLEX_ID; }
279+
{Number} { return FLEX_NUMBER; }
280+
. { return FLEX_CHAR; }
281+
}
282+
283+
<EXPRBAR> {
284+
{WSP}+ { return WHITE_SPACE; }
285+
{NL}+ { return FLEX_NEWLINE; }
286+
. { yypushback(yylength()); yybegin(REGEXP); }
287+
}
288+
289+
<REPEATEXP, MACROUSE> {
290+
"}" { yybegin(REGEXP); return FLEX_BRACE2; }
291+
"," { return FLEX_COMMA; }
292+
{Number} { return FLEX_NUMBER; }
293+
{Ident} { return FLEX_ID; }
294+
295+
{WSP}+ { return WHITE_SPACE; }
296+
. { yypushback(yylength()); yybegin(REGEXP); } // fallback
297+
}
298+
299+
<CHARCLASS> {
300+
"{" {Ident} "}" { return FLEX_CHAR; }
301+
"[" { bracketCount++; return FLEX_BRACK1; }
302+
"]" { if (bracketCount > 0) bracketCount--; else yybegin(REGEXP); return FLEX_BRACK2; }
303+
"^" { return FLEX_HAT; }
304+
"-" { return FLEX_DASH; }
305+
"--" { return FLEX_DASHDASH; }
306+
"&&" { return FLEX_AMPAMP; }
307+
"||" { return FLEX_BARBAR; }
308+
"~~" { return FLEX_TILDETILDE; }
309+
"\\u{" {HexDigit}{1,6} "}" { return FLEX_CHAR; }
310+
311+
// this is a hack to keep JLex compatibilty with char class
312+
// expressions like [+-]
313+
"-]" { yypushback(1); return FLEX_CHAR; }
314+
315+
\" { nextState=CHARCLASS; yybegin(STRING_CONTENT); }
316+
. { return FLEX_CHAR; }
317+
{NL} { bracketCount=0; yypushback(yylength()); nextState=REGEXP; yybegin(REPORT_UNCLOSED); } // fallback
318+
319+
<<EOF>> { nextState=REGEXP; yybegin(REPORT_UNCLOSED); return FLEX_BRACK2; }
320+
}
321+
322+
<STRING_CONTENT> {
323+
\" { yybegin(nextState); return FLEX_STRING; }
324+
\\\" { }
325+
[^\"\\\u2028\u2029\u000A\u000B\u000C\u000D\u0085]+ { }
326+
327+
{HexNumber} { }
328+
{OctNumber} { }
329+
{Unicode4} { }
330+
{Unicode6} { }
331+
332+
"\\u{" { yybegin(STRING_CODEPOINT_SEQUENCE); }
333+
334+
\\(b|t|f|r) { }
335+
\\. { }
336+
337+
{NL} { yypushback(yylength()); yybegin(nextState); return FLEX_STRING; }
338+
<<EOF>> { yybegin(REPORT_UNCLOSED); return FLEX_STRING; }
339+
}
340+
341+
342+
<REGEXP, CHARCLASS> {
343+
{HexNumber} { return FLEX_CHAR; }
344+
{OctNumber} { return FLEX_CHAR; }
345+
{Unicode4} { return FLEX_CHAR; }
346+
{Unicode6} { return FLEX_CHAR; }
347+
348+
\\(b|t|f|r) { return FLEX_CHAR; }
349+
\\. { return FLEX_CHAR; }
350+
351+
"[:jletter:]" { return FLEX_CHAR_CLASS; }
352+
"[:jletterdigit:]" { return FLEX_CHAR_CLASS; }
353+
"[:letter:]" { return FLEX_CHAR_CLASS; }
354+
"[:uppercase:]" { return FLEX_CHAR_CLASS; }
355+
"[:lowercase:]" { return FLEX_CHAR_CLASS; }
356+
"[:digit:]" { return FLEX_CHAR_CLASS; }
357+
"\\d" { return FLEX_CHAR_CLASS; }
358+
"\\D" { return FLEX_CHAR_CLASS; }
359+
"\\s" { return FLEX_CHAR_CLASS; }
360+
"\\S" { return FLEX_CHAR_CLASS; }
361+
"\\w" { return FLEX_CHAR_CLASS; }
362+
"\\W" { return FLEX_CHAR_CLASS; }
363+
"\\p{"[^}]*"}" { return FLEX_CHAR_CLASS; }
364+
"\\P{"[^}]*"}" { return FLEX_CHAR_CLASS; }
365+
}
366+
367+
368+
<JAVA_CODE> {
369+
"{" { braceCount++; }
370+
"}" { if (braceCount > 0) braceCount--; else { yypushback(1); yybegin(REGEXPSTART); return FLEX_RAW; } }
371+
{JavaCode} { }
372+
<<EOF>> { nextState=REGEXPSTART; yybegin(REPORT_UNCLOSED); return FLEX_RAW; }
373+
}
374+
375+
376+
<REGEXP_CODEPOINT_SEQUENCE> {
377+
"}" { yybegin(REGEXP); return FLEX_BRACE2; }
378+
{HexDigit}{1,6} { return FLEX_NUMBER; }
379+
{WSP}+ { return WHITE_SPACE; }
380+
{NL}+ { return FLEX_NEWLINE; }
381+
}
382+
383+
<STRING_CODEPOINT_SEQUENCE> {
384+
"}" { yybegin(STRING_CONTENT); }
385+
{HexDigit}{1,6} { }
386+
{WSP}+ { }
387+
{NL} { yypushback(yylength()); yybegin(REPORT_UNCLOSED); return FLEX_STRING; }
388+
}
389+
390+
391+
[^] { return BAD_CHARACTER; }

‎support/org/intellij/jflex/psi/impl/JFlexFile.java

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* Copyright 2011-2016 Gregory Shrago
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.intellij.jflex.psi.impl;
18+
19+
import com.intellij.psi.PsiFile;
20+
21+
/**
22+
* @author gregsh
23+
*/
24+
public interface JFlexFile extends PsiFile {
25+
}

‎support/org/intellij/jflex/psi/impl/JFlexFileImpl.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
/**
2727
* @author gregsh
2828
*/
29-
public class JFlexFileImpl extends PsiFileBase {
29+
public class JFlexFileImpl extends PsiFileBase implements JFlexFile {
3030
public JFlexFileImpl(FileViewProvider fileViewProvider) {
3131
super(fileViewProvider, JFlexLanguage.INSTANCE);
3232
}

‎testData/generator/FlexGrammar.PSI.expected.java

Lines changed: 138 additions & 79 deletions
Large diffs are not rendered by default.

‎testData/generator/FlexGrammar.expected.java

Lines changed: 195 additions & 265 deletions
Large diffs are not rendered by default.

‎testData/jflex/parser/CharClassOp.txt

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
JFLEX_FILE
2+
JFlexUserCodeSectionImpl(FLEX_USER_CODE_SECTION)
3+
JFlexJavaCodeImpl(FLEX_JAVA_CODE)
4+
<empty list>
5+
PsiElement(%%)('%%')
6+
PsiWhiteSpace('\n')
7+
JFlexDeclarationsSectionImpl(FLEX_DECLARATIONS_SECTION)
8+
<empty list>
9+
PsiElement(%%)('%%')
10+
PsiWhiteSpace('\n')
11+
JFlexLexicalRulesSectionImpl(FLEX_LEXICAL_RULES_SECTION)
12+
JFlexRuleImpl(FLEX_RULE)
13+
JFlexSequenceExpressionImpl(FLEX_SEQUENCE_EXPRESSION)
14+
JFlexLiteralExpressionImpl(FLEX_LITERAL_EXPRESSION)
15+
PsiElement(id)('N')
16+
JFlexLiteralExpressionImpl(FLEX_LITERAL_EXPRESSION)
17+
PsiElement(char)('=')
18+
JFlexClassExpressionImpl(FLEX_CLASS_EXPRESSION)
19+
PsiElement([)('[')
20+
PsiElement(char)('\w')
21+
PsiElement(char)('_')
22+
PsiElement(--)('--')
23+
PsiElement(char)('\d')
24+
PsiElement(])(']')
25+
JFlexClassExpressionImpl(FLEX_CLASS_EXPRESSION)
26+
PsiElement([)('[')
27+
PsiElement(char)('\w')
28+
PsiElement(char)('_')
29+
PsiElement(||)('||')
30+
PsiElement(char)('\d')
31+
PsiElement(])(']')
32+
JFlexClassExpressionImpl(FLEX_CLASS_EXPRESSION)
33+
PsiElement([)('[')
34+
JFlexClassExpressionImpl(FLEX_CLASS_EXPRESSION)
35+
PsiElement([)('[')
36+
PsiElement(char)('\w')
37+
PsiElement(char)('_')
38+
PsiElement(])(']')
39+
PsiElement(&&)('&&')
40+
PsiElement(char)('\d')
41+
PsiElement(])(']')
42+
JFlexClassExpressionImpl(FLEX_CLASS_EXPRESSION)
43+
PsiElement([)('[')
44+
PsiElement(char)('\w')
45+
PsiElement(char)('_')
46+
PsiElement(~~)('~~')
47+
JFlexClassExpressionImpl(FLEX_CLASS_EXPRESSION)
48+
PsiElement([)('[')
49+
PsiElement(char)('\d')
50+
PsiElement(])(']')
51+
PsiElement(])(']')
52+
PsiWhiteSpace(' ')
53+
JFlexJavaCodeImpl(FLEX_JAVA_CODE)
54+
PsiElement({)('{')
55+
PsiElement(})('}')

‎testData/jflex/parser/EOF1.txt

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
JFLEX_FILE
2+
JFlexUserCodeSectionImpl(FLEX_USER_CODE_SECTION)
3+
JFlexJavaCodeImpl(FLEX_JAVA_CODE)
4+
PsiElement(code block)(' ')
5+
PsiErrorElement:<new-line> expected, got '%%'
6+
<empty list>
7+
JFlexDeclarationsSectionImpl(FLEX_DECLARATIONS_SECTION)
8+
<empty list>
9+
JFlexLexicalRulesSectionImpl(FLEX_LEXICAL_RULES_SECTION)
10+
<empty list>
11+
PsiElement(%%)('%%')
12+
PsiWhiteSpace('\n\n')
13+
PsiElement(%%)('%%')
14+
PsiWhiteSpace('\n')
15+
PsiElement(string)('"')

‎testData/jflex/parser/ParserFixes.flex

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,15 @@ package sample.lexer;
1111

1212
%state STATE, STATE2,
1313

14-
1514
FUNCTION = [_a-zA-Z]([$_a-zA-Z0-9])*?[:]([^\n\r])*?(->|=>)
1615
NOTTAGORCOMMENTBEGIN = ("<"[^c/C!] | "<c"[^fF] | "<C"[^fF] | "</"[^Cc] | "</c"[^Ff] | "</C"[^fF] |
1716
"<!"[^-] | "<!-"[^-] | "<!--"[^-])
17+
%x STAT
18+
%s STAT
19+
%{
20+
String escapes="\\ \n \r \t \f" + '\\' + '\n' + '\r' + '\t' + '\f' ;
21+
%}
22+
1823
ID_CHAR = [a-zA-Z_0-9-]
1924
INT_LITERAL = [-+]?[0-9]+
2025
EXPONENT_PART=[Ee][+-]?[0-9]+
@@ -24,6 +29,7 @@ NO_STATE=<TAG>
2429
VARIABLE = "$"(::)?{NAME}(::{NAME})*
2530
BOOL = true|yes|on|false|no|off|undefined|null
2631
SYNTAX = ,|\.
32+
M=(-?0b[01]+)
2733

2834
%%
2935
<YYINITIAL> {
@@ -70,5 +76,8 @@ SYNTAX = ,|\.
7076
}
7177

7278

73-
79+
{HexDigit}{1,6} { }
80+
{HexDigit}{ 1 , 6 } { }
81+
[\w_--\d] { }
82+
<<EOF>> { }
7483
}

‎testData/jflex/parser/ParserFixes.txt

Lines changed: 181 additions & 374 deletions
Large diffs are not rendered by default.

‎testData/jflex/parser/ParserFixes2.txt

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,11 @@ JFLEX_FILE
2424
JFlexClassExpressionImpl(FLEX_CLASS_EXPRESSION)
2525
PsiElement([)('[')
2626
PsiElement(^)('^')
27-
PsiErrorElement:<char class>, ']', char or escaped_char expected, got '['
27+
JFlexClassExpressionImpl(FLEX_CLASS_EXPRESSION)
28+
PsiElement([)('[')
29+
PsiElement(])(']')
30+
PsiErrorElement:<char> or ']' expected, got ''
2831
<empty list>
29-
PsiElement(BAD_CHARACTER)('[')
30-
PsiElement(])(']')
3132
PsiWhiteSpace('\n')
3233
JFlexMacroDefinitionImpl(FLEX_MACRO_DEFINITION)
3334
PsiElement(id)('NO_MACRO_IN_LINE')
@@ -36,7 +37,7 @@ JFLEX_FILE
3637
JFlexLiteralExpressionImpl(FLEX_LITERAL_EXPRESSION)
3738
PsiElement(id)('abc')
3839
JFlexLiteralExpressionImpl(FLEX_LITERAL_EXPRESSION)
39-
PsiElement(=)('=')
40+
PsiElement(char)('=')
4041
JFlexLiteralExpressionImpl(FLEX_LITERAL_EXPRESSION)
4142
PsiElement(id)('d')
4243
PsiWhiteSpace('\n')
@@ -47,10 +48,9 @@ JFLEX_FILE
4748
PsiWhiteSpace(' ')
4849
JFlexClassExpressionImpl(FLEX_CLASS_EXPRESSION)
4950
PsiElement([)('[')
50-
PsiErrorElement:<char class>, ']', char or escaped_char expected, got 'DIGITS'
51+
PsiErrorElement:<char> or ']' expected, got ''
5152
<empty list>
52-
PsiWhiteSpace('\n')
53-
PsiWhiteSpace('\n')
53+
PsiWhiteSpace('\n\n')
5454
JFlexMacroDefinitionImpl(FLEX_MACRO_DEFINITION)
5555
PsiElement(id)('DIGITS')
5656
PsiElement(=)('=')
@@ -63,28 +63,26 @@ JFLEX_FILE
6363
PsiElement(+)('+')
6464
PsiWhiteSpace('\n')
6565
PsiErrorElement:<expression> expected, got '#'
66-
PsiElement(char)('#')
66+
PsiElement(BAD_CHARACTER)('#')
6767
PsiElement(DUMMY_BLOCK)
6868
PsiElement(id)('if')
69-
PsiElement(()('(')
70-
PsiElement($)('$')
71-
PsiElement(id)('dialect')
69+
PsiElement(BAD_CHARACTER)('(')
70+
PsiElement(id)('$dialect')
7271
PsiWhiteSpace(' ')
7372
PsiElement(=)('=')
74-
PsiElement(=)('=')
73+
PsiElement(char)('=')
7574
PsiWhiteSpace(' ')
7675
PsiElement(char)(''')
7776
PsiElement(number)('111')
7877
PsiElement(char)(''')
7978
PsiWhiteSpace(' ')
8079
PsiElement(|)('|')
81-
PsiElement(|)('|')
80+
PsiElement(|)('|')
8281
PsiWhiteSpace(' ')
83-
PsiElement($)('$')
84-
PsiElement(id)('dialect')
82+
PsiElement(id)('$dialect')
8583
PsiWhiteSpace(' ')
86-
PsiElement(=)('=')
87-
PsiElement(=)('=')
84+
PsiElement(char)('=')
85+
PsiElement(char)('=')
8886
PsiWhiteSpace(' ')
8987
PsiElement(char)(''')
9088
PsiElement(number)('222')
@@ -110,7 +108,7 @@ JFLEX_FILE
110108
PsiElement(])(']')
111109
PsiWhiteSpace('\n')
112110
PsiErrorElement:<expression> expected, got '#'
113-
PsiElement(char)('#')
111+
PsiElement(BAD_CHARACTER)('#')
114112
PsiElement(id)('end')
115113
PsiWhiteSpace('\n\n')
116114
JFlexMacroDefinitionImpl(FLEX_MACRO_DEFINITION)
@@ -126,7 +124,7 @@ JFLEX_FILE
126124
PsiWhiteSpace('\n')
127125
PsiWhiteSpace(' ')
128126
JFlexPredefinedClassExpressionImpl(FLEX_PREDEFINED_CLASS_EXPRESSION)
129-
PsiElement([:digit:])('[:digit:]')
127+
PsiElement(char_class)('[:digit:]')
130128
PsiWhiteSpace('\n')
131129
PsiWhiteSpace(' ')
132130
JFlexClassExpressionImpl(FLEX_CLASS_EXPRESSION)

‎testData/jflex/parser/SelfBnf.txt

Lines changed: 50 additions & 428 deletions
Large diffs are not rendered by default.

‎testData/jflex/parser/SelfFlex.txt

Lines changed: 240 additions & 1008 deletions
Large diffs are not rendered by default.

‎tests/org/intellij/jflex/JFlexParserTest.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,12 @@ protected void setUp() throws Exception {
4444
public void testEmpty1() throws Exception { doCodeTest(""); }
4545
public void testEmpty2() throws Exception { doCodeTest("%%\n"); }
4646
public void testEmpty3() throws Exception { doCodeTest("%%\n%%"); }
47+
public void testEOF1() throws Exception { doCodeTest(" %%\n\n%%\n\""); }
4748
public void testParserFixes() { doTest(true); }
4849
public void testParserFixes2() { doTest(true); }
4950

51+
public void testCharClassOp() throws Exception { doCodeTest("%%\n%%\nN=[\\w_--\\d][\\w_||\\d][[\\w_]&&\\d][\\w_~~[\\d]] {}"); }
52+
5053
@Override
5154
protected String loadFile(@NonNls String name) throws IOException {
5255
if (name.startsWith("Self")) {
@@ -56,5 +59,8 @@ protected String loadFile(@NonNls String name) throws IOException {
5659
FileUtil.loadFile(new File(myFullDataPath, name1).getCanonicalFile(), CharsetToolkit.UTF8).trim());
5760
}
5861
return super.loadFile(name);
62+
//String s = super.loadFile(name);
63+
//System.out.println(LexerTestCase.printTokens(s, 0, new JFlexLexer()));
64+
//return s;
5965
}
6066
}

0 commit comments

Comments
 (0)
Please sign in to comment.