Skip to content

Commit 44a3299

Browse files
sharwellsharwell
committed
Use Vocabulary interface instead of token names
1 parent 82bd8c3 commit 44a3299

8 files changed

+79
-120
lines changed

‎org-antlr-works-editor/src/org/antlr/works/editor/grammar/completion/KeywordCompletionItem.java

+12-17
Original file line numberDiff line numberDiff line change
@@ -40,24 +40,19 @@ public class KeywordCompletionItem extends GrammarCompletionItem {
4040
GrammarLexer.CHANNELS,
4141
GrammarLexer.OPTIONS);
4242

43-
private static final Map<Integer, String> KEYWORDS =
44-
new HashMap<Integer, String>() {{
45-
for (int i : KEYWORD_TYPES.toArray()) {
46-
String keyword = GrammarLexer.tokenNames[i].toLowerCase();
47-
if (keyword.charAt(0) == '\'') {
48-
keyword = keyword.substring(1, keyword.length() - 1);
49-
}
50-
51-
put(i, keyword);
52-
}
53-
}};
43+
private static final Map<Integer, String> KEYWORDS = new HashMap<>();
44+
static {
45+
for (int i : KEYWORD_TYPES.toArray()) {
46+
KEYWORDS.put(i, GrammarLexer.VOCABULARY.getSymbolicName(i).toLowerCase());
47+
}
48+
}
5449

55-
public static final Map<Integer, KeywordCompletionItem> KEYWORD_ITEMS =
56-
new HashMap<Integer, KeywordCompletionItem>() {{
57-
for (Map.Entry<Integer, String> keyword : KEYWORDS.entrySet()) {
58-
put(keyword.getKey(), new KeywordCompletionItem(keyword.getValue()));
59-
}
60-
}};
50+
public static final Map<Integer, KeywordCompletionItem> KEYWORD_ITEMS = new HashMap<>();
51+
static {
52+
for (Map.Entry<Integer, String> keyword : KEYWORDS.entrySet()) {
53+
KEYWORD_ITEMS.put(keyword.getKey(), new KeywordCompletionItem(keyword.getValue()));
54+
}
55+
}
6156

6257
private final String keyword;
6358
private String leftText;

‎org-antlr-works-editor/src/org/antlr/works/editor/grammar/debugger/AbstractGrammarDebuggerEditorKit.java

+14-9
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import javax.swing.text.Document;
2424
import javax.xml.bind.DatatypeConverter;
2525
import org.antlr.v4.runtime.CommonTokenStream;
26+
import org.antlr.v4.runtime.VocabularyImpl;
2627
import org.antlr.v4.runtime.atn.ATN;
2728
import org.antlr.v4.runtime.atn.ATNDeserializer;
2829
import org.antlr.works.editor.grammar.debugger.LexerDebuggerTokenHighlighterLayerFactory.LexerOpCode;
@@ -50,10 +51,9 @@ public abstract class AbstractGrammarDebuggerEditorKit extends NbEditorKit {
5051
public static final String PROP_CHANNELS = "Channels";
5152

5253
/**
53-
* The names of tokens in the associated grammar, stored as an array of
54-
* strings {@link String}{@code []}.
54+
* The token vocabulary, stored as a {@link Vocabulary} instance.
5555
*/
56-
public static final String PROP_TOKEN_NAMES = "Token Names";
56+
public static final String PROP_VOCABULARY = "Vocabulary";
5757
/**
5858
* The names of rules in the associated grammar, stored as an array of
5959
* strings {@link String}{@code []}.
@@ -80,19 +80,24 @@ public void read(Reader in, Document doc, int pos) throws IOException, BadLocati
8080
super.read(new InputStreamReader(inputStream, UTF_8), doc, pos);
8181

8282
// read the token names
83-
int tokenNamesOffset = 4 + inputSize;
84-
int tokenNamesSize = readInteger(binary, tokenNamesOffset);
85-
String[] tokenNames = readStrings(binary, tokenNamesOffset + 4, tokenNamesSize);
86-
doc.putProperty(PROP_TOKEN_NAMES, tokenNames);
83+
int literalNamesOffset = 4 + inputSize;
84+
int literalNamesSize = readInteger(binary, literalNamesOffset);
85+
String[] literalNames = readStrings(binary, literalNamesOffset + 4, literalNamesSize);
86+
87+
int symbolicNamesOffset = literalNamesOffset + 4 + literalNamesSize;
88+
int symbolicNamesSize = readInteger(binary, symbolicNamesOffset);
89+
String[] symbolicNames = readStrings(binary, symbolicNamesOffset + 4, symbolicNamesSize);
90+
91+
doc.putProperty(PROP_VOCABULARY, new VocabularyImpl(literalNames, symbolicNames));
8792

8893
// read the rule names
89-
int ruleNamesOffset = 4 + inputSize;
94+
int ruleNamesOffset = symbolicNamesOffset + 4 + symbolicNamesSize;
9095
int ruleNamesSize = readInteger(binary, ruleNamesOffset);
9196
String[] ruleNames = readStrings(binary, ruleNamesOffset + 4, ruleNamesSize);
9297
doc.putProperty(PROP_RULE_NAMES, ruleNames);
9398

9499
// read the mode names
95-
int modeNamesOffset = tokenNamesOffset + 4 + tokenNamesSize;
100+
int modeNamesOffset = ruleNamesOffset + 4 + ruleNamesSize;
96101
int modeNamesSize = readInteger(binary, modeNamesOffset);
97102
String[] modeNames = readStrings(binary, modeNamesOffset + 4, modeNamesSize);
98103
doc.putProperty(PROP_MODE_NAMES, modeNames);

‎org-antlr-works-editor/src/org/antlr/works/editor/grammar/debugger/AbstractInterpreterData.java

+2-45
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,8 @@
99

1010
package org.antlr.works.editor.grammar.debugger;
1111

12-
import java.util.ArrayList;
1312
import java.util.List;
14-
import java.util.Map;
15-
import org.antlr.v4.misc.Utils;
16-
import org.antlr.v4.tool.Grammar;
17-
import org.antlr.works.editor.grammar.debugger.LexerDebuggerControllerTopComponent.TokenDescriptor;
13+
import org.antlr.v4.runtime.Vocabulary;
1814

1915
/**
2016
*
@@ -24,47 +20,8 @@ public class AbstractInterpreterData {
2420

2521
public String grammarFileName;
2622
public String serializedAtn;
27-
public List<TokenDescriptor> tokenNames;
23+
public Vocabulary vocabulary;
2824
public List<String> ruleNames;
2925
public int startRuleIndex;
3026

31-
public static TokenDescriptor[] getTokenNames(Grammar grammar) {
32-
int numTokens = grammar.getMaxTokenType();
33-
List<String> typeToStringLiteralList = new ArrayList<>(grammar.typeToStringLiteralList);
34-
Utils.setSize(typeToStringLiteralList, numTokens + 1);
35-
for (Map.Entry<String, Integer> entry : grammar.stringLiteralToTypeMap.entrySet()) {
36-
if (entry.getValue() < 0 || entry.getValue() >= typeToStringLiteralList.size()) {
37-
continue;
38-
}
39-
40-
typeToStringLiteralList.set(entry.getValue(), entry.getKey());
41-
}
42-
43-
TokenDescriptor[] tokenNames = new TokenDescriptor[numTokens+1];
44-
for (int i = 0; i < tokenNames.length; i++) {
45-
tokenNames[i] = new TokenDescriptor();
46-
}
47-
48-
for (String tokenName : grammar.tokenNameToTypeMap.keySet()) {
49-
Integer ttype = grammar.tokenNameToTypeMap.get(tokenName);
50-
if (ttype < 0 || ttype >= tokenNames.length) {
51-
continue;
52-
}
53-
54-
if ( tokenName!=null && tokenName.startsWith(Grammar.AUTO_GENERATED_TOKEN_NAME_PREFIX) ) {
55-
if (ttype < typeToStringLiteralList.size()) {
56-
String literal = typeToStringLiteralList.get(ttype);
57-
if (literal != null) {
58-
tokenNames[ttype].literal = literal;
59-
}
60-
}
61-
}
62-
63-
tokenNames[ttype].name = tokenName;
64-
tokenNames[ttype].value = ttype;
65-
}
66-
67-
return tokenNames;
68-
}
69-
7027
}

‎org-antlr-works-editor/src/org/antlr/works/editor/grammar/debugger/LexerDebuggerControllerTopComponent.java

+37-22
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@
3131
import javax.swing.text.JTextComponent;
3232
import javax.swing.text.StyledDocument;
3333
import org.antlr.v4.runtime.Lexer;
34+
import org.antlr.v4.runtime.Token;
35+
import org.antlr.v4.runtime.Vocabulary;
36+
import org.antlr.v4.runtime.VocabularyImpl;
3437
import org.netbeans.api.annotations.common.NonNull;
3538
import org.netbeans.api.editor.EditorRegistry;
3639
import org.netbeans.api.settings.ConvertAsProperties;
@@ -601,29 +604,46 @@ public void propertyChange(PropertyChangeEvent evt) {
601604
}
602605

603606
Document document = component.getDocument();
604-
TokenDescriptor[] tokenDescriptorArray = (TokenDescriptor[])document.getProperty(LexerDebuggerEditorKit.PROP_TOKEN_NAMES);
607+
Vocabulary vocabulary = (Vocabulary)document.getProperty(LexerDebuggerEditorKit.PROP_VOCABULARY);
605608
String[] modeNamesArray = (String[])document.getProperty(LexerDebuggerEditorKit.PROP_MODE_NAMES);
606-
List<TokenDescriptor> tokenDescriptors = tokenDescriptorArray != null ? Arrays.asList(tokenDescriptorArray) : Collections.<TokenDescriptor>emptyList();
607609
List<String> modeNames = modeNamesArray != null ? Arrays.asList(modeNamesArray) : Collections.<String>emptyList();
608-
if (tokenDescriptors.isEmpty()) {
610+
if (vocabulary == null) {
609611
LexerInterpreterData lexerInterpreterData = (LexerInterpreterData)document.getProperty(LexerDebuggerEditorKit.PROP_LEXER_INTERP_DATA);
610612
if (lexerInterpreterData != null) {
611-
tokenDescriptors = lexerInterpreterData.tokenNames;
613+
vocabulary = lexerInterpreterData.vocabulary;
612614
modeNames = lexerInterpreterData.modeNames;
613615
}
614616
}
615617

616-
final List<TokenDescriptor> finalTokenDescriptors = tokenDescriptors;
618+
final Vocabulary finalVocabulary = vocabulary != null ? vocabulary : VocabularyImpl.EMPTY_VOCABULARY;
617619
final List<String> finalModeNames = modeNames;
618620

619621
currentComponent = component;
620622

621623
tblTokenTypes.setModel(new AbstractTableModel() {
622-
private final List<TokenDescriptor> elements = finalTokenDescriptors;
624+
private final List<String> literalNames = new ArrayList<>();
625+
private final List<String> symbolicNames = new ArrayList<>();
626+
private final List<Integer> values = new ArrayList<>();
627+
628+
{
629+
// TODO: Find a better way to communicate this value
630+
int maxTokenType = 1024;
631+
for (int i = 0; i <= maxTokenType; i++) {
632+
String literalName = finalVocabulary.getLiteralName(i);
633+
String symbolicName = finalVocabulary.getSymbolicName(i);
634+
if (literalName == null && symbolicName == null) {
635+
continue;
636+
}
637+
638+
literalNames.add(literalName != null ? literalName : "");
639+
symbolicNames.add(symbolicName != null ? symbolicName : literalName);
640+
values.add(i);
641+
}
642+
}
623643

624644
@Override
625645
public int getRowCount() {
626-
return elements.size();
646+
return literalNames.size();
627647
}
628648

629649
@Override
@@ -664,11 +684,11 @@ public Class<?> getColumnClass(int columnIndex) {
664684
public Object getValueAt(int rowIndex, int columnIndex) {
665685
switch (columnIndex) {
666686
case 0:
667-
return elements.get(rowIndex).name;
687+
return symbolicNames.get(rowIndex);
668688
case 1:
669-
return elements.get(rowIndex).literal;
689+
return literalNames.get(rowIndex);
670690
case 2:
671-
return elements.get(rowIndex).value;
691+
return values.get(rowIndex);
672692
default:
673693
throw new IllegalArgumentException();
674694
}
@@ -690,15 +710,16 @@ public TraceToken getElementAt(int index) {
690710

691711
});
692712

693-
String[] tokenNamesArray = new String[tokenDescriptors.size()];
694-
for (int i = 0; i < tokenDescriptors.size(); i++) {
695-
tokenNamesArray[i] = tokenDescriptors.get(i).literal;
696-
if (tokenNamesArray[i] == null || tokenNamesArray[i].isEmpty()) {
697-
tokenNamesArray[i] = tokenDescriptors.get(i).name;
713+
List<String> tokenNames = new ArrayList<>();
714+
for (int i = Token.EOF; i < 1024; i++) {
715+
if (finalVocabulary.getLiteralName(i) == null && finalVocabulary.getSymbolicName(i) == null) {
716+
continue;
698717
}
718+
719+
tokenNames.add(finalVocabulary.getDisplayName(i));
699720
}
700721

701-
lstTokens.setCellRenderer(new TraceTokenListCellRenderer(Arrays.asList(tokenNamesArray)));
722+
lstTokens.setCellRenderer(new TraceTokenListCellRenderer(tokenNames));
702723

703724
lstChannels.setModel(new AbstractListModel<Object>() {
704725
private final Object[] elements = { defaultChannelText, hiddenChannelText };
@@ -824,10 +845,4 @@ public Component getListCellRendererComponent(JList<?> list, Object value, int i
824845
}
825846

826847
}
827-
828-
public static class TokenDescriptor {
829-
public String name = "";
830-
public String literal = "";
831-
public int value;
832-
}
833848
}

‎org-antlr-works-editor/src/org/antlr/works/editor/grammar/debugger/LexerInterpreterData.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ public static LexerInterpreterData buildFromSnapshot(DocumentSnapshot snapshot)
8686
LexerInterpreterData data = new LexerInterpreterData();
8787
data.grammarFileName = lexerGrammar.fileName;
8888
data.serializedAtn = ATNSerializer.getSerializedAsString(lexerGrammar.atn, Arrays.asList(lexerGrammar.getRuleNames()));
89-
data.tokenNames = new ArrayList<>(Arrays.asList(getTokenNames(lexerGrammar)));
89+
data.vocabulary = lexerGrammar.getVocabulary();
9090
data.ruleNames = new ArrayList<>(lexerGrammar.rules.keySet());
9191
data.modeNames = new ArrayList<>(lexerGrammar.modes.keySet());
9292
return data;

‎org-antlr-works-editor/src/org/antlr/works/editor/grammar/debugger/ParserDebuggerReferenceAnchorsParserTask.java

+5-9
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
*/
99
package org.antlr.works.editor.grammar.debugger;
1010

11-
import java.util.ArrayList;
1211
import java.util.Arrays;
1312
import java.util.Collection;
1413
import java.util.Collections;
@@ -43,6 +42,7 @@
4342
import org.antlr.v4.runtime.Token;
4443
import org.antlr.v4.runtime.TokenSource;
4544
import org.antlr.v4.runtime.TokenStream;
45+
import org.antlr.v4.runtime.Vocabulary;
4646
import org.antlr.v4.runtime.atn.ATN;
4747
import org.antlr.v4.runtime.atn.ATNDeserializer;
4848
import org.antlr.v4.runtime.atn.ATNState;
@@ -53,7 +53,6 @@
5353
import org.antlr.works.editor.antlr4.classification.TaggerTokenSource;
5454
import org.antlr.works.editor.antlr4.parsing.DescriptiveErrorListener;
5555
import org.antlr.works.editor.antlr4.parsing.SyntaxErrorListener;
56-
import org.antlr.works.editor.grammar.debugger.LexerDebuggerControllerTopComponent.TokenDescriptor;
5756
import org.netbeans.api.editor.mimelookup.MimeRegistration;
5857

5958
/**
@@ -92,13 +91,10 @@ public void parse(ParserTaskManager taskManager, ParseContext context, DocumentS
9291

9392
ParserInterpreterData parserInterpreterData = (ParserInterpreterData)snapshot.getVersionedDocument().getDocument().getProperty(ParserDebuggerEditorKit.PROP_PARSER_INTERP_DATA);
9493
String grammarFileName = parserInterpreterData.grammarFileName;
95-
List<String> tokenNames = new ArrayList<>(parserInterpreterData.tokenNames.size());
96-
for (TokenDescriptor tokenDescriptor : parserInterpreterData.tokenNames) {
97-
tokenNames.add(tokenDescriptor.name);
98-
}
94+
Vocabulary vocabulary = parserInterpreterData.vocabulary;
9995
List<String> ruleNames = parserInterpreterData.ruleNames;
10096
ATN atn = new ATNDeserializer().deserialize(parserInterpreterData.serializedAtn.toCharArray());
101-
TracingParserInterpreter parser = new TracingParserInterpreter(grammarFileName, tokenNames, ruleNames, atn, tokenStream);
97+
TracingParserInterpreter parser = new TracingParserInterpreter(grammarFileName, vocabulary, ruleNames, atn, tokenStream);
10298

10399
long startTime = System.nanoTime();
104100
parser.setInterpreter(new StatisticsParserATNSimulator(parser, atn));
@@ -128,8 +124,8 @@ public void parse(ParserTaskManager taskManager, ParseContext context, DocumentS
128124
public static class TracingParserInterpreter extends ParserInterpreter {
129125
public final Map<ParseTree, Transition> associatedTransitions = new IdentityHashMap<>();
130126

131-
public TracingParserInterpreter(String grammarFileName, Collection<String> tokenNames, Collection<String> ruleNames, ATN atn, TokenStream input) {
132-
super(grammarFileName, tokenNames, ruleNames, atn, input);
127+
public TracingParserInterpreter(String grammarFileName, Vocabulary vocabulary, Collection<String> ruleNames, ATN atn, TokenStream input) {
128+
super(grammarFileName, vocabulary, ruleNames, atn, input);
133129
}
134130

135131
@Override

‎org-antlr-works-editor/src/org/antlr/works/editor/grammar/debugger/ParserDebuggerTokensTaskTaggerSnapshot.java

+7-16
Original file line numberDiff line numberDiff line change
@@ -8,19 +8,18 @@
88
*/
99
package org.antlr.works.editor.grammar.debugger;
1010

11-
import java.util.ArrayList;
1211
import java.util.Collection;
1312
import java.util.List;
1413
import org.antlr.netbeans.editor.text.DocumentSnapshot;
1514
import org.antlr.v4.runtime.CharStream;
1615
import org.antlr.v4.runtime.LexerInterpreter;
1716
import org.antlr.v4.runtime.TokenSource;
17+
import org.antlr.v4.runtime.Vocabulary;
1818
import org.antlr.v4.runtime.atn.ATN;
1919
import org.antlr.v4.runtime.atn.ATNDeserializer;
2020
import org.antlr.works.editor.antlr4.classification.AbstractTokensTaskTaggerSnapshot;
2121
import org.antlr.works.editor.antlr4.classification.SimpleLexerState;
2222
import org.antlr.works.editor.antlr4.highlighting.TokenSourceWithStateV4;
23-
import org.antlr.works.editor.grammar.debugger.LexerDebuggerControllerTopComponent.TokenDescriptor;
2423
import org.netbeans.api.annotations.common.NonNull;
2524

2625
/**
@@ -49,31 +48,23 @@ protected SimpleLexerState getStartState() {
4948
@Override
5049
protected TokenSourceWithStateV4<SimpleLexerState> createLexer(CharStream input, SimpleLexerState startState) {
5150
ATN atn = new ATNDeserializer().deserialize(lexerInterpreterData.serializedAtn.toCharArray());
52-
List<String> tokenNames = new ArrayList<>();
53-
for (TokenDescriptor tokenDescriptor : lexerInterpreterData.tokenNames) {
54-
tokenNames.add(tokenDescriptor.name);
55-
}
56-
51+
Vocabulary vocabulary = lexerInterpreterData.vocabulary;
5752
String grammarFileName = lexerInterpreterData.grammarFileName;
5853
List<String> ruleNames = lexerInterpreterData.ruleNames;
5954
List<String> modeNames = lexerInterpreterData.modeNames;
60-
ParserDebuggerLexerWrapper lexer = new ParserDebuggerLexerWrapper(grammarFileName, tokenNames, ruleNames, modeNames, atn, input);
55+
ParserDebuggerLexerWrapper lexer = new ParserDebuggerLexerWrapper(grammarFileName, vocabulary, ruleNames, modeNames, atn, input);
6156
startState.apply(lexer);
6257
return lexer;
6358
}
6459

6560
@Override
6661
protected TokenSource getEffectiveTokenSource(TokenSourceWithStateV4<SimpleLexerState> lexer) {
6762
ATN atn = new ATNDeserializer().deserialize(lexerInterpreterData.serializedAtn.toCharArray());
68-
List<String> tokenNames = new ArrayList<>();
69-
for (TokenDescriptor tokenDescriptor : lexerInterpreterData.tokenNames) {
70-
tokenNames.add(tokenDescriptor.name);
71-
}
72-
63+
Vocabulary vocabulary = lexerInterpreterData.vocabulary;
7364
String grammarFileName = lexerInterpreterData.grammarFileName;
7465
List<String> ruleNames = lexerInterpreterData.ruleNames;
7566
List<String> modeNames = lexerInterpreterData.modeNames;
76-
return new ParserDebuggerLexerWrapper(grammarFileName, tokenNames, ruleNames, modeNames, atn, lexer.getInputStream());
67+
return new ParserDebuggerLexerWrapper(grammarFileName, vocabulary, ruleNames, modeNames, atn, lexer.getInputStream());
7768
}
7869

7970
@Override
@@ -83,8 +74,8 @@ protected ParserDebuggerTokensTaskTaggerSnapshot translateToImpl(@NonNull Docume
8374

8475
private static class ParserDebuggerLexerWrapper extends LexerInterpreter implements TokenSourceWithStateV4<SimpleLexerState> {
8576

86-
public ParserDebuggerLexerWrapper(String grammarFileName, Collection<String> tokenNames, Collection<String> ruleNames, Collection<String> modeNames, ATN atn, CharStream input) {
87-
super(grammarFileName, tokenNames, ruleNames, modeNames, atn, input);
77+
public ParserDebuggerLexerWrapper(String grammarFileName, Vocabulary vocabulary, Collection<String> ruleNames, Collection<String> modeNames, ATN atn, CharStream input) {
78+
super(grammarFileName, vocabulary, ruleNames, modeNames, atn, input);
8879
}
8980

9081
@Override

0 commit comments

Comments
 (0)