add init method

Author: RobEin

examples/test.py example.pyexamples/test.py renamed to example.py

@@ -33,35 +33,37 @@ THE SOFTWARE.
 public abstract class PythonLexerBase : Lexer
 {
     // A stack that keeps track of the indentation lengths
-    private Stack<int> _indentLengthStack = new Stack<int>();
+    private Stack<int> _indentLengthStack;
     // A list where tokens are waiting to be loaded into the token stream
-    private LinkedList<IToken> _pendingTokens = new LinkedList<IToken>();
+    private LinkedList<IToken> _pendingTokens;
     // last pending token types
-    private int _previousPendingTokenType = 0;
-    private int _lastPendingTokenTypeFromDefaultChannel = 0;
+    private int _previousPendingTokenType;
+    private int _lastPendingTokenTypeFromDefaultChannel;
 
     // The amount of opened parentheses, square brackets, or curly braces
-    private int _opened = 0;
+    private int _opened;
 
-    private bool _wasSpaceIndentation = false;
-    private bool _wasTabIndentation = false;
-    private bool _wasIndentationMixedWithSpacesAndTabs = false;
+    private bool _wasSpaceIndentation;
+    private bool _wasTabIndentation;
+    private bool _wasIndentationMixedWithSpacesAndTabs;
     private const int INVALID_LENGTH = -1;
 
-    private CommonToken _curToken = null!; // current (under processing) token
-    private IToken _ffgToken = null!;      // following (look ahead) token
+    private CommonToken _curToken; // current (under processing) token
+    private IToken _ffgToken;      // following (look ahead) token
 
     private const string _ERR_TXT = " ERROR: ";
 
     protected PythonLexerBase(ICharStream input) : base(input)
     {
+        Init();
     }
 
     protected PythonLexerBase(ICharStream input, TextWriter output, TextWriter errorOutput) : base(input, output, errorOutput)
     {
+        Init();
     }
 
-    public override void Reset()
+    private void Init()
     {
         _indentLengthStack = new Stack<int>();
         _pendingTokens = new LinkedList<IToken>();
@@ -73,7 +75,6 @@ public override void Reset()
         _wasIndentationMixedWithSpacesAndTabs = false;
         _curToken = null!;
         _ffgToken = null!;
-        base.Reset();
     }
 
     public override IToken NextToken() // reading the input stream until a return EOF
@@ -385,4 +386,10 @@ private void ReportError(string errMsg)
         // the ERROR_TOKEN will raise an error in the parser
         CreateAndAddPendingToken(PythonLexer.ERROR_TOKEN, TokenConstants.DefaultChannel, _ERR_TXT + errMsg, _ffgToken);
     }
+
+    public override void Reset()
+    {
+        Init();
+        base.Reset();
+    }
 }

port_CSharp/PythonLexerBase.cs

@@ -1,7 +1,7 @@
 ### C#
 
 #### Command line example for Windows:
-- first create a C# project called grun_tokens then copy the two grammar files and test.py to this directory
+- first create a C# project called grun_tokens then copy the two grammar files and example.py to this directory
 ```bash
     dotnet new console -o . -n grun_tokens -f netcoreapp3.1
     del program.cs
@@ -10,14 +10,14 @@
 
 ```bash
     copy ..\*.g4
-    copy ..\examples\test.py
+    copy ..\example.py
 ```
 
 ```bash
 antlr4 -Dlanguage=CSharp PythonLexer.g4
 antlr4 -Dlanguage=CSharp PythonParser.g4
 dotnet build
-dotnet run test.py --no-build
+dotnet run example.py --no-build
 ```
 
 #### Related links:

port_CSharp/README.md

@@ -31,7 +31,7 @@ private static string GetTokenMetaDataWithTokenName(PythonParser parser, IToken
             return metaData.Substring(0, lesserPos + 2) // modified format:   [@TokenIndex,StartIndex:StopIndex='Text',<TokenName>,channel=Channel,Line:Column]
                  + parser.Vocabulary.GetSymbolicName(token.Type)
                  + metaData.Substring(greaterPos);
-        
+
         }
     }
 }

port_CSharp/grun_tokens.cs

@@ -33,33 +33,33 @@ of this software and associated documentation files (the "Software"), to deal
 
 public abstract class PythonLexerBase extends Lexer {
     // A stack that keeps track of the indentation lengths
-    private Deque<Integer> _indentLengthStack = new ArrayDeque<>();
+    private Deque<Integer> _indentLengthStack;
     // A linked where tokens are waiting to be loaded into the token stream
-    private LinkedList<Token> _pendingTokens = new LinkedList<>();
+    private LinkedList<Token> _pendingTokens;
 
     // last pending token types
-    private int _previousPendingTokenType = 0;
-    private int _lastPendingTokenTypeFromDefaultChannel = 0;
+    private int _previousPendingTokenType;
+    private int _lastPendingTokenTypeFromDefaultChannel;
 
     // The amount of opened parentheses, square brackets or curly braces
-    private int _opened = 0;
+    private int _opened;
 
-    private boolean _wasSpaceIndentation = false;
-    private boolean _wasTabIndentation = false;
-    private boolean _wasIndentationMixedWithSpacesAndTabs = false;
+    private boolean _wasSpaceIndentation;
+    private boolean _wasTabIndentation;
+    private boolean _wasIndentationMixedWithSpacesAndTabs;
     private final int _INVALID_LENGTH = -1;
 
-    private CommonToken _curToken = null; // current (under processing) token
-    private Token _ffgToken = null; // following (look ahead) token
+    private CommonToken _curToken; // current (under processing) token
+    private Token _ffgToken; // following (look ahead) token
 
     private final String _ERR_TXT = " ERROR: ";
 
     protected PythonLexerBase(CharStream input) {
         super(input);
+        init();
     }
 
-    @Override
-    public void reset() {
+    private void init() {
         _indentLengthStack = new ArrayDeque<>();
         _pendingTokens = new LinkedList<>();
         _previousPendingTokenType = 0;
@@ -70,7 +70,6 @@ public void reset() {
         _wasIndentationMixedWithSpacesAndTabs = false;
         _curToken = null;
         _ffgToken = null;
-        super.reset();
     }
 
     @Override
@@ -316,4 +315,10 @@ private void reportError(final String errMsg) {
         // the ERROR_TOKEN will raise an error in the parser
         createAndAddPendingToken(PythonLexer.ERROR_TOKEN, Token.DEFAULT_CHANNEL, _ERR_TXT + errMsg, _ffgToken);
     }
+
+    @Override
+    public void reset() {
+        init();
+        super.reset();
+    }
 }

port_Java/PythonLexerBase.java

@@ -1,26 +1,26 @@
 ### Java 8
 
 #### Command line example:
-- first copy the two grammar files and the test.py to this directory
+- first copy the two grammar files and the example.py to this directory
 
 Unix:
 ```bash
     cp ../*.g4 .
-    cp ../examples/test.py .
+    cp ../example.py .
 ```
 
 Windows:
 ```bash
     copy ..\*.g4
-    copy ..\examples\test.py
+    copy ..\example.py
 ```
 
 ```bash
 antlr4 PythonLexer.g4
 antlr4 PythonParser.g4
 javac *.java
-grun Python file_input -tokens test.py
-grun Python file_input -gui test.py
+grun Python file_input -tokens example.py
+grun Python file_input -gui example.py
 ```
 
 #### Related link:

port_Java/README.md

@@ -35,29 +35,31 @@ export default class PythonLexerBase extends antlr4.Lexer {
         super(input);
 
         // A stack that keeps track of the indentation lengths
-        this._indentLengthStack = [];
+        this._indentLengthStack;
         // A list where tokens are waiting to be loaded into the token stream
-        this._pendingTokens = [];
+        this._pendingTokens;
 
         // last pending token types
-        this._previousPendingTokenType = 0;
-        this._lastPendingTokenTypeFromDefaultChannel = 0;
+        this._previousPendingTokenType;
+        this._lastPendingTokenTypeFromDefaultChannel;
 
         // The amount of opened parentheses, square brackets or curly braces
-        this._opened = 0;
+        this._opened;
 
-        this._wasSpaceIndentation = false;
-        this._wasTabIndentation = false;
-        this._wasIndentationMixedWithSpacesAndTabs = false;
-        this._INVALID_LENGTH = -1;
+        this._wasSpaceIndentation;
+        this._wasTabIndentation;
+        this._wasIndentationMixedWithSpacesAndTabs;
+        this._INVALID_LENGTH;
 
-        this._curToken = null; // current (under processing) token
-        this._ffgToken = null; // following (look ahead) token
+        this._curToken; // current (under processing) token
+        this._ffgToken; // following (look ahead) token
+
+        this._ERR_TXT;
 
-        this._ERR_TXT = " ERROR: ";
+        this.init();
     }
 
-    reset() {
+    init() {
         this._indentLengthStack = [];
         this._pendingTokens = [];
         this._previousPendingTokenType = 0;
@@ -70,7 +72,6 @@ export default class PythonLexerBase extends antlr4.Lexer {
         this._curToken = null;
         this._ffgToken = null;
         this._ERR_TXT = " ERROR: ";        
-        super.reset();
     }
 
     nextToken() { // reading the input stream until a return EOF
@@ -325,4 +326,9 @@ export default class PythonLexerBase extends antlr4.Lexer {
         // the ERROR_TOKEN will raise an error in the parser
         this.createAndAddPendingToken(PythonLexer.ERROR_TOKEN, Token.DEFAULT_CHANNEL, this._ERR_TXT + errMsg, this._ffgToken);
     }
+
+    reset() {
+        this.init();
+        super.reset();
+    }
 }

port_JavaScript/PythonLexerBase.js

@@ -20,4 +20,4 @@ tokens.fill();
 for (const token of tokens.tokens) {
     console.log(getTokenMetaDataWithTokenName(token));
 }
-const tree = parser.file_input();
+const tree = parser.file_input();

port_JavaScript/grun_tokens.js

@@ -33,29 +33,31 @@ def __init__(self, input: InputStream, output: TextIO = sys.stdout):
         super().__init__(input, output)
 
         # A stack that keeps track of the indentation lengths
-        self._indent_length_stack: Deque[int] = deque()
+        self._indent_length_stack: Deque[int]
 
         # A list where tokens are waiting to be loaded into the token stream
-        self._pending_tokens: list[CommonToken] = []
+        self._pending_tokens: list[CommonToken]
 
         # last pending token types
-        self._previous_pending_token_type: int = 0
-        self._last_pending_token_type_from_default_channel: int = 0
+        self._previous_pending_token_type: int
+        self._last_pending_token_type_from_default_channel: int
 
         # The amount of opened parentheses, square brackets or curly braces
-        self._opened: int = 0
+        self._opened: int
 
-        self._was_space_indentation: bool = False
-        self._was_tab_indentation: bool = False
-        self._was_indentation_mixed_with_spaces_and_tabs: bool = False
-        self._INVALID_LENGTH: int = -1
+        self._was_space_indentation: bool
+        self._was_tab_indentation: bool
+        self._was_indentation_mixed_with_spaces_and_tabs: bool
+        self._INVALID_LENGTH: int
 
-        self._cur_token: CommonToken = None # current (under processing) token
-        self._ffg_token: CommonToken = None # following (look ahead) token
+        self._cur_token: CommonToken # current (under processing) token
+        self._ffg_token: CommonToken # following (look ahead) token
 
-        self._ERR_TXT: str = " ERROR: "
+        self._ERR_TXT: str
 
-    def reset(self):
+        self.init()
+
+    def init(self):
         self._indent_length_stack = deque()
         self._pending_tokens = []
         self._previous_pending_token_type = 0
@@ -68,7 +70,6 @@ def reset(self):
         self._cur_token = None
         self._ffg_token = None
         self._ERR_TXT = " ERROR: "
-        super().reset()
 
     def nextToken(self) -> CommonToken: # reading the input stream until a return EOF
         self.check_next_token()
@@ -254,3 +255,7 @@ def report_error(self, err_msg):
 
         # the ERROR_TOKEN will raise an error in the parser
         self.create_and_add_pending_token(self.ERROR_TOKEN, Token.DEFAULT_CHANNEL, self._ERR_TXT + err_msg, self._ffg_token)
+
+    def reset(self):
+        self.init()
+        super().reset()