update PythonLexerBase

Author: RobEin

port_CSharp/PythonLexerBase.cs

@@ -33,9 +33,9 @@ THE SOFTWARE.
 public abstract class PythonLexerBase : Lexer
 {
     // A stack that keeps track of the indentation lengths
-    private readonly Stack<int> _indentLengthStack = new Stack<int>();
+    private Stack<int> _indentLengthStack = new Stack<int>();
     // A list where tokens are waiting to be loaded into the token stream
-    private readonly LinkedList<IToken> _pendingTokens = new LinkedList<IToken>();
+    private LinkedList<IToken> _pendingTokens = new LinkedList<IToken>();
     // last pending token types
     private int _previousPendingTokenType = 0;
     private int _lastPendingTokenTypeFromDefaultChannel = 0;
@@ -61,6 +61,21 @@ protected PythonLexerBase(ICharStream input, TextWriter output, TextWriter error
     {
     }
 
+    public override void Reset()
+    {
+        _indentLengthStack = new Stack<int>();
+        _pendingTokens = new LinkedList<IToken>();
+        _previousPendingTokenType = 0;
+        _lastPendingTokenTypeFromDefaultChannel = 0;
+        _opened = 0;
+        _wasSpaceIndentation = false;
+        _wasTabIndentation = false;
+        _wasIndentationMixedWithSpacesAndTabs = false;
+        _curToken = null!;
+        _ffgToken = null!;
+        base.Reset();
+    }
+
     public override IToken NextToken() // reading the input stream until a return EOF
     {
         CheckNextToken();

port_JavaScript/PythonLexerBase.js

@@ -51,8 +51,8 @@ export default class PythonLexerBase extends antlr4.Lexer {
         this._wasIndentationMixedWithSpacesAndTabs = false;
         this._INVALID_LENGTH = -1;
 
-        this._curToken; // current (under processing) token
-        this._ffgToken; // following (look ahead) token
+        this._curToken = null; // current (under processing) token
+        this._ffgToken = null; // following (look ahead) token
 
         this._ERR_TXT = " ERROR: ";
     }
@@ -309,4 +309,20 @@ export default class PythonLexerBase extends antlr4.Lexer {
         // the ERROR_TOKEN will raise an error in the parser
         this.createAndAddPendingToken(PythonLexer.ERROR_TOKEN, Token.DEFAULT_CHANNEL, this._ERR_TXT + errMsg, this._ffgToken);
     }
+
+    reset() {
+        this._indentLengthStack = [];
+        this._pendingTokens = [];
+        this._previousPendingTokenType = 0;
+        this._lastPendingTokenTypeFromDefaultChannel = 0;
+        this._opened = 0;
+        this._wasSpaceIndentation = false;
+        this._wasTabIndentation = false;
+        this._wasIndentationMixedWithSpacesAndTabs = false;
+        this._INVALID_LENGTH = -1;
+        this._curToken = null;
+        this._ffgToken = null;
+        this._ERR_TXT = " ERROR: ";        
+        super.reset();
+    }
 }

port_Python3/PythonLexerBase.py

@@ -55,6 +55,21 @@ def __init__(self, input: InputStream, output: TextIO = sys.stdout):
 
         self._ERR_TXT: str = " ERROR: "
 
+    def reset(self):
+        self._indent_length_stack = deque()
+        self._pending_tokens = []
+        self._previous_pending_token_type = 0
+        self._last_pending_token_type_from_default_channel = 0
+        self._opened = 0
+        self._was_space_indentation = False
+        self._was_tab_indentation = False
+        self._was_indentation_mixed_with_spaces_and_tabs = False
+        self._INVALID_LENGTH = -1
+        self._cur_token = None
+        self._ffg_token = None
+        self._ERR_TXT = " ERROR: "
+        super().reset()
+
     def nextToken(self) -> CommonToken: # reading the input stream until a return EOF
         self.check_next_token()
         return self._pending_tokens.pop(0) # add the queued token to the token stream