Simplify the generation of parsers & bump antlr to 4.9.3 (#58)

**Motivation**: I tried to extend the grammar manually and found
following all single steps to generate the parser(s) tediously. So, I
would like to propose bundling the generation within one single Bash
script.

**Changes**: Add a Bash script in the fashion of
[speedy-antlr-example](https://github.com/amykyta3/speedy-antlr-example/blob/master/src/spam/parser/generate_parsers.sh);
update documentation accordingly.

**Update**: I added `speedy-antlr-tool=1.3.1` to conda-forge, which
requires `antlr=4.9.3` (instead of `.2`). So @SimeonStoykovQC & I
decided to bump the entire used `antlr` version within this PR.

Author: svengiegerich

docs/source/development.rst

@@ -14,5 +14,6 @@ dependencies:
   - sphinx_rtd_theme
   - sphinxcontrib-apidoc
   - sqlalchemy
-  - antlr-python-runtime==4.9.2
   - pyodbc
+  - speedy-antlr-tool==1.3.1
+  - antlr4-python3-runtime==4.9.3

environment.yml

@@ -37,7 +37,7 @@ requires-python = ">=3.7.0"
 
 dependencies = [
     "sqlalchemy >=1.4",
-    "antlr4-python3-runtime ==4.9.2",
+    "antlr4-python3-runtime ==4.9.3",
     "pyodbc >=4.0.30"
 ]
 

pyproject.toml

@@ -6,6 +6,7 @@
 #pragma once
 
 #include "ProxyErrorListener.h"
+#include "support/Casts.h"
 
 namespace antlr4 {
 
@@ -79,7 +80,7 @@ namespace antlr4 {
     /// @returns The ATN interpreter used by the recognizer for prediction.
     template <class T>
     T* getInterpreter() const {
-      return dynamic_cast<T *>(_interpreter);
+      return antlrcpp::downCast<T *>(_interpreter);
     }
 
     /**

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/ANTLRErrorListener.h

@@ -7,7 +7,7 @@
 
 using namespace antlr4;
 
-const std::string RuntimeMetaData::VERSION = "4.9.2";
+const std::string RuntimeMetaData::VERSION = "4.9.3";
 
 std::string RuntimeMetaData::getRuntimeVersion() {
   return VERSION;

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/ANTLRErrorStrategy.h

@@ -34,7 +34,6 @@
 #include <exception>
 #include <bitset>
 #include <condition_variable>
-#include <functional>
 
 #ifndef USE_UTF8_INSTEAD_OF_CODECVT
   #include <codecvt>
@@ -108,7 +107,7 @@
   #endif
 #endif
 
-#include "support/guid.h"
+#include "support/Guid.h"
 #include "support/Declarations.h"
 
 #if !defined(HAS_NOEXCEPT)

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/ANTLRFileStream.cpp

@@ -131,9 +131,10 @@
 #include "support/Any.h"
 #include "support/Arrays.h"
 #include "support/BitSet.h"
+#include "support/Casts.h"
 #include "support/CPPUtils.h"
 #include "support/StringUtils.h"
-#include "support/guid.h"
+#include "support/Guid.h"
 #include "tree/AbstractParseTreeVisitor.h"
 #include "tree/ErrorNode.h"
 #include "tree/ErrorNodeImpl.h"

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/ANTLRFileStream.h

@@ -7,6 +7,12 @@
 
 #include "RuleContext.h"
 
+// GCC generates a warning when forward-declaring ATN if ATN has already been
+// declared due to the attributes added by ANTLR4CPP_PUBLIC.
+// See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=39159
+// Add constant that can be checked so forward-declarations can be omitted.
+#define ANTLR4CPP_ATN_DECLARED
+
 namespace antlr4 {
 namespace atn {
 

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/ANTLRInputStream.cpp

@@ -7,7 +7,7 @@
 
 using namespace antlr4::atn;
 
-ATNDeserializationOptions ATNDeserializationOptions::defaultOptions;
+const ATNDeserializationOptions ATNDeserializationOptions::defaultOptions;
 
 ATNDeserializationOptions::ATNDeserializationOptions() {
   InitializeInstanceFields();

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/ANTLRInputStream.h

@@ -12,7 +12,7 @@ namespace atn {
 
   class ANTLR4CPP_PUBLIC ATNDeserializationOptions {
   private:
-    static ATNDeserializationOptions defaultOptions;
+    static const ATNDeserializationOptions defaultOptions;
 
     bool readOnly;
     bool verifyATN;

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/BailErrorStrategy.cpp

@@ -115,32 +115,32 @@ ATNDeserializer::~ATNDeserializer() {
  * This value should never change. Updates following this version are
  * reflected as change in the unique ID SERIALIZED_UUID.
  */
-Guid ATNDeserializer::ADDED_PRECEDENCE_TRANSITIONS() {
-  return Guid("1DA0C57D-6C06-438A-9B27-10BCB3CE0F61");
+antlrcpp::Guid ATNDeserializer::ADDED_PRECEDENCE_TRANSITIONS() {
+  return antlrcpp::Guid("1DA0C57D-6C06-438A-9B27-10BCB3CE0F61");
 }
 
-Guid ATNDeserializer::ADDED_LEXER_ACTIONS() {
-  return Guid("AADB8D7E-AEEF-4415-AD2B-8204D6CF042E");
+antlrcpp::Guid ATNDeserializer::ADDED_LEXER_ACTIONS() {
+  return antlrcpp::Guid("AADB8D7E-AEEF-4415-AD2B-8204D6CF042E");
 }
 
-Guid ATNDeserializer::ADDED_UNICODE_SMP() {
-  return Guid("59627784-3BE5-417A-B9EB-8131A7286089");
+antlrcpp::Guid ATNDeserializer::ADDED_UNICODE_SMP() {
+  return antlrcpp::Guid("59627784-3BE5-417A-B9EB-8131A7286089");
 }
 
-Guid ATNDeserializer::SERIALIZED_UUID() {
+antlrcpp::Guid ATNDeserializer::SERIALIZED_UUID() {
   return ADDED_UNICODE_SMP();
 }
 
-Guid ATNDeserializer::BASE_SERIALIZED_UUID() {
-  return Guid("33761B2D-78BB-4A43-8B0B-4F5BEE8AACF3");
+antlrcpp::Guid ATNDeserializer::BASE_SERIALIZED_UUID() {
+  return antlrcpp::Guid("33761B2D-78BB-4A43-8B0B-4F5BEE8AACF3");
 }
 
-std::vector<Guid>& ATNDeserializer::SUPPORTED_UUIDS() {
-  static std::vector<Guid> singleton = { BASE_SERIALIZED_UUID(), ADDED_PRECEDENCE_TRANSITIONS(), ADDED_LEXER_ACTIONS(), ADDED_UNICODE_SMP() };
+std::vector<antlrcpp::Guid>& ATNDeserializer::SUPPORTED_UUIDS() {
+  static std::vector<antlrcpp::Guid> singleton = { BASE_SERIALIZED_UUID(), ADDED_PRECEDENCE_TRANSITIONS(), ADDED_LEXER_ACTIONS(), ADDED_UNICODE_SMP() };
   return singleton;
 }
 
-bool ATNDeserializer::isFeatureSupported(const Guid &feature, const Guid &actualUuid) {
+bool ATNDeserializer::isFeatureSupported(const antlrcpp::Guid &feature, const antlrcpp::Guid &actualUuid) {
   auto featureIterator = std::find(SUPPORTED_UUIDS().begin(), SUPPORTED_UUIDS().end(), feature);
   if (featureIterator == SUPPORTED_UUIDS().end()) {
     return false;
@@ -169,7 +169,7 @@ ATN ATNDeserializer::deserialize(const std::vector<uint16_t>& input) {
     throw UnsupportedOperationException(reason);
   }
 
-  Guid uuid = toUUID(data.data(), p);
+  antlrcpp::Guid uuid = toUUID(data.data(), p);
   p += 8;
   auto uuidIterator = std::find(SUPPORTED_UUIDS().begin(), SUPPORTED_UUIDS().end(), uuid);
   if (uuidIterator == SUPPORTED_UUIDS().end()) {
@@ -628,8 +628,8 @@ void ATNDeserializer::checkCondition(bool condition, const std::string &message)
   }
 }
 
-Guid ATNDeserializer::toUUID(const unsigned short *data, size_t offset) {
-  return Guid((uint16_t *)data + offset, true);
+antlrcpp::Guid ATNDeserializer::toUUID(const unsigned short *data, size_t offset) {
+  return antlrcpp::Guid((uint16_t *)data + offset, true);
 }
 
 /* mem check: all created instances are freed in the d-tor of the ATNState they are added to. */

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/BailErrorStrategy.h

@@ -23,13 +23,13 @@ namespace atn {
 
     /// This is the current serialized UUID.
     // ml: defined as function to avoid the “static initialization order fiasco”.
-    static Guid SERIALIZED_UUID();
+    static antlrcpp::Guid SERIALIZED_UUID();
 
     ATNDeserializer();
     ATNDeserializer(const ATNDeserializationOptions& dso);
     virtual ~ATNDeserializer();
 
-    static Guid toUUID(const unsigned short *data, size_t offset);
+    static antlrcpp::Guid toUUID(const unsigned short *data, size_t offset);
 
     virtual ATN deserialize(const std::vector<uint16_t> &input);
     virtual void verifyATN(const ATN &atn);
@@ -54,35 +54,35 @@ namespace atn {
     /// <returns> {@code true} if the {@code actualUuid} value represents a
     /// serialized ATN at or after the feature identified by {@code feature} was
     /// introduced; otherwise, {@code false}. </returns>
-    virtual bool isFeatureSupported(const Guid &feature, const Guid &actualUuid);
+    virtual bool isFeatureSupported(const antlrcpp::Guid &feature, const antlrcpp::Guid &actualUuid);
     void markPrecedenceDecisions(const ATN &atn);
     Ref<LexerAction> lexerActionFactory(LexerActionType type, int data1, int data2);
 
   private:
     /// This is the earliest supported serialized UUID.
-    static Guid BASE_SERIALIZED_UUID();
+    static antlrcpp::Guid BASE_SERIALIZED_UUID();
 
     /// This UUID indicates an extension of <seealso cref="BASE_SERIALIZED_UUID"/> for the
     /// addition of precedence predicates.
-    static Guid ADDED_PRECEDENCE_TRANSITIONS();
+    static antlrcpp::Guid ADDED_PRECEDENCE_TRANSITIONS();
 
     /**
      * This UUID indicates an extension of ADDED_PRECEDENCE_TRANSITIONS
      * for the addition of lexer actions encoded as a sequence of
      * LexerAction instances.
      */
-    static Guid ADDED_LEXER_ACTIONS();
+    static antlrcpp::Guid ADDED_LEXER_ACTIONS();
 
     /**
      * This UUID indicates the serialized ATN contains two sets of
      * IntervalSets, where the second set's values are encoded as
      * 32-bit integers to support the full Unicode SMP range up to U+10FFFF.
      */
-    static Guid ADDED_UNICODE_SMP();
+    static antlrcpp::Guid ADDED_UNICODE_SMP();
 
     /// This list contains all of the currently supported UUIDs, ordered by when
     /// the feature first appeared in this branch.
-    static std::vector<Guid>& SUPPORTED_UUIDS();
+    static std::vector<antlrcpp::Guid>& SUPPORTED_UUIDS();
 
     ATNDeserializationOptions deserializationOptions;
   };

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/BaseErrorListener.cpp

@@ -390,7 +390,7 @@ std::string ATNSerializer::decode(const std::wstring &inpdata) {
     throw UnsupportedOperationException("ATN Serializer" + reason);
   }
 
-  Guid uuid = ATNDeserializer::toUUID(data.data(), p);
+  antlrcpp::Guid uuid = ATNDeserializer::toUUID(data.data(), p);
   p += 8;
   if (uuid != ATNDeserializer::SERIALIZED_UUID()) {
     std::string reason = "Could not deserialize ATN with UUID " + uuid.toString() + " (expected " +
@@ -602,10 +602,10 @@ std::string ATNSerializer::getDecoded(ATN *atn, std::vector<std::string> &tokenN
   return ATNSerializer(atn, tokenNames).decode(serialized);
 }
 
-void ATNSerializer::serializeUUID(std::vector<size_t> &data, Guid uuid) {
+void ATNSerializer::serializeUUID(std::vector<size_t> &data, antlrcpp::Guid uuid) {
   unsigned int twoBytes = 0;
   bool firstByte = true;
-  for( std::vector<unsigned char>::const_reverse_iterator rit = uuid.rbegin(); rit != uuid.rend(); ++rit )
+  for(antlrcpp::Guid::const_reverse_iterator rit = uuid.rbegin(); rit != uuid.rend(); ++rit )
   {
      if (firstByte) {
        twoBytes = *rit;

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/BaseErrorListener.h

@@ -54,7 +54,7 @@ namespace atn {
   private:
     std::vector<std::string> _tokenNames;
 
-    void serializeUUID(std::vector<size_t> &data, Guid uuid);
+    void serializeUUID(std::vector<size_t> &data, antlrcpp::Guid uuid);
   };
 
 } // namespace atn

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/BufferedTokenStream.cpp

@@ -70,7 +70,14 @@ namespace atn {
   ///
   /// <embed src="images/OptionalNonGreedy.svg" type="image/svg+xml"/>
   /// </summary>
+
+// GCC generates a warning here if ATN has already been declared due to the
+// attributes added by ANTLR4CPP_PUBLIC.
+// See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=39159
+// Only forward-declare if it hasn't already been declared.
+#ifndef ANTLR4CPP_ATN_DECLARED
   class ANTLR4CPP_PUBLIC ATN;
+#endif
 
   class ANTLR4CPP_PUBLIC ATNState {
   public:

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/BufferedTokenStream.h

@@ -47,7 +47,7 @@ void LexerATNSimulator::SimState::InitializeInstanceFields() {
   charPos = INVALID_INDEX;
 }
 
-int LexerATNSimulator::match_calls = 0;
+std::atomic<int> LexerATNSimulator::match_calls(0);
 
 
 LexerATNSimulator::LexerATNSimulator(const ATN &atn, std::vector<dfa::DFA> &decisionToDFA,
@@ -69,7 +69,7 @@ void LexerATNSimulator::copyState(LexerATNSimulator *simulator) {
 }
 
 size_t LexerATNSimulator::match(CharStream *input, size_t mode) {
-  match_calls++;
+  match_calls.fetch_add(1, std::memory_order_relaxed);
   _mode = mode;
   ssize_t mark = input->mark();
 

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/CharStream.cpp

@@ -5,6 +5,8 @@
 
 #pragma once
 
+#include <atomic>
+
 #include "atn/ATNSimulator.h"
 #include "atn/LexerATNConfig.h"
 #include "atn/ATNConfigSet.h"
@@ -89,7 +91,7 @@ namespace atn {
     SimState _prevAccept;
 
   public:
-    static int match_calls;
+    static std::atomic<int> match_calls;
 
     LexerATNSimulator(const ATN &atn, std::vector<dfa::DFA> &decisionToDFA, PredictionContextCache &sharedContextCache);
     LexerATNSimulator(Lexer *recog, const ATN &atn, std::vector<dfa::DFA> &decisionToDFA, PredictionContextCache &sharedContextCache);

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/CharStream.h

@@ -20,12 +20,12 @@ using namespace antlr4::atn;
 
 using namespace antlrcpp;
 
-size_t PredictionContext::globalNodeCount = 0;
+std::atomic<size_t> PredictionContext::globalNodeCount(0);
 const Ref<PredictionContext> PredictionContext::EMPTY = std::make_shared<EmptyPredictionContext>();
 
 //----------------- PredictionContext ----------------------------------------------------------------------------------
 
-PredictionContext::PredictionContext(size_t cachedHashCode) : id(globalNodeCount++), cachedHashCode(cachedHashCode)  {
+PredictionContext::PredictionContext(size_t cachedHashCode) : id(globalNodeCount.fetch_add(1, std::memory_order_relaxed)), cachedHashCode(cachedHashCode)  {
 }
 
 PredictionContext::~PredictionContext() {

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/CommonToken.cpp

@@ -5,6 +5,8 @@
 
 #pragma once
 
+#include <atomic>
+
 #include "Recognizer.h"
 #include "atn/ATN.h"
 #include "atn/ATNState.h"
@@ -48,7 +50,7 @@ namespace atn {
 #endif
 
   public:
-    static size_t globalNodeCount;
+    static std::atomic<size_t> globalNodeCount;
     const size_t id;
 
     /// <summary>

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/CommonToken.h

@@ -65,16 +65,26 @@ struct ANTLR4CPP_PUBLIC Any
     return derived->value;
   }
 
-  template<class U>
+  template<class U, typename std::enable_if<std::is_copy_constructible<U>::value || std::is_copy_assignable<U>::value>::value>
   operator U() {
     return as<StorageType<U>>();
   }
 
-  template<class U>
+  template<class U, typename std::enable_if<(!std::is_copy_constructible<U>::value && !std::is_copy_assignable<U>::value) && (std::is_move_constructible<U>::value || std::is_move_assignable<U>::value)>::value>
+  operator U() {
+    return std::move(as<StorageType<U>>());
+  }
+
+  template<class U, typename std::enable_if<std::is_copy_constructible<U>::value || std::is_copy_assignable<U>::value>::value>
   operator const U() const {
     return as<const StorageType<U>>();
   }
 
+  template<class U, typename std::enable_if<!(!std::is_copy_constructible<U>::value && !std::is_copy_assignable<U>::value) && (std::is_move_constructible<U>::value || std::is_move_assignable<U>::value)>::value>
+  operator const U() const {
+    return std::move(as<const StorageType<U>>());
+  }
+
   Any& operator = (const Any& a) {
     if (_ptr == a._ptr)
       return *this;
@@ -99,7 +109,7 @@ struct ANTLR4CPP_PUBLIC Any
 
   virtual ~Any();
 
-  virtual bool equals(Any other) const {
+  virtual bool equals(const Any& other) const {
     return _ptr == other._ptr;
   }