Bump antlr to 4.11 (#63)

**Motivation**: Version 4.11.1 is the latest of `antlr`. I don't see a
clear reason not to use that version (other than the package needs
`c++17` compared to `c++13`).
Besides, updating antlr could make it possible to use [other grammar
definitions](https://github.com/antlr/grammars-v4/blob/master/sql/tsql/TSqlParser.g4).

**Changes**: See the commit history:
1. bump dependencies, 
2. update `cpp_src/`, 
3. run `helper_generate_parsers.sh`, 
4. increase `c++` version.

---------

Co-authored-by: Uwe L. Korn <[email protected]>

Author: svengiegerich

docs/source/development.rst

@@ -68,8 +68,8 @@ in C++ for better performance. The parsed tree is then converted into the Python
 
 If you want to adapt the grammar please have a look at ``pytsql/src/pytsql/grammar/tsql.g4``.
 All files in ``pytsql/src/pytsql/grammar/cpp_src/antlr4-cpp-runtime`` are taken directly from
-the `ANTLR repository release 4.9.3
-<https://github.com/antlr/antlr4/tree/4.9.3/runtime/Cpp/runtime>`_
+the `ANTLR repository release 4.11.1
+<https://github.com/antlr/antlr4/tree/4.11.1/runtime/Cpp/runtime>`_
 and the rest of the files in ``pytsql/src/pytsql/grammar`` are generated by ``antlr`` or
 ``speedy-antlr-tool``.
 

environment.yml

@@ -15,5 +15,5 @@ dependencies:
   - sphinxcontrib-apidoc
   - sqlalchemy
   - pyodbc
-  - speedy-antlr-tool==1.3.1
-  - antlr4-python3-runtime==4.9.3
+  - speedy-antlr-tool==1.4.1
+  - antlr4-python3-runtime==4.11.1

pyproject.toml

@@ -37,7 +37,7 @@ requires-python = ">=3.7.0"
 
 dependencies = [
     "sqlalchemy >=1.4",
-    "antlr4-python3-runtime ==4.9.3",
+    "antlr4-python3-runtime ==4.11",
     "pyodbc >=4.0.30"
 ]
 

setup.py

@@ -40,10 +40,10 @@ def create_extension() -> setuptools.Extension:
     """Create the ANTLR C++ extension to be passed to `setuptools.setup`"""
 
     extra_compile_args = {
-        "windows": ["/DANTLR4CPP_STATIC", "/Zc:__cplusplus"],
-        "linux": ["-std=c++11"],
-        "darwin": ["-std=c++11"],
-        "cygwin": ["-std=c++11"],
+        "windows": ["/DANTLR4CPP_STATIC", "/Zc:__cplusplus", "/std:c++17"],
+        "linux": ["-std=c++17"],
+        "darwin": ["-std=c++17", "-D_LIBCPP_DISABLE_AVAILABILITY"],
+        "cygwin": ["-std=c++17"],
     }
 
     sources = glob.glob("src/pytsql/grammar/cpp_src/**/*.cpp", recursive=True)

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/ANTLRFileStream.cpp

@@ -3,8 +3,6 @@
  * can be found in the LICENSE.txt file in the project root.
  */
 
-#include "support/StringUtils.h"
-
 #include "ANTLRFileStream.h"
 
 using namespace antlr4;
@@ -15,11 +13,7 @@ void ANTLRFileStream::loadFromFile(const std::string &fileName) {
     return;
   }
 
-#ifdef _MSC_VER
-  std::ifstream stream(antlrcpp::s2ws(fileName), std::ios::binary);
-#else
   std::ifstream stream(fileName, std::ios::binary);
-#endif
 
   ANTLRInputStream::load(stream);
 }

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/ANTLRInputStream.cpp

@@ -9,7 +9,7 @@
 #include "misc/Interval.h"
 #include "IntStream.h"
 
-#include "support/StringUtils.h"
+#include "support/Utf8.h"
 #include "support/CPPUtils.h"
 
 #include "ANTLRInputStream.h"
@@ -23,15 +23,9 @@ ANTLRInputStream::ANTLRInputStream() {
   InitializeInstanceFields();
 }
 
-#if __cplusplus >= 201703L
-ANTLRInputStream::ANTLRInputStream(const std::string_view &input): ANTLRInputStream() {
+ANTLRInputStream::ANTLRInputStream(std::string_view input): ANTLRInputStream() {
   load(input.data(), input.length());
 }
-#endif
-
-ANTLRInputStream::ANTLRInputStream(const std::string &input): ANTLRInputStream() {
-  load(input.data(), input.size());
-}
 
 ANTLRInputStream::ANTLRInputStream(const char *data, size_t length) {
   load(data, length);
@@ -41,28 +35,37 @@ ANTLRInputStream::ANTLRInputStream(std::istream &stream): ANTLRInputStream() {
   load(stream);
 }
 
-void ANTLRInputStream::load(const std::string &input) {
-  load(input.data(), input.size());
+void ANTLRInputStream::load(const std::string &input, bool lenient) {
+  load(input.data(), input.size(), lenient);
 }
 
-void ANTLRInputStream::load(const char *data, size_t length) {
+void ANTLRInputStream::load(const char *data, size_t length, bool lenient) {
   // Remove the UTF-8 BOM if present.
   const char *bom = "\xef\xbb\xbf";
-  if (length >= 3 && strncmp(data, bom, 3) == 0)
-    _data = antlrcpp::utf8_to_utf32(data + 3, data + length);
-  else
-    _data = antlrcpp::utf8_to_utf32(data, data + length);
+  if (length >= 3 && strncmp(data, bom, 3) == 0) {
+    data += 3;
+    length -= 3;
+  }
+  if (lenient) {
+    _data = Utf8::lenientDecode(std::string_view(data, length));
+  } else {
+    auto maybe_utf32 = Utf8::strictDecode(std::string_view(data, length));
+    if (!maybe_utf32.has_value()) {
+      throw IllegalArgumentException("UTF-8 string contains an illegal byte sequence");
+    }
+    _data = std::move(maybe_utf32).value();
+  }
   p = 0;
 }
 
-void ANTLRInputStream::load(std::istream &stream) {
+void ANTLRInputStream::load(std::istream &stream, bool lenient) {
   if (!stream.good() || stream.eof()) // No fail, bad or EOF.
     return;
 
   _data.clear();
 
   std::string s((std::istreambuf_iterator<char>(stream)), std::istreambuf_iterator<char>());
-  load(s.data(), s.length());
+  load(s.data(), s.length(), lenient);
 }
 
 void ANTLRInputStream::reset() {
@@ -150,7 +153,11 @@ std::string ANTLRInputStream::getText(const Interval &interval) {
     return "";
   }
 
-  return antlrcpp::utf32_to_utf8(_data.substr(start, count));
+  auto maybeUtf8 = Utf8::strictEncode(std::u32string_view(_data).substr(start, count));
+  if (!maybeUtf8.has_value()) {
+    throw IllegalArgumentException("Input stream contains invalid Unicode code points");
+  }
+  return std::move(maybeUtf8).value();
 }
 
 std::string ANTLRInputStream::getSourceName() const {
@@ -161,7 +168,11 @@ std::string ANTLRInputStream::getSourceName() const {
 }
 
 std::string ANTLRInputStream::toString() const {
-  return antlrcpp::utf32_to_utf8(_data);
+  auto maybeUtf8 = Utf8::strictEncode(_data);
+  if (!maybeUtf8.has_value()) {
+    throw IllegalArgumentException("Input stream contains invalid Unicode code points");
+  }
+  return std::move(maybeUtf8).value();
 }
 
 void ANTLRInputStream::InitializeInstanceFields() {

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/ANTLRInputStream.h

@@ -5,6 +5,8 @@
 
 #pragma once
 
+#include <string_view>
+
 #include "CharStream.h"
 
 namespace antlr4 {
@@ -16,7 +18,7 @@ namespace antlr4 {
   protected:
     /// The data being scanned.
     // UTF-32
-    UTF32String _data;
+    std::u32string _data;
 
     /// 0..n-1 index into string of next char </summary>
     size_t p;
@@ -26,18 +28,19 @@ namespace antlr4 {
     std::string name;
 
     ANTLRInputStream();
-    
-#if __cplusplus >= 201703L
-    ANTLRInputStream(const std::string_view &input);
-#endif
-    
-    ANTLRInputStream(const std::string &input);
+
+    ANTLRInputStream(std::string_view input);
+
     ANTLRInputStream(const char *data, size_t length);
     ANTLRInputStream(std::istream &stream);
 
-    virtual void load(const std::string &input);
-    virtual void load(const char *data, size_t length);
-    virtual void load(std::istream &stream);
+    virtual void load(const std::string &input, bool lenient);
+    virtual void load(const char *data, size_t length, bool lenient);
+    virtual void load(std::istream &stream, bool lenient);
+
+    virtual void load(const std::string &input) { load(input, false); }
+    virtual void load(const char *data, size_t length) { load(data, length, false); }
+    virtual void load(std::istream &stream) { load(stream, false); }
 
     /// Reset the stream so that it's in the same state it was
     /// when the object was created *except* the data array is not

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/CommonToken.cpp

@@ -10,8 +10,8 @@
 
 #include "misc/Interval.h"
 
-#include "support/StringUtils.h"
 #include "support/CPPUtils.h"
+#include "support/StringUtils.h"
 
 #include "CommonToken.h"
 
@@ -165,9 +165,7 @@ std::string CommonToken::toString(Recognizer *r) const {
   }
   std::string txt = getText();
   if (!txt.empty()) {
-    antlrcpp::replaceAll(txt, "\n", "\\n");
-    antlrcpp::replaceAll(txt, "\r", "\\r");
-    antlrcpp::replaceAll(txt, "\t", "\\t");
+    txt = antlrcpp::escapeWhitespace(txt);
   } else {
     txt = "<no text>";
   }

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/DefaultErrorStrategy.cpp

@@ -12,10 +12,11 @@
 #include "atn/RuleTransition.h"
 #include "atn/ATN.h"
 #include "atn/ATNState.h"
+#include "support/StringUtils.h"
+#include "support/Casts.h"
 #include "Parser.h"
 #include "CommonToken.h"
 #include "Vocabulary.h"
-#include "support/StringUtils.h"
 
 #include "DefaultErrorStrategy.h"
 
@@ -106,19 +107,19 @@ void DefaultErrorStrategy::sync(Parser *recognizer) {
   }
 
   switch (s->getStateType()) {
-    case atn::ATNState::BLOCK_START:
-    case atn::ATNState::STAR_BLOCK_START:
-    case atn::ATNState::PLUS_BLOCK_START:
-    case atn::ATNState::STAR_LOOP_ENTRY:
+    case atn::ATNStateType::BLOCK_START:
+    case atn::ATNStateType::STAR_BLOCK_START:
+    case atn::ATNStateType::PLUS_BLOCK_START:
+    case atn::ATNStateType::STAR_LOOP_ENTRY:
       // report error and recover if possible
       if (singleTokenDeletion(recognizer) != nullptr) {
         return;
       }
 
       throw InputMismatchException(recognizer);
 
-    case atn::ATNState::PLUS_LOOP_BACK:
-    case atn::ATNState::STAR_LOOP_BACK: {
+    case atn::ATNStateType::PLUS_LOOP_BACK:
+    case atn::ATNStateType::STAR_LOOP_BACK: {
       reportUnwantedToken(recognizer);
       misc::IntervalSet expecting = recognizer->getExpectedTokens();
       misc::IntervalSet whatFollowsLoopIterationOrRule = expecting.Or(getErrorRecoverySet(recognizer));
@@ -292,11 +293,13 @@ size_t DefaultErrorStrategy::getSymbolType(Token *symbol) {
 }
 
 std::string DefaultErrorStrategy::escapeWSAndQuote(const std::string &s) const {
-  std::string result = s;
-  antlrcpp::replaceAll(result, "\n", "\\n");
-  antlrcpp::replaceAll(result, "\r","\\r");
-  antlrcpp::replaceAll(result, "\t","\\t");
-  return "'" + result + "'";
+  std::string result;
+  result.reserve(s.size() + 2);
+  result.push_back('\'');
+  antlrcpp::escapeWhitespace(result, s);
+  result.push_back('\'');
+  result.shrink_to_fit();
+  return result;
 }
 
 misc::IntervalSet DefaultErrorStrategy::getErrorRecoverySet(Parser *recognizer) {
@@ -306,7 +309,7 @@ misc::IntervalSet DefaultErrorStrategy::getErrorRecoverySet(Parser *recognizer)
   while (ctx->invokingState != ATNState::INVALID_STATE_NUMBER) {
     // compute what follows who invoked us
     atn::ATNState *invokingState = atn.states[ctx->invokingState];
-    atn::RuleTransition *rt = dynamic_cast<atn::RuleTransition*>(invokingState->transitions[0]);
+    const atn::RuleTransition *rt = downCast<const atn::RuleTransition*>(invokingState->transitions[0].get());
     misc::IntervalSet follow = atn.nextTokens(rt->followState);
     recoverSet.addAll(follow);
 

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/Exceptions.cpp

@@ -10,7 +10,7 @@ using namespace antlr4;
 RuntimeException::RuntimeException(const std::string &msg) : std::exception(), _message(msg) {
 }
 
-const char* RuntimeException::what() const NOEXCEPT {
+const char* RuntimeException::what() const noexcept {
   return _message.c_str();
 }
 
@@ -19,7 +19,7 @@ const char* RuntimeException::what() const NOEXCEPT {
 IOException::IOException(const std::string &msg) : std::exception(), _message(msg) {
 }
 
-const char* IOException::what() const NOEXCEPT {
+const char* IOException::what() const noexcept {
   return _message.c_str();
 }
 

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/Exceptions.h

@@ -16,7 +16,7 @@ namespace antlr4 {
   public:
     RuntimeException(const std::string &msg = "");
 
-    virtual const char* what() const NOEXCEPT override;
+    virtual const char* what() const noexcept override;
   };
 
   class ANTLR4CPP_PUBLIC IllegalStateException : public RuntimeException {
@@ -77,7 +77,7 @@ namespace antlr4 {
   public:
     IOException(const std::string &msg = "");
 
-    virtual const char* what() const NOEXCEPT override;
+    virtual const char* what() const noexcept override;
   };
 
   class ANTLR4CPP_PUBLIC CancellationException : public IllegalStateException {

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/FailedPredicateException.cpp

@@ -8,6 +8,7 @@
 #include "atn/PredicateTransition.h"
 #include "atn/ATN.h"
 #include "atn/ATNState.h"
+#include "support/Casts.h"
 #include "support/CPPUtils.h"
 
 #include "FailedPredicateException.h"
@@ -26,10 +27,10 @@ FailedPredicateException::FailedPredicateException(Parser *recognizer, const std
                          recognizer->getInputStream(), recognizer->getContext(), recognizer->getCurrentToken()) {
 
   atn::ATNState *s = recognizer->getInterpreter<atn::ATNSimulator>()->atn.states[recognizer->getState()];
-  atn::Transition *transition = s->transitions[0];
-  if (is<atn::PredicateTransition*>(transition)) {
-    _ruleIndex = static_cast<atn::PredicateTransition *>(transition)->ruleIndex;
-    _predicateIndex = static_cast<atn::PredicateTransition *>(transition)->predIndex;
+  const atn::Transition *transition = s->transitions[0].get();
+  if (transition->getTransitionType() == atn::TransitionType::PREDICATE) {
+    _ruleIndex = downCast<const atn::PredicateTransition&>(*transition).getRuleIndex();
+    _predicateIndex = downCast<const atn::PredicateTransition&>(*transition).getPredIndex();
   } else {
     _ruleIndex = 0;
     _predicateIndex = 0;

src/pytsql/grammar/cpp_src/antlr4-cpp-runtime/FailedPredicateException.h

@@ -15,7 +15,7 @@ namespace antlr4 {
   /// prediction.
   class ANTLR4CPP_PUBLIC FailedPredicateException : public RecognitionException {
   public:
-    FailedPredicateException(Parser *recognizer);
+    explicit FailedPredicateException(Parser *recognizer);
     FailedPredicateException(Parser *recognizer, const std::string &predicate);
     FailedPredicateException(Parser *recognizer, const std::string &predicate, const std::string &message);