feat: literal expressions for the textplan parser (#61)

* Adds support for literals and column names on the parser side
      * Adds the datetime submodule for parsing dates 
   * Initial output to binary plans
      * Adds the protobuf-matchers submodule for testing

Author: EpsilonPrime

.github/workflows/build_test.yml

@@ -38,7 +38,7 @@ jobs:
         run: ./scripts/setup-ubuntu.sh
       - run: mkdir build
       - name: Run cmake
-        run: cmake -Bbuild -GNinja -DCMAKE_BUILD_TYPE=Debug
+        run: cmake -Bbuild -GNinja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TZ_LIB=ON
       - name: Build
         run: ninja -C build
       - name: Test

.gitmodules

@@ -13,3 +13,9 @@
 [submodule "third_party/abseil-cpp"]
 	path = third_party/abseil-cpp
 	url = https://github.com/abseil/abseil-cpp.git
+[submodule "third_party/datetime"]
+	path = third_party/datetime
+	url = https://github.com/HowardHinnant/date.git
+[submodule "third_party/protobuf-matchers"]
+	path = third_party/protobuf-matchers
+	url = https://github.com/EpsilonPrime/protobuf-matchers.git

.licenserc.yaml

@@ -14,8 +14,10 @@ header:
     - '.licenserc.yaml'
     - 'third_party/abseil-cpp'
     - 'third_party/antlr4'
+    - 'third_party/datetime'
     - 'third_party/fmt'
     - 'third_party/googletest'
+    - 'third_party/protobuf-matchers'
     - 'third_party/substrait'
     - 'third_party/yaml-cpp'
     - '**/*.md'

include/substrait/expression/DecimalLiteral.h

@@ -18,6 +18,9 @@ class DecimalLiteral {
   static DecimalLiteral fromProto(
       const ::substrait::proto::Expression_Literal_Decimal& proto);
 
+  static DecimalLiteral
+  fromString(const std::string& str, int32_t precision, int32_t scale);
+
   // Validates that the constructed decimal has an exactly 16 byte value with
   // a stated precision between 1 and 38.
   bool isValid();
@@ -29,6 +32,9 @@ class DecimalLiteral {
   // a string.
   std::string toString();
 
+  // Emits a proto version of the corresponding decimal.
+  ::substrait::proto::Expression_Literal_Decimal toProto();
+
   [[nodiscard]] int32_t precision() const {
     return precision_;
   }

scripts/setup-ubuntu.sh

@@ -23,6 +23,7 @@ sudo --preserve-env apt install -y \
   protobuf-compiler \
   clang-format \
   uuid-dev \
-  default-jre
+  default-jre \
+  libcurl4-openssl-dev
 
 pip install cmake-format

src/substrait/common/NumberUtils.h

@@ -24,4 +24,4 @@ class NumberUtils {
   }
 };
 
-} // namespace io::substrait::common
+} // namespace io::substrait::common

src/substrait/expression/CMakeLists.txt

@@ -2,7 +2,8 @@
 
 add_library(substrait_expression DecimalLiteral.cpp)
 
-target_link_libraries(substrait_expression substrait_proto absl::numeric)
+target_link_libraries(substrait_expression substrait_proto absl::numeric
+                      absl::strings)
 
 if(${SUBSTRAIT_CPP_BUILD_TESTING})
   add_subdirectory(tests)

src/substrait/expression/DecimalLiteral.cpp

@@ -5,6 +5,7 @@
 #include <sstream>
 
 #include "absl/numeric/int128.h"
+#include "absl/strings/numbers.h"
 #include "substrait/proto/algebra.pb.h"
 
 namespace io::substrait::expression {
@@ -14,7 +15,7 @@ namespace {
 // negate flips the sign of a two-complements value.
 std::string negate(const std::string& value) {
   std::string newValue = value;
-  // Flip all of the bits and add one.
+  // Flip all the bits and add one.
   bool carryover = true;
   for (char& b : newValue) {
     uint8_t newB = ~(static_cast<uint8_t>(b));
@@ -27,13 +28,37 @@ std::string negate(const std::string& value) {
   return newValue;
 }
 
+void uint128ToBytes(const absl::uint128& value, std::uint8_t* bytes) {
+  // Copy the low 64 bits of the uint128 value into the first 8 bytes of the
+  // output buffer.
+  std::memcpy(bytes, &value, 8);
+
+  // Copy the high 64 bits of the uint128 value into the next 8 bytes of the
+  // output buffer.
+  std::memcpy(bytes + 8, reinterpret_cast<const std::uint8_t*>(&value) + 8, 8);
+}
+
 } // namespace
 
 DecimalLiteral DecimalLiteral::fromProto(
     const ::substrait::proto::Expression_Literal_Decimal& proto) {
   return {proto.value(), proto.precision(), proto.scale()};
 }
 
+DecimalLiteral DecimalLiteral::fromString(
+    const std::string& str,
+    int32_t precision,
+    int32_t scale) {
+  absl::uint128 v;
+  if (!absl::SimpleAtoi(str, &v)) {
+    // TODO -- Store the parse errors so that they can be examined later.
+    return {"", 0, 0};
+  }
+  std::uint8_t valueBytes[16];
+  uint128ToBytes(v, valueBytes);
+  return {std::string((const char*)valueBytes, 16), precision, scale};
+}
+
 bool DecimalLiteral::isValid() {
   return value_.size() == 16 && precision_ >= 1 && precision_ <= 38;
 }
@@ -92,4 +117,12 @@ std::string DecimalLiteral::toString() {
   return decimalString.str();
 }
 
+::substrait::proto::Expression_Literal_Decimal DecimalLiteral::toProto() {
+  ::substrait::proto::Expression_Literal_Decimal result;
+  result.set_value(value_);
+  result.set_precision(precision_);
+  result.set_scale(scale_);
+  return result;
+};
+
 } // namespace io::substrait::expression

src/substrait/textplan/Location.cpp

@@ -2,6 +2,8 @@
 
 #include "substrait/textplan/Location.h"
 
+#include <functional>
+
 namespace io::substrait::textplan {
 
 constexpr Location Location::kUnknownLocation(

src/substrait/textplan/SymbolTable.h

@@ -30,6 +30,7 @@ enum class SymbolType {
 };
 
 enum class RelationType {
+  // Logical plans
   kUnknown = 0,
   kRead = 1,
   kProject = 2,
@@ -40,13 +41,17 @@ enum class RelationType {
   kSort = 7,
   kFilter = 8,
   kSet = 9,
-  kExchange = 10,
-  kDdl = 11,
-  kWrite = 12,
-  kHashJoin = 13,
-  kMergeJoin = 14,
-  kReference = 15,
 
+  // Physical plans
+  kHashJoin = 31,
+  kMergeJoin = 32,
+
+  // Write relations, currently unreachable in Plan protos.
+  kExchange = 50,
+  kDdl = 51,
+  kWrite = 52,
+
+  // Extensions
   kExtensionLeaf = 100,
   kExtensionSingle = 101,
   kExtensionMulti = 102,

src/substrait/textplan/SymbolTablePrinter.cpp

@@ -154,9 +154,6 @@ std::string typeToText(const ::substrait::proto::Type& type) {
 std::string relationToText(
     const SymbolTable& symbolTable,
     const SymbolInfo& info) {
-  if (info.blob.type() != typeid(std::shared_ptr<RelationData>)) {
-    return "not-yet-implemented";
-  }
   auto relationData = ANY_CAST(std::shared_ptr<RelationData>, info.blob);
   if (relationData->relation.rel_type_case() ==
       ::substrait::proto::Rel::REL_TYPE_NOT_SET) {
@@ -193,9 +190,6 @@ std::string outputPipelinesSection(const SymbolTable& symbolTable) {
         info.type != SymbolType::kRelation) {
       continue;
     }
-    if (info.blob.type() != typeid(std::shared_ptr<RelationData>)) {
-      return "not-yet-implemented";
-    }
     auto relationData = ANY_CAST(std::shared_ptr<RelationData>, info.blob);
     for (auto pipelineStart : relationData->newPipelines) {
       auto pipeline = pipelineToPath(symbolTable, pipelineStart);
@@ -270,7 +264,7 @@ std::string outputSchemaSection(const SymbolTable& symbolTable) {
   return text.str();
 }
 
-std::string outputSourceSection(const SymbolTable& symbolTable) {
+std::string outputSourcesSection(const SymbolTable& symbolTable) {
   std::stringstream text;
   bool hasPreviousText = false;
   for (const SymbolInfo& info : symbolTable) {
@@ -419,7 +413,7 @@ std::string SymbolTablePrinter::outputToText(const SymbolTable& symbolTable) {
     hasPreviousText = true;
   }
 
-  newText = outputSourceSection(symbolTable);
+  newText = outputSourcesSection(symbolTable);
   if (!newText.empty()) {
     if (hasPreviousText) {
       text << "\n";
@@ -438,4 +432,20 @@ std::string SymbolTablePrinter::outputToText(const SymbolTable& symbolTable) {
   return text.str();
 }
 
+::substrait::proto::Plan SymbolTablePrinter::outputToBinaryPlan(
+    const SymbolTable& symbolTable) {
+  ::substrait::proto::Plan plan;
+  for (const SymbolInfo& info : symbolTable) {
+    if (info.type != SymbolType::kRelation) {
+      continue;
+    }
+    auto relationData = ANY_CAST(std::shared_ptr<RelationData>, info.blob);
+    auto relation = plan.add_relations();
+    // TODO -- Figure out when to use rel_root and when to use rel.
+    *relation->mutable_rel() = relationData->relation;
+  }
+
+  return plan;
+}
+
 } // namespace io::substrait::textplan

src/substrait/textplan/SymbolTablePrinter.h

@@ -6,11 +6,18 @@
 
 #include "SymbolTable.h"
 
+namespace substrait::proto {
+class Plan;
+}
+
 namespace io::substrait::textplan {
 
 class SymbolTablePrinter {
  public:
   static std::string outputToText(const SymbolTable& symbolTable);
+
+  static ::substrait::proto::Plan outputToBinaryPlan(
+      const SymbolTable& symbolTable);
 };
 
 } // namespace io::substrait::textplan

src/substrait/textplan/converter/BasePlanProtoVisitor.cpp

@@ -649,6 +649,7 @@ std::any BasePlanProtoVisitor::visitExpression(
     case ::substrait::proto::Expression::RexTypeCase::REX_TYPE_NOT_SET:
       break;
   }
+  // TODO -- Use an error listener instead.
   SUBSTRAIT_UNSUPPORTED(
       "Unsupported expression type encountered: " +
       std::to_string(expression.rex_type_case()));

src/substrait/textplan/converter/InitialPlanProtoVisitor.cpp

@@ -119,7 +119,7 @@ std::any InitialPlanProtoVisitor::visitExtensionUri(
 std::any InitialPlanProtoVisitor::visitPlanRelation(
     const ::substrait::proto::PlanRel& relation) {
   BasePlanProtoVisitor::visitPlanRelation(relation);
-  std::string name =
+  const std::string name =
       ::substrait::proto::planRelTypeCaseName(relation.rel_type_case());
   auto uniqueName = symbolTable_->getUniqueName(name);
   auto relationData = std::make_shared<RelationData>();
@@ -134,7 +134,7 @@ std::any InitialPlanProtoVisitor::visitPlanRelation(
 
 std::any InitialPlanProtoVisitor::visitRelation(
     const ::substrait::proto::Rel& relation) {
-  std::string name =
+  const std::string name =
       ::substrait::proto::relTypeCaseName(relation.rel_type_case());
 
   auto previousRelationScope = currentRelationScope_;
@@ -303,7 +303,7 @@ void InitialPlanProtoVisitor::updateLocalSchema(
                 std::nullopt,
                 std::nullopt);
           }
-          relationData->fieldReferences.push_back(symbol);
+          relationData->fieldReferences.emplace_back(symbol);
         }
       }
       break;

src/substrait/textplan/converter/InitialPlanProtoVisitor.h

@@ -82,7 +82,7 @@ class InitialPlanProtoVisitor : public BasePlanProtoVisitor {
   std::shared_ptr<SymbolTable> symbolTable_;
   std::shared_ptr<SubstraitErrorListener> errorListener_;
 
-  const ::substrait::proto::Rel* currentRelationScope_;
+  const ::substrait::proto::Rel* currentRelationScope_{nullptr};
   std::map<const ::substrait::proto::Rel*, const SymbolInfo*>
       readRelationSources_;
   std::map<const ::substrait::proto::Rel*, const SymbolInfo*>

src/substrait/textplan/converter/PipelineVisitor.h

@@ -10,8 +10,8 @@ namespace io::substrait::textplan {
 
 class PipelineVisitor : public BasePlanProtoVisitor {
  public:
-  explicit PipelineVisitor(const SymbolTable& symbol_table) {
-    symbolTable_ = std::make_shared<SymbolTable>(symbol_table);
+  explicit PipelineVisitor(const SymbolTable& symbolTable) {
+    symbolTable_ = std::make_shared<SymbolTable>(symbolTable);
   }
 
   [[nodiscard]] std::shared_ptr<const SymbolTable> getSymbolTable() const {

src/substrait/textplan/converter/PlanPrinterVisitor.cpp

@@ -88,17 +88,8 @@ std::string PlanPrinterVisitor::lookupFunctionReference(
     if (symbol->type != SymbolType::kFunction) {
       continue;
     }
-    if (symbol->blob.type() !=
-        typeid(const ::substrait::proto::extensions::
-                   SimpleExtensionDeclaration_ExtensionFunction*)) {
-      // TODO -- Implement function references for text plans.
-      continue;
-    }
-    auto function = ANY_CAST(
-        const ::substrait::proto::extensions::
-            SimpleExtensionDeclaration_ExtensionFunction*,
-        symbol->blob);
-    if (function->function_anchor() == function_reference) {
+    auto function = ANY_CAST(std::shared_ptr<FunctionData>, symbol->blob);
+    if (function->anchor == function_reference) {
       return symbol->name;
     }
   }
@@ -603,6 +594,7 @@ std::any PlanPrinterVisitor::visitExpression(
     const ::substrait::proto::Expression& expression) {
   if (expression.rex_type_case() ==
       ::substrait::proto::Expression::RexTypeCase::REX_TYPE_NOT_SET) {
+    // TODO -- Remove this check after expressions are finished.
     return std::string("EXPR-NOT-YET-IMPLEMENTED");
   }
   return BasePlanProtoVisitor::visitExpression(expression);

src/substrait/textplan/parser/CMakeLists.txt

@@ -8,16 +8,26 @@ add_library(
   SubstraitPlanVisitor.h
   SubstraitPlanPipelineVisitor.cpp
   SubstraitPlanPipelineVisitor.h
+  SubstraitPlanRelationVisitor.cpp
+  SubstraitPlanRelationVisitor.h
   ParseText.cpp
   ParseText.h
   SubstraitParserErrorListener.cpp)
 
-target_link_libraries(substrait_textplan_loader symbol_table error_listener
-                      substrait_proto textplan_grammar fmt::fmt-header-only)
+target_link_libraries(
+  substrait_textplan_loader
+  symbol_table
+  error_listener
+  substrait_proto
+  substrait_type
+  substrait_expression
+  textplan_grammar
+  fmt::fmt-header-only
+  date::date
+  date::date-tz)
 
 add_executable(planparser Tool.cpp)
 
-include_directories(${textplan_grammar_INCLUDE_DIRS})
 target_link_libraries(planparser substrait_textplan_loader error_listener)
 
 if(${SUBSTRAIT_CPP_BUILD_TESTING})