Resolves #1026: index matching mechanics (#1027)

* Resolves #987: introduce concept of correlations (using quantifiers)

* matching infrastructure

* address review comments

Author: normen662

.idea/runConfigurations/ReplRunner.xml

@@ -24,7 +24,7 @@ In this realase, the various implementations of the `RecordQueryPlan` interface
 * **Performance** Improvement 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
 * **Performance** Improvement 4 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
 * **Performance** Improvement 5 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
-* **Feature** Feature 1 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
+* **Feature** General index matching framework and mechanics needed for the new planner implementation. [(Issue #1026)](https://github.com/FoundationDB/fdb-record-layer/issues/1026)
 * **Feature** Feature 2 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
 * **Feature** Feature 3 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)
 * **Feature** Feature 4 [(Issue #NNN)](https://github.com/FoundationDB/fdb-record-layer/issues/NNN)

docs/ReleaseNotes.md

@@ -27,6 +27,10 @@ if (!hasProperty('coreNotStrict')) {
     apply from: rootProject.file('gradle/strict.gradle')
 }
 
+configurations {
+    repl.extendsFrom testRuntime
+}
+
 dependencies {
     compile project(':fdb-extensions')
 
@@ -48,10 +52,13 @@ dependencies {
     testRuntime "org.apache.logging.log4j:log4j-slf4j-impl:${log4jVersion}" // binding
     testCompile "org.apache.logging.log4j:log4j-core:${log4jVersion}" // library
 
-    testImplementation "org.junit.jupiter:junit-jupiter-api:${junitVersion}"
+    testCompile "org.junit.jupiter:junit-jupiter-api:${junitVersion}"
     testCompile "org.junit.jupiter:junit-jupiter-params:${junitVersion}"
     testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${junitVersion}"
     testAnnotationProcessor "com.google.auto.service:auto-service:${autoServiceVersion}"
+    testCompile "org.jline:jline:${jlineVersion}"
+    testCompile "org.junit.platform:junit-platform-launcher:${junitPlatformVersion}"
+    repl "org.junit.jupiter:junit-jupiter-engine:${junitVersion}"
 }
 
 sourceSets {
@@ -118,6 +125,18 @@ task testShadowJar(type: com.github.jengelman.gradle.plugins.shadow.tasks.Shadow
     exclude 'log4j.properties'
 }
 
+task replJar(type: com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
+    classifier = 'repl'
+    from sourceSets.main.output
+    from sourceSets.test.output
+    configurations = [ project.configurations.repl ]
+    manifest {
+        inheritFrom project.tasks.jar.manifest
+        attributes 'Main-Class': 'com.apple.foundationdb.record.query.plan.debug.ReplRunner'
+    }
+    mergeServiceFiles()
+}
+
 publishing {
     publications {
         library(MavenPublication) {

fdb-record-layer-core/fdb-record-layer-core.gradle

@@ -172,6 +172,7 @@ public RecordQueryCoveringIndexPlan rebase(@Nonnull final AliasMap translationMa
         return new RecordQueryCoveringIndexPlan(indexPlan, recordTypeName, availableFields, toRecord);
     }
 
+    @Nonnull
     @Override
     public boolean equalsWithoutChildren(@Nonnull RelationalExpression otherExpression,
                                          @Nonnull final AliasMap equivalencesMap) {

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryCoveringIndexPlan.java

@@ -156,7 +156,8 @@ public PlannerGraph rewritePlannerGraph(@Nonnull List<? extends PlannerGraph> ch
                         NodeInfo.NESTED_LOOP_JOIN_OPERATOR);
         final PlannerGraph graphForInner = Iterables.getOnlyElement(childGraphs);
         final PlannerGraph.NodeWithInfo explodeNode =
-                new PlannerGraph.LogicalOperatorNodeWithInfo(NodeInfo.TABLE_FUNCTION_OPERATOR,
+                new PlannerGraph.LogicalOperatorNodeWithInfo(this,
+                        NodeInfo.TABLE_FUNCTION_OPERATOR,
                         ImmutableList.of("EXPLODE({{externalBinding}})"),
                         ImmutableMap.of("externalBinding", Attribute.gml(externalBinding)));
         final PlannerGraph.Edge fromExplodeEdge = new PlannerGraph.Edge();

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryInParameterJoinPlan.java

@@ -30,23 +30,19 @@
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase;
 import com.apple.foundationdb.record.query.plan.AvailableFields;
 import com.apple.foundationdb.record.query.plan.temp.AliasMap;
-import com.apple.foundationdb.record.query.plan.temp.CorrelationIdentifier;
 import com.apple.foundationdb.record.query.plan.temp.GroupExpressionRef;
 import com.apple.foundationdb.record.query.plan.temp.Quantifier;
 import com.apple.foundationdb.record.query.plan.temp.Quantifiers;
 import com.apple.foundationdb.record.query.plan.temp.RelationalExpression;
 import com.apple.foundationdb.record.query.plan.temp.explain.PlannerGraphRewritable;
 import com.apple.foundationdb.record.query.plan.visitor.RecordQueryPlannerSubstitutionVisitor;
 import com.google.common.base.Verify;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Sets;
 import com.google.protobuf.Message;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.util.List;
 import java.util.Objects;
-import java.util.Set;
 
 /**
  * An executable query plan for producing records.
@@ -181,7 +177,7 @@ default int structuralHashCode() {
      */
     @API(API.Status.EXPERIMENTAL)
     default boolean structuralEquals(@Nullable final Object other) {
-        return structuralEquals(other, AliasMap.empty());
+        return structuralEquals(other, AliasMap.emptyMap());
     }
 
     /**
@@ -222,37 +218,17 @@ default boolean structuralEquals(@Nullable final Object other,
             return false;
         }
 
-        final Set<CorrelationIdentifier> correlatedTo = getCorrelatedTo();
-        final Set<CorrelationIdentifier> otherCorrelatedTo = otherExpression.getCorrelatedTo();
-
-        Sets.SetView<CorrelationIdentifier> unboundCorrelatedTo = Sets.difference(correlatedTo, equivalenceMap.sources());
-        Sets.SetView<CorrelationIdentifier> unboundOtherCorrelatedTo = Sets.difference(otherCorrelatedTo, equivalenceMap.targets());
-
-        final Sets.SetView<CorrelationIdentifier> commonUnbound = Sets.intersection(unboundCorrelatedTo, unboundOtherCorrelatedTo);
-        final AliasMap identitiesMap = AliasMap.identitiesFor(commonUnbound);
-        unboundCorrelatedTo = Sets.difference(correlatedTo, commonUnbound);
-        unboundOtherCorrelatedTo = Sets.difference(otherCorrelatedTo, commonUnbound);
-
         final Iterable<AliasMap> boundCorrelatedReferencesIterable =
-                AliasMap.empty()
-                        .match(unboundCorrelatedTo,
-                                alias -> ImmutableSet.of(),
-                                unboundOtherCorrelatedTo,
-                                otherAlias -> ImmutableSet.of(),
-                                false,
-                                (alias, otherAlias, nestedEquivalencesMap) -> true);
+                enumerateUnboundCorrelatedTo(equivalenceMap, otherExpression);
 
         for (final AliasMap boundCorrelatedReferencesMap : boundCorrelatedReferencesIterable) {
-            final AliasMap.Builder boundEquivalenceMapBuilder = equivalenceMap.derived();
-
-            boundEquivalenceMapBuilder.putAll(identitiesMap);
-            boundEquivalenceMapBuilder.putAll(boundCorrelatedReferencesMap);
+            final AliasMap.Builder boundCorrelatedToBuilder = boundCorrelatedReferencesMap.derived();
 
-            AliasMap boundEquivalenceMap = AliasMap.empty();
+            AliasMap boundCorrelatedToMap = AliasMap.emptyMap();
 
             int i;
             for (i = 0; i < quantifiers.size(); i++) {
-                boundEquivalenceMap = boundEquivalenceMapBuilder.build();
+                boundCorrelatedToMap = boundCorrelatedToBuilder.build();
 
                 final Quantifier.Physical quantifier = quantifiers.get(i);
                 final Quantifier.Physical otherQuantifier = otherQuantifiers.get(i);
@@ -266,11 +242,11 @@ default boolean structuralEquals(@Nullable final Object other,
                 }
 
                 if (canCorrelate()) {
-                    boundEquivalenceMapBuilder.put(quantifier.getAlias(), otherQuantifier.getAlias());
+                    boundCorrelatedToBuilder.put(quantifier.getAlias(), otherQuantifier.getAlias());
                 }
             }
 
-            if (i == quantifiers.size() && (equalsWithoutChildren(otherExpression, boundEquivalenceMap))) {
+            if (i == quantifiers.size() && (equalsWithoutChildren(otherExpression, boundCorrelatedToMap))) {
                 return true;
             }
         }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryPlan.java

@@ -144,6 +144,7 @@ public boolean equalsWithoutChildren(@Nonnull RelationalExpression otherExpressi
                filter.semanticEquals(otherPlan.getPredicate(), equivalencesMap);
     }
 
+    @Nonnull
     @Override
     public String toString() {
         return getInnerPlan() + " | " + getPredicate();

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryPredicateFilterPlan.java

@@ -152,7 +152,6 @@ public boolean equalsWithoutChildren(@Nonnull RelationalExpression otherExpressi
         if (getClass() != otherExpression.getClass()) {
             return false;
         }
-
         return comparisonKey.equals(((RecordQueryUnorderedDistinctPlan)otherExpression).comparisonKey);
     }