Skip to content

Commit f614809

Browse files
pengpeng-lupengpeng-lu
committed
Merge branch 'main' into udf
2 parents d7bc203 + 8a3c17f commit f614809

File tree

61 files changed

+1713
-1003
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+1713
-1003
lines changed

‎.github/workflows/pr_labels.yml

+3-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ name: Pull Request Labels
22

33
on:
44
pull_request:
5-
types: [opened, labeled, unlabeled]
5+
# Run whenever the labels change or the PR HEAD changes. In theory, the latter is a bit redundant,
6+
# but it's necessary to make sure that the check is applied to the final commit for a PR
7+
types: [opened, reopened, labeled, unlabeled, synchronize]
68

79
jobs:
810
labels:

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -1679,7 +1679,7 @@ public int hashCode() {
16791679
}
16801680

16811681
public int computeHashCode() {
1682-
return Objects.hash(type, relatedByEquality());
1682+
return Objects.hash(type.name(), relatedByEquality());
16831683
}
16841684

16851685
private Set<String> relatedByEquality() {
@@ -2091,7 +2091,7 @@ public boolean equals(Object o) {
20912091

20922092
@Override
20932093
public int hashCode() {
2094-
return Objects.hash(type);
2094+
return Objects.hash(type.name());
20952095
}
20962096

20972097
@Override

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesRuleCall.java

+29-20
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,15 @@
3232
import com.google.common.base.Preconditions;
3333
import com.google.common.base.Verify;
3434
import com.google.common.collect.ImmutableList;
35+
import com.google.common.collect.Iterables;
3536
import com.google.common.collect.Sets;
3637

3738
import javax.annotation.Nonnull;
3839
import java.util.Arrays;
3940
import java.util.Collection;
4041
import java.util.Collections;
4142
import java.util.Deque;
43+
import java.util.Objects;
4244
import java.util.Optional;
4345
import java.util.Set;
4446
import java.util.function.Function;
@@ -241,8 +243,7 @@ public Reference memoizeExpression(@Nonnull final RelationalExpression expressio
241243
if (expression.getQuantifiers().isEmpty()) {
242244
return memoizeLeafExpression(expression);
243245
}
244-
245-
Debugger.withDebugger(debugger -> debugger.onEvent(new Debugger.InsertIntoMemoEvent(Debugger.Location.BEGIN)));
246+
Debugger.withDebugger(debugger -> debugger.onEvent(Debugger.InsertIntoMemoEvent.begin()));
246247
try {
247248
Preconditions.checkArgument(!(expression instanceof RecordQueryPlan));
248249

@@ -278,28 +279,33 @@ public Reference memoizeExpression(@Nonnull final RelationalExpression expressio
278279
commonReferencingExpressions.retainAll(referencingExpressionsIterator.next());
279280
}
280281

281-
for (final var commonReferencingExpression : commonReferencingExpressions) {
282-
if (Reference.isMemoizedExpression(commonReferencingExpression, expression)) {
283-
Debugger.withDebugger(debugger -> debugger.onEvent(new Debugger.InsertIntoMemoEvent(Debugger.Location.REUSED)));
284-
final var reference = expressionToReferenceMap.get(commonReferencingExpression);
285-
Verify.verifyNotNull(reference);
286-
Verify.verify(reference != this.root);
287-
return reference;
288-
}
282+
commonReferencingExpressions.removeIf(commonReferencingExpression -> !Reference.isMemoizedExpression(expression, commonReferencingExpression));
283+
284+
if (!commonReferencingExpressions.isEmpty()) {
285+
Debugger.withDebugger(debugger ->
286+
debugger.onEvent(Debugger.InsertIntoMemoEvent.reusedExpWithReferences(expression,
287+
commonReferencingExpressions.stream()
288+
.map(expressionToReferenceMap::get)
289+
.collect(ImmutableList.toImmutableList()))));
290+
final var reference = expressionToReferenceMap.get(Objects.requireNonNull(Iterables.getFirst(commonReferencingExpressions, null)));
291+
Verify.verifyNotNull(reference);
292+
Verify.verify(reference != this.root);
293+
return reference;
289294
}
290-
Debugger.withDebugger(debugger -> debugger.onEvent(new Debugger.InsertIntoMemoEvent(Debugger.Location.NEW)));
295+
296+
Debugger.withDebugger(debugger -> debugger.onEvent(Debugger.InsertIntoMemoEvent.newExp(expression)));
291297
final var newRef = Reference.of(expression);
292298
traversal.addExpression(newRef, expression);
293299
return newRef;
294300
} finally {
295-
Debugger.withDebugger(debugger -> debugger.onEvent(new Debugger.InsertIntoMemoEvent(Debugger.Location.END)));
301+
Debugger.withDebugger(debugger -> debugger.onEvent(Debugger.InsertIntoMemoEvent.end()));
296302
}
297303
}
298304

299305
@Nonnull
300306
@Override
301307
public Reference memoizeLeafExpression(@Nonnull final RelationalExpression expression) {
302-
Debugger.withDebugger(debugger -> debugger.onEvent(new Debugger.InsertIntoMemoEvent(Debugger.Location.BEGIN)));
308+
Debugger.withDebugger(debugger -> debugger.onEvent(Debugger.InsertIntoMemoEvent.begin()));
303309
try {
304310
Preconditions.checkArgument(!(expression instanceof RecordQueryPlan));
305311
Preconditions.checkArgument(expression.getQuantifiers().isEmpty());
@@ -309,17 +315,17 @@ public Reference memoizeLeafExpression(@Nonnull final RelationalExpression expre
309315
for (final var leafRef : leafRefs) {
310316
for (final var member : leafRef.getMembers()) {
311317
if (Reference.isMemoizedExpression(expression, member)) {
312-
Debugger.withDebugger(debugger -> debugger.onEvent(new Debugger.InsertIntoMemoEvent(Debugger.Location.REUSED)));
318+
Debugger.withDebugger(debugger -> debugger.onEvent(Debugger.InsertIntoMemoEvent.reusedExp(expression)));
313319
return leafRef;
314320
}
315321
}
316322
}
317-
Debugger.withDebugger(debugger -> debugger.onEvent(new Debugger.InsertIntoMemoEvent(Debugger.Location.NEW)));
323+
Debugger.withDebugger(debugger -> debugger.onEvent(Debugger.InsertIntoMemoEvent.newExp(expression)));
318324
final var newRef = Reference.of(expression);
319325
traversal.addExpression(newRef, expression);
320326
return newRef;
321327
} finally {
322-
Debugger.withDebugger(debugger -> debugger.onEvent(new Debugger.InsertIntoMemoEvent(Debugger.Location.END)));
328+
Debugger.withDebugger(debugger -> debugger.onEvent(Debugger.InsertIntoMemoEvent.end()));
323329
}
324330
}
325331

@@ -351,7 +357,8 @@ public Reference memoizeReference(@Nonnull final Reference reference) {
351357
@Nonnull
352358
private Reference memoizeExpressionsExactly(@Nonnull final Collection<? extends RelationalExpression> expressions,
353359
@Nonnull Function<Set<? extends RelationalExpression>, Reference> referenceCreator) {
354-
Debugger.withDebugger(debugger -> debugger.onEvent(new Debugger.InsertIntoMemoEvent(Debugger.Location.BEGIN)));
360+
Debugger.withDebugger(debugger -> expressions.forEach(
361+
expression -> debugger.onEvent(Debugger.InsertIntoMemoEvent.begin())));
355362
try {
356363
final var expressionSet = new LinkedIdentitySet<>(expressions);
357364

@@ -367,19 +374,21 @@ private Reference memoizeExpressionsExactly(@Nonnull final Collection<? extends
367374
final Optional<Reference> memoizedRefMaybe = findExpressionsInMemo(expressionSet);
368375
if (memoizedRefMaybe.isPresent()) {
369376
Debugger.withDebugger(debugger ->
370-
expressionSet.forEach(plan -> debugger.onEvent(new Debugger.InsertIntoMemoEvent(Debugger.Location.REUSED))));
377+
expressionSet.forEach(
378+
plan -> debugger.onEvent(Debugger.InsertIntoMemoEvent.reusedExpWithReferences(plan, ImmutableList.of(memoizedRefMaybe.get())))));
371379
return memoizedRefMaybe.get();
372380
}
373381
}
374382

375383
final var newRef = referenceCreator.apply(expressionSet);
376384
for (final var plan : expressionSet) {
377-
Debugger.withDebugger(debugger -> debugger.onEvent(new Debugger.InsertIntoMemoEvent(Debugger.Location.NEW)));
385+
Debugger.withDebugger(debugger -> expressions.forEach(
386+
expression -> debugger.onEvent(Debugger.InsertIntoMemoEvent.newExp(expression))));
378387
traversal.addExpression(newRef, plan);
379388
}
380389
return newRef;
381390
} finally {
382-
Debugger.withDebugger(debugger -> debugger.onEvent(new Debugger.InsertIntoMemoEvent(Debugger.Location.END)));
391+
Debugger.withDebugger(debugger -> debugger.onEvent(Debugger.InsertIntoMemoEvent.end()));
383392
}
384393
}
385394

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/LinkedIdentityMap.java

+5
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,11 @@ public void replaceAll(final BiFunction<? super K, ? super V, ? extends V> funct
128128
map.replaceAll((wrappedK, v) -> function.apply(wrappedK.get(), v));
129129
}
130130

131+
@Override
132+
public V remove(final Object key) {
133+
return map.remove(identity.wrap(key));
134+
}
135+
131136
@Override
132137
public boolean equals(final Object o) {
133138
if (!(o instanceof LinkedIdentityMap)) {

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Reference.java

+9-4
Original file line numberDiff line numberDiff line change
@@ -185,18 +185,23 @@ public boolean insert(@Nonnull final RelationalExpression newValue) {
185185
* otherwise.
186186
*/
187187
private boolean insert(@Nonnull final RelationalExpression newValue, @Nullable final Map<PlanProperty<?>, ?> precomputedPropertiesMap) {
188-
Debugger.withDebugger(debugger -> debugger.onEvent(new Debugger.InsertIntoMemoEvent(Debugger.Location.BEGIN)));
188+
Debugger.withDebugger(debugger -> debugger.onEvent(Debugger.InsertIntoMemoEvent.begin()));
189189
try {
190190
final boolean containsInMemo = containsInMemo(newValue);
191-
Debugger.withDebugger(debugger -> debugger.onEvent(new Debugger.InsertIntoMemoEvent(containsInMemo ? Debugger.Location.REUSED : Debugger.Location.NEW)));
192-
191+
Debugger.withDebugger(debugger -> {
192+
if (containsInMemo) {
193+
debugger.onEvent(Debugger.InsertIntoMemoEvent.reusedExpWithReferences(newValue, ImmutableList.of(this)));
194+
} else {
195+
debugger.onEvent(Debugger.InsertIntoMemoEvent.newExp(newValue));
196+
}
197+
});
193198
if (!containsInMemo) {
194199
insertUnchecked(newValue, precomputedPropertiesMap);
195200
return true;
196201
}
197202
return false;
198203
} finally {
199-
Debugger.withDebugger(debugger -> debugger.onEvent(new Debugger.InsertIntoMemoEvent(Debugger.Location.END)));
204+
Debugger.withDebugger(debugger -> debugger.onEvent(Debugger.InsertIntoMemoEvent.end()));
200205
}
201206
}
202207

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Traversal.java

+10-5
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,14 @@
2525
import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression;
2626
import com.google.common.base.Verify;
2727
import com.google.common.collect.ImmutableSet;
28-
import com.google.common.collect.Maps;
2928
import com.google.common.collect.Multimaps;
3029
import com.google.common.collect.SetMultimap;
3130
import com.google.common.collect.Sets;
31+
import com.google.common.graph.ElementOrder;
3232
import com.google.common.graph.EndpointPair;
3333
import com.google.common.graph.MutableNetwork;
3434
import com.google.common.graph.NetworkBuilder;
35+
import com.google.common.graph.StableStandardMutableNetwork;
3536

3637
import javax.annotation.Nonnull;
3738
import java.util.Set;
@@ -172,15 +173,19 @@ public void removeExpression(@Nonnull final Reference reference, @Nonnull final
172173
* @return a new traversal object
173174
*/
174175
public static Traversal withRoot(final Reference rootRef) {
176+
// This diverges from the standard construction of Guava network; it delegates the construction to a slightly
177+
// modified version that guarantees a stable iteration order over the edges, which is required to guarantee a
178+
// deterministic planning behavior.
175179
final MutableNetwork<Reference, ReferencePath> network =
176-
NetworkBuilder.directed()
180+
new StableStandardMutableNetwork<>(NetworkBuilder.directed()
177181
.allowsParallelEdges(true)
178182
.allowsSelfLoops(true)
179-
.build();
183+
.edgeOrder(ElementOrder.insertion())
184+
.nodeOrder(ElementOrder.insertion()));
180185

181186
final SetMultimap<RelationalExpression, Reference> containedInMap =
182-
Multimaps.newSetMultimap(Maps.newIdentityHashMap(), Sets::newHashSet);
183-
final Set<Reference> leafRefs = Sets.newHashSet();
187+
Multimaps.newSetMultimap(new LinkedIdentityMap<>(), LinkedIdentitySet::new);
188+
final Set<Reference> leafRefs = new LinkedIdentitySet<>();
184189
collectNetwork(network, containedInMap, leafRefs, rootRef);
185190

186191
return new Traversal(rootRef, network, containedInMap, leafRefs);

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/debug/Debugger.java

+63-3
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,15 @@
4646
import com.apple.foundationdb.record.query.plan.cascades.debug.eventprotos.PTransformRuleCallEvent;
4747
import com.apple.foundationdb.record.query.plan.cascades.debug.eventprotos.PTranslateCorrelationsEvent;
4848
import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression;
49+
import com.google.common.collect.ImmutableList;
4950
import com.google.errorprone.annotations.CanIgnoreReturnValue;
5051
import com.google.protobuf.Message;
5152

5253
import javax.annotation.Nonnull;
5354
import javax.annotation.Nullable;
55+
import java.util.Collection;
5456
import java.util.Deque;
57+
import java.util.List;
5558
import java.util.Optional;
5659
import java.util.function.Consumer;
5760
import java.util.function.Function;
@@ -919,11 +922,26 @@ public PEvent.Builder toEventBuilder() {
919922
* structures of the planner.
920923
*/
921924
class InsertIntoMemoEvent implements Event {
925+
@Nullable
926+
private final RelationalExpression expression;
927+
922928
@Nonnull
923929
private final Location location;
924930

925-
public InsertIntoMemoEvent(@Nonnull final Location location) {
931+
@Nonnull
932+
private final List<Reference> reusedExpressionReferences;
933+
934+
private InsertIntoMemoEvent(@Nonnull final Location location,
935+
@Nullable final RelationalExpression expression,
936+
@Nonnull final Collection<Reference> reusedExpressionReferences) {
937+
if (expression != null) {
938+
Debugger.registerExpression(expression);
939+
}
940+
this.expression = expression;
926941
this.location = location;
942+
this.reusedExpressionReferences = ImmutableList.copyOf(reusedExpressionReferences);
943+
// Call debugger hook to potentially register this new reference.
944+
this.reusedExpressionReferences.forEach(Debugger::registerReference);
927945
}
928946

929947
@Override
@@ -938,6 +956,16 @@ public Shorthand getShorthand() {
938956
return Shorthand.INSERT_INTO_MEMO;
939957
}
940958

959+
@Nullable
960+
public RelationalExpression getExpression() {
961+
return expression;
962+
}
963+
964+
@Nonnull
965+
public Collection<Reference> getReusedExpressionReferences() {
966+
return reusedExpressionReferences;
967+
}
968+
941969
@Nonnull
942970
@Override
943971
public Location getLocation() {
@@ -947,9 +975,15 @@ public Location getLocation() {
947975
@Nonnull
948976
@Override
949977
public PInsertIntoMemoEvent toProto() {
950-
return PInsertIntoMemoEvent.newBuilder()
978+
final var builder = PInsertIntoMemoEvent.newBuilder()
951979
.setLocation(getLocation().name())
952-
.build();
980+
.addAllReusedExpressionReferences(getReusedExpressionReferences().stream()
981+
.map(Event::toReferenceProto)
982+
.collect(ImmutableList.toImmutableList()));
983+
if (expression != null) {
984+
builder.setExpression(Event.toExpressionProto(expression));
985+
}
986+
return builder.build();
953987
}
954988

955989
@Nonnull
@@ -958,6 +992,32 @@ public PEvent.Builder toEventBuilder() {
958992
return PEvent.newBuilder()
959993
.setInsertIntoMemoEvent(toProto());
960994
}
995+
996+
@Nonnull
997+
public static InsertIntoMemoEvent begin() {
998+
return new InsertIntoMemoEvent(Location.BEGIN, null, ImmutableList.of());
999+
}
1000+
1001+
@Nonnull
1002+
public static InsertIntoMemoEvent end() {
1003+
return new InsertIntoMemoEvent(Location.END, null, ImmutableList.of());
1004+
}
1005+
1006+
@Nonnull
1007+
public static InsertIntoMemoEvent newExp(@Nonnull final RelationalExpression expression) {
1008+
return new InsertIntoMemoEvent(Location.NEW, expression, ImmutableList.of());
1009+
}
1010+
1011+
@Nonnull
1012+
public static InsertIntoMemoEvent reusedExp(@Nonnull final RelationalExpression expression) {
1013+
return new InsertIntoMemoEvent(Location.REUSED, expression, ImmutableList.of());
1014+
}
1015+
1016+
@Nonnull
1017+
public static InsertIntoMemoEvent reusedExpWithReferences(@Nonnull final RelationalExpression expression,
1018+
@Nonnull final List<Reference> references) {
1019+
return new InsertIntoMemoEvent(Location.REUSED, expression, references);
1020+
}
9611021
}
9621022

9631023
/**

0 commit comments

Comments
 (0)