Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Impose runtime limits on temp table and recursive queries #2999

Draft
wants to merge 32 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
ebfe5e2
introduce RecursiveUnionExpression and RecursiveUnionQueryPlan.
hatyo Nov 14, 2024
8259bfb
Introduce RecursiveUnionCursor initial implementation.
hatyo Nov 14, 2024
3a7dc97
Initial implementation of recursion algorithm.
hatyo Nov 15, 2024
e64a384
add implementation rule to RecursionUnionExpression.
hatyo Nov 15, 2024
aba773e
add recursive union test, refactor TempTableTest.
hatyo Nov 18, 2024
b6d7fcf
Refactoring, more tests, add ordering pushdown rules for new plan ope…
hatyo Nov 18, 2024
b2cab41
Fix checkstyle violations, add documentation.
hatyo Nov 18, 2024
1dc9552
Resolve post-rebase conflicts.
hatyo Nov 19, 2024
811d4f3
add test for TempTableScan across continuations.
hatyo Nov 19, 2024
751c04b
Factor out hierarchy plan creation logic in test.
hatyo Nov 20, 2024
d2ab23d
Refactor recursive union test to make it possible to test execution w…
hatyo Nov 20, 2024
b5c1120
WIP - add method to init the temp table in evaluation context
hatyo Nov 20, 2024
560dd0c
WIP - implement Factory of TempTable and use it in EvaluationContext.
hatyo Nov 20, 2024
9f7aa0a
Refactor recursive union with correct temp table ownership.
hatyo Nov 20, 2024
e9fee48
Refactor recursive union continuation, add isInitialState flag.
hatyo Nov 20, 2024
4da95cb
checkpoint all tests are passing.
hatyo Nov 21, 2024
b15c909
Refactoring, recursive union tests with continuations now work.
hatyo Nov 22, 2024
4c8b6bf
WIP - edit this.
hatyo Nov 26, 2024
2c984b9
WIP. This must be cleaned up.
hatyo Nov 28, 2024
6b59c86
Algorithm is correct, refactoring, cleanup.
hatyo Dec 2, 2024
44bcaf9
more hierarchy test utils.
hatyo Dec 2, 2024
80f940f
add random hierarchy test.
hatyo Dec 2, 2024
4d89808
Use PTempTable instead of bytes where applicable.
hatyo Dec 3, 2024
d276f3d
documentation, refactoring.
hatyo Dec 3, 2024
8ef5b1a
refactoring, documentation, cleanup.
hatyo Dec 3, 2024
2f017fa
Merge remote-tracking branch 'upstream/main' into recursive-cte
hatyo Dec 3, 2024
cf03c8e
fix post-merge conflicts.
hatyo Dec 3, 2024
0f25f33
reduce size of randmized hierarchy to 1000 elements per level.
hatyo Dec 3, 2024
660c1e5
cleanup.
hatyo Dec 3, 2024
8ebc39d
add row limit to temp table insert to control its growth.
hatyo Dec 10, 2024
b96e503
add recursive union test with row limit violation.
hatyo Dec 10, 2024
7cbf4a2
add maximum allowed recursion depth runtime limit.
hatyo Dec 10, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@
import com.apple.foundationdb.annotation.API;
import com.apple.foundationdb.record.logging.LogMessageKeys;
import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
import com.apple.foundationdb.record.query.plan.cascades.TempTable;
import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository;
import com.google.common.collect.ImmutableMap;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
Expand All @@ -45,7 +47,11 @@ public class EvaluationContext {
@Nonnull
private final TypeRepository typeRepository;

public static final EvaluationContext EMPTY = new EvaluationContext(Bindings.EMPTY_BINDINGS, TypeRepository.EMPTY_SCHEMA);
@Nonnull
private final TempTable.Factory tempTableFactory;

public static final EvaluationContext EMPTY = new EvaluationContext(Bindings.EMPTY_BINDINGS, TypeRepository.EMPTY_SCHEMA,
TempTable.Factory.instance());

/**
* Get an empty evaluation context.
Expand All @@ -56,9 +62,12 @@ public static EvaluationContext empty() {
return EMPTY;
}

private EvaluationContext(@Nonnull Bindings bindings, @Nonnull TypeRepository typeRepository) {
private EvaluationContext(@Nonnull final Bindings bindings,
@Nonnull final TypeRepository typeRepository,
@Nonnull final TempTable.Factory tempTableFactory) {
this.bindings = bindings;
this.typeRepository = typeRepository;
this.tempTableFactory = tempTableFactory;
}

/**
Expand All @@ -69,24 +78,65 @@ private EvaluationContext(@Nonnull Bindings bindings, @Nonnull TypeRepository ty
*/
@Nonnull
public static EvaluationContext forBindings(@Nonnull Bindings bindings) {
return new EvaluationContext(bindings, TypeRepository.EMPTY_SCHEMA);
return new EvaluationContext(bindings, TypeRepository.EMPTY_SCHEMA, TempTable.Factory.instance());
}

/**
* Create a new {@link EvaluationContext} around a given set of {@link Bindings} and a {@link TypeRepository}.
* from parameter names to values.
* Create a new {@link EvaluationContext} around a given set of {@link Bindings} from parameter names to values,
* and a {@link TypeRepository}.
* @param bindings a mapping from parameter name to values
* @param typeRepository a type repository
* @return a new evaluation context with the bindings and the schema.
*/
@Nonnull
public static EvaluationContext forBindingsAndTypeRepository(@Nonnull Bindings bindings, @Nonnull TypeRepository typeRepository) {
return new EvaluationContext(bindings, typeRepository);
public static EvaluationContext forBindingsAndTypeRepository(@Nonnull final Bindings bindings,
@Nonnull final TypeRepository typeRepository) {
return new EvaluationContext(bindings, typeRepository, TempTable.Factory.instance());
}

@Nonnull
public static EvaluationContext forTypeRepository(@Nonnull TypeRepository typeRepository) {
return new EvaluationContext(Bindings.EMPTY_BINDINGS, typeRepository);
public static EvaluationContext forTypeRepository(@Nonnull final TypeRepository typeRepository) {
return new EvaluationContext(Bindings.EMPTY_BINDINGS, typeRepository, TempTable.Factory.instance());
}

/**
* Create a new {@link EvaluationContext} around a given set of {@link Bindings} from parameter names to values,
* and a custom {@link TempTable.Factory}.
* @param bindings a mapping from parameter name to values
* @param tempTableFactory a factory of {@link TempTable} instances.
* @return a new evaluation context with the bindings and the schema.
*/
@Nonnull
public static EvaluationContext forBindingsAndTempTableFactory(@Nonnull final Bindings bindings,
@Nonnull final TempTable.Factory tempTableFactory) {
return new EvaluationContext(bindings, TypeRepository.EMPTY_SCHEMA, tempTableFactory);
}

/**
* Create a new {@link EvaluationContext} around a custom {@link TempTable.Factory} and a {@link TypeRepository}.
* @param typeRepository a type repository
* @param tempTableFactory a factory of {@link TempTable} instances.
* @return a new evaluation context with the bindings and the schema.
*/
@Nonnull
public static EvaluationContext forTypeRepositoryAndTempTableFactory(@Nonnull final TypeRepository typeRepository,
@Nonnull final TempTable.Factory tempTableFactory) {
return new EvaluationContext(Bindings.EMPTY_BINDINGS, typeRepository, tempTableFactory);
}

/**
* Create a new {@link EvaluationContext} around a given set of {@link Bindings} from parameter names to values,
* a {@link TypeRepository}, and a custom {@link TempTable.Factory}.
* @param bindings a mapping from parameter name to values
* @param typeRepository a type repository
* @param tempTableFactory a factory of {@link TempTable} instances.
* @return a new evaluation context with the bindings and the schema.
*/
@Nonnull
public static EvaluationContext forBindingsAndTypeRepositoryAndTempTableFactory(@Nonnull final Bindings bindings,
@Nonnull final TypeRepository typeRepository,
@Nonnull final TempTable.Factory tempTableFactory) {
return new EvaluationContext(bindings, typeRepository, TempTable.Factory.instance());
}

/**
Expand All @@ -98,7 +148,8 @@ public static EvaluationContext forTypeRepository(@Nonnull TypeRepository typeRe
*/
@Nonnull
public static EvaluationContext forBinding(@Nonnull String bindingName, @Nullable Object value) {
return new EvaluationContext(Bindings.newBuilder().set(bindingName, value).build(), TypeRepository.EMPTY_SCHEMA);
return new EvaluationContext(Bindings.newBuilder().set(bindingName, value).build(), TypeRepository.EMPTY_SCHEMA,
TempTable.Factory.instance());
}

/**
Expand Down Expand Up @@ -209,7 +260,29 @@ public EvaluationContext withBinding(@Nonnull String bindingName, @Nullable Obje
*
* @return a new <code>EvaluationContext</code> with the new binding
*/
public EvaluationContext withBinding(final Bindings.Internal type, @Nonnull CorrelationIdentifier alias, @Nullable Object value) {
@Nonnull
public EvaluationContext withBinding(@Nonnull final Bindings.Internal type,
@Nonnull final CorrelationIdentifier alias,
@Nullable final Object value) {
return childBuilder().setBinding(type.bindingName(alias.getId()), value).build(typeRepository);
}

@Nonnull
public EvaluationContext withNewTempTableBinding(@Nonnull final Bindings.Internal type,
@Nonnull final CorrelationIdentifier alias) {
final var tempTable = getTempTableFactory().createTempTable();
// CONSTANT and CORRELATION types expect a different layout of the values.
// ideally this should be streamlined differently.
if (type == Bindings.Internal.CONSTANT) {
final ImmutableMap.Builder<String, Object> constants = ImmutableMap.builder();
constants.put(alias.getId(), tempTable);
return childBuilder().setBinding(type.bindingName(alias.getId()), constants.build()).build(typeRepository);
}
return childBuilder().setBinding(type.bindingName(alias.getId()), getTempTableFactory().createTempTable()).build(typeRepository);
}

@Nonnull
public TempTable.Factory getTempTableFactory() {
return tempTableFactory;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

import com.apple.foundationdb.annotation.API;
import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
import com.apple.foundationdb.record.query.plan.cascades.TempTable;
import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository;

import javax.annotation.Nonnull;
Expand All @@ -37,12 +38,15 @@
public class EvaluationContextBuilder {
@Nonnull
protected final Bindings.Builder bindings;
@Nonnull
protected final TempTable.Factory tempTableFactory;

/**
* Create an empty builder.
*/
protected EvaluationContextBuilder() {
this.bindings = Bindings.newBuilder();
this.tempTableFactory = TempTable.Factory.instance();
}

/**
Expand All @@ -54,6 +58,7 @@ protected EvaluationContextBuilder() {
*/
protected EvaluationContextBuilder(@Nonnull EvaluationContext original) {
this.bindings = original.getBindings().childBuilder();
this.tempTableFactory = original.getTempTableFactory();
}

/**
Expand Down Expand Up @@ -116,6 +121,6 @@ public EvaluationContextBuilder setConstant(@Nonnull CorrelationIdentifier alias
*/
@Nonnull
public EvaluationContext build(@Nonnull final TypeRepository typeRepository) {
return EvaluationContext.forBindingsAndTypeRepository(bindings.build(), typeRepository);
return EvaluationContext.forBindingsAndTypeRepositoryAndTempTableFactory(bindings.build(), typeRepository, tempTableFactory);
}
}
Loading