Skip to content

Commit

Permalink
Split functions configuration from FeaturesConfig
Browse files Browse the repository at this point in the history
  • Loading branch information
@tdcmeehan
tdcmeehan committed Aug 30, 2024
1 parent cd2fb54 commit cac6682
Show file tree
Hide file tree
Showing 40 changed files with 589 additions and 469 deletions.
17 changes: 10 additions & 7 deletions presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import com.facebook.presto.sql.analyzer.FeaturesConfig.RandomizeOuterJoinNullKeyStrategy;
import com.facebook.presto.sql.analyzer.FeaturesConfig.ShardedJoinStrategy;
import com.facebook.presto.sql.analyzer.FeaturesConfig.SingleStreamSpillerChoice;
import com.facebook.presto.sql.analyzer.FunctionsConfig;
import com.facebook.presto.sql.planner.CompilerConfig;
import com.facebook.presto.sql.tree.CreateView;
import com.facebook.presto.tracing.TracingConfig;
Expand Down Expand Up @@ -369,6 +370,7 @@ public SystemSessionProperties()
new TaskManagerConfig(),
new MemoryManagerConfig(),
new FeaturesConfig(),
new FunctionsConfig(),
new NodeMemoryConfig(),
new WarningCollectorConfig(),
new NodeSchedulerConfig(),
Expand All @@ -384,6 +386,7 @@ public SystemSessionProperties(
TaskManagerConfig taskManagerConfig,
MemoryManagerConfig memoryManagerConfig,
FeaturesConfig featuresConfig,
FunctionsConfig functionsConfig,
NodeMemoryConfig nodeMemoryConfig,
WarningCollectorConfig warningCollectorConfig,
NodeSchedulerConfig nodeSchedulerConfig,
Expand Down Expand Up @@ -898,12 +901,12 @@ public SystemSessionProperties(
booleanProperty(
LEGACY_ROW_FIELD_ORDINAL_ACCESS,
"Allow accessing anonymous row field with .field0, .field1, ...",
featuresConfig.isLegacyRowFieldOrdinalAccess(),
functionsConfig.isLegacyRowFieldOrdinalAccess(),
false),
booleanProperty(
LEGACY_MAP_SUBSCRIPT,
"Do not fail the query if map key is missing",
featuresConfig.isLegacyMapSubscript(),
functionsConfig.isLegacyMapSubscript(),
true),
booleanProperty(
ITERATIVE_OPTIMIZER,
Expand Down Expand Up @@ -946,7 +949,7 @@ public SystemSessionProperties(
booleanProperty(
LEGACY_TIMESTAMP,
"Use legacy TIME & TIMESTAMP semantics (warning: this will be removed)",
featuresConfig.isLegacyTimestamp(),
functionsConfig.isLegacyTimestamp(),
true),
booleanProperty(
ENABLE_INTERMEDIATE_AGGREGATIONS,
Expand All @@ -966,7 +969,7 @@ public SystemSessionProperties(
booleanProperty(
PARSE_DECIMAL_LITERALS_AS_DOUBLE,
"Parse decimal literals as DOUBLE instead of DECIMAL",
featuresConfig.isParseDecimalLiteralsAsDouble(),
functionsConfig.isParseDecimalLiteralsAsDouble(),
false),
booleanProperty(
FORCE_SINGLE_NODE_OUTPUT,
Expand Down Expand Up @@ -1803,12 +1806,12 @@ public SystemSessionProperties(
booleanProperty(
FIELD_NAMES_IN_JSON_CAST_ENABLED,
"Include field names in json output when casting rows",
featuresConfig.isFieldNamesInJsonCastEnabled(),
functionsConfig.isFieldNamesInJsonCastEnabled(),
false),
booleanProperty(
LEGACY_JSON_CAST,
"Keep the legacy json cast behavior, do not reserve the case for field names when casting to row type",
featuresConfig.isLegacyJsonCast(),
functionsConfig.isLegacyJsonCast(),
true),
booleanProperty(
OPTIMIZE_JOIN_PROBE_FOR_EMPTY_BUILD_RUNTIME,
Expand Down Expand Up @@ -2000,7 +2003,7 @@ public SystemSessionProperties(
false),
booleanProperty(WARN_ON_COMMON_NAN_PATTERNS,
"Whether to give a warning for some common issues relating to NaNs",
featuresConfig.getWarnOnCommonNanPatterns(),
functionsConfig.getWarnOnCommonNanPatterns(),
false),
booleanProperty(INLINE_PROJECTIONS_ON_VALUES,
"Whether to evaluate project node on values node",
Expand Down
44 changes: 22 additions & 22 deletions ...in/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@
import com.facebook.presto.spi.function.SqlFunctionVisibility;
import com.facebook.presto.spi.function.SqlInvokedFunction;
import com.facebook.presto.spi.function.SqlInvokedScalarFunctionImplementation;
import com.facebook.presto.sql.analyzer.FeaturesConfig;
import com.facebook.presto.sql.analyzer.FunctionsConfig;
import com.facebook.presto.sql.analyzer.TypeSignatureProvider;
import com.facebook.presto.type.BigintOperators;
import com.facebook.presto.type.BooleanOperators;
Expand Down Expand Up @@ -556,7 +556,7 @@ public class BuiltInTypeAndFunctionNamespaceManager

public BuiltInTypeAndFunctionNamespaceManager(
BlockEncodingSerde blockEncodingSerde,
FeaturesConfig featuresConfig,
FunctionsConfig functionsConfig,
Set<Type> types,
FunctionAndTypeManager functionAndTypeManager)
{
Expand Down Expand Up @@ -612,15 +612,15 @@ public BuiltInTypeAndFunctionNamespaceManager(
.expireAfterWrite(1, HOURS)
.build(CacheLoader.from(this::instantiateParametricType));

registerBuiltInFunctions(getBuiltInFunctions(featuresConfig));
registerBuiltInTypes(featuresConfig);
registerBuiltInFunctions(getBuiltInFunctions(functionsConfig));
registerBuiltInTypes(functionsConfig);

for (Type type : requireNonNull(types, "types is null")) {
addType(type);
}
}

private void registerBuiltInTypes(FeaturesConfig featuresConfig)
private void registerBuiltInTypes(FunctionsConfig functionsConfig)
{
// always add the built-in types; Presto will not function without these
addType(UNKNOWN);
Expand All @@ -629,7 +629,7 @@ private void registerBuiltInTypes(FeaturesConfig featuresConfig)
addType(INTEGER);
addType(SMALLINT);
addType(TINYINT);
if (!featuresConfig.getUseNewNanDefinition()) {
if (!functionsConfig.getUseNewNanDefinition()) {
addType(OLD_NAN_DOUBLE);
addType(OLD_NAN_REAL);
}
Expand Down Expand Up @@ -678,7 +678,7 @@ private void registerBuiltInTypes(FeaturesConfig featuresConfig)
addParametricType(VARCHAR_ENUM);
}

private List<? extends SqlFunction> getBuiltInFunctions(FeaturesConfig featuresConfig)
private List<? extends SqlFunction> getBuiltInFunctions(FunctionsConfig functionsConfig)
{
FunctionListBuilder builder = new FunctionListBuilder()
.window(RowNumberFunction.class)
Expand Down Expand Up @@ -807,7 +807,7 @@ private List<? extends SqlFunction> getBuiltInFunctions(FeaturesConfig featuresC
.scalars(DoubleOperators.class)
.scalars(RealOperators.class);

if (featuresConfig.getUseNewNanDefinition()) {
if (functionsConfig.getUseNewNanDefinition()) {
builder.scalars(DoubleComparisonOperators.class)
.scalar(DoubleComparisonOperators.DoubleDistinctFromOperator.class)
.scalars(RealComparisonOperators.class)
Expand Down Expand Up @@ -933,11 +933,11 @@ private List<? extends SqlFunction> getBuiltInFunctions(FeaturesConfig featuresC
.function(ARRAY_CONCAT_FUNCTION)
.functions(ARRAY_CONSTRUCTOR, ARRAY_SUBSCRIPT, ARRAY_TO_JSON, JSON_TO_ARRAY, JSON_STRING_TO_ARRAY)
.functions(SET_AGG, SET_UNION)
.function(new ArrayAggregationFunction(featuresConfig.isLegacyArrayAgg(), featuresConfig.getArrayAggGroupImplementation()))
.functions(new MapSubscriptOperator(featuresConfig.isLegacyMapSubscript()))
.function(new ArrayAggregationFunction(functionsConfig.isLegacyArrayAgg(), functionsConfig.getArrayAggGroupImplementation()))
.functions(new MapSubscriptOperator(functionsConfig.isLegacyMapSubscript()))
.functions(MAP_CONSTRUCTOR, MAP_TO_JSON, JSON_TO_MAP, JSON_STRING_TO_MAP)
.functions(MAP_AGG, MAP_UNION, MAP_UNION_SUM)
.function(new ReduceAggregationFunction(featuresConfig.isReduceAggForComplexTypesEnabled()))
.function(new ReduceAggregationFunction(functionsConfig.isReduceAggForComplexTypesEnabled()))
.functions(DECIMAL_TO_VARCHAR_CAST, DECIMAL_TO_INTEGER_CAST, DECIMAL_TO_BIGINT_CAST, DECIMAL_TO_DOUBLE_CAST, DECIMAL_TO_REAL_CAST, DECIMAL_TO_BOOLEAN_CAST, DECIMAL_TO_TINYINT_CAST, DECIMAL_TO_SMALLINT_CAST)
.functions(VARCHAR_TO_DECIMAL_CAST, INTEGER_TO_DECIMAL_CAST, BIGINT_TO_DECIMAL_CAST, DOUBLE_TO_DECIMAL_CAST, REAL_TO_DECIMAL_CAST, BOOLEAN_TO_DECIMAL_CAST, TINYINT_TO_DECIMAL_CAST, SMALLINT_TO_DECIMAL_CAST)
.functions(JSON_TO_DECIMAL_CAST, DECIMAL_TO_JSON_CAST)
Expand All @@ -952,7 +952,7 @@ private List<? extends SqlFunction> getBuiltInFunctions(FeaturesConfig featuresC
.functions(DECIMAL_TO_TINYINT_SATURATED_FLOOR_CAST, TINYINT_TO_DECIMAL_SATURATED_FLOOR_CAST)
.function(DECIMAL_BETWEEN_OPERATOR)
.function(DECIMAL_DISTINCT_FROM_OPERATOR)
.function(new Histogram(featuresConfig.getHistogramGroupImplementation()))
.function(new Histogram(functionsConfig.getHistogramGroupImplementation()))
.function(CHECKSUM_AGGREGATION)
.function(IDENTITY_CAST)
.function(ARBITRARY_AGGREGATION)
Expand All @@ -967,8 +967,8 @@ private List<? extends SqlFunction> getBuiltInFunctions(FeaturesConfig featuresC
.functions(ROW_HASH_CODE, ROW_TO_JSON, JSON_TO_ROW, JSON_STRING_TO_ROW, ROW_DISTINCT_FROM, ROW_EQUAL, ROW_GREATER_THAN, ROW_GREATER_THAN_OR_EQUAL, ROW_LESS_THAN, ROW_LESS_THAN_OR_EQUAL, ROW_NOT_EQUAL, ROW_TO_ROW_CAST, ROW_INDETERMINATE)
.functions(VARCHAR_CONCAT, VARBINARY_CONCAT)
.function(DECIMAL_TO_DECIMAL_CAST)
.function(castVarcharToRe2JRegexp(featuresConfig.getRe2JDfaStatesLimit(), featuresConfig.getRe2JDfaRetries()))
.function(castCharToRe2JRegexp(featuresConfig.getRe2JDfaStatesLimit(), featuresConfig.getRe2JDfaRetries()))
.function(castVarcharToRe2JRegexp(functionsConfig.getRe2JDfaStatesLimit(), functionsConfig.getRe2JDfaRetries()))
.function(castCharToRe2JRegexp(functionsConfig.getRe2JDfaStatesLimit(), functionsConfig.getRe2JDfaRetries()))
.function(DECIMAL_AVERAGE_AGGREGATION)
.function(DECIMAL_SUM_AGGREGATION)
.function(DECIMAL_MOD_FUNCTION)
Expand Down Expand Up @@ -1015,7 +1015,7 @@ private List<? extends SqlFunction> getBuiltInFunctions(FeaturesConfig featuresC
.scalars(KllSketchFunctions.class)
.scalars(KllSketchOperators.class);

switch (featuresConfig.getRegexLibrary()) {
switch (functionsConfig.getRegexLibrary()) {
case JONI:
builder.scalars(JoniRegexpFunctions.class);
builder.scalar(JoniRegexpReplaceLambdaFunction.class);
Expand All @@ -1026,20 +1026,20 @@ private List<? extends SqlFunction> getBuiltInFunctions(FeaturesConfig featuresC
break;
}

if (featuresConfig.isLegacyLogFunction()) {
if (functionsConfig.isLegacyLogFunction()) {
builder.scalar(LegacyLogFunction.class);
}

// Replace some aggregations for Velox to override intermediate aggregation type.
if (featuresConfig.isUseAlternativeFunctionSignatures()) {
if (functionsConfig.isUseAlternativeFunctionSignatures()) {
builder.override(ARBITRARY_AGGREGATION, ALTERNATIVE_ARBITRARY_AGGREGATION);
builder.override(ANY_VALUE_AGGREGATION, ALTERNATIVE_ANY_VALUE_AGGREGATION);
builder.override(MAX_AGGREGATION, ALTERNATIVE_MAX);
builder.override(MIN_AGGREGATION, ALTERNATIVE_MIN);
builder.override(MAX_BY, ALTERNATIVE_MAX_BY);
builder.override(MIN_BY, ALTERNATIVE_MIN_BY);
builder.functions(AlternativeApproxPercentile.getFunctions());
builder.function(new AlternativeMultimapAggregationFunction(featuresConfig.getMultimapAggGroupImplementation()));
builder.function(new AlternativeMultimapAggregationFunction(functionsConfig.getMultimapAggGroupImplementation()));
}
else {
builder.aggregates(ApproximateLongPercentileAggregations.class);
Expand All @@ -1048,12 +1048,12 @@ private List<? extends SqlFunction> getBuiltInFunctions(FeaturesConfig featuresC
builder.aggregates(ApproximateDoublePercentileArrayAggregations.class);
builder.aggregates(ApproximateRealPercentileAggregations.class);
builder.aggregates(ApproximateRealPercentileArrayAggregations.class);
builder.function(new MultimapAggregationFunction(featuresConfig.getMultimapAggGroupImplementation()));
builder.function(new MultimapAggregationFunction(functionsConfig.getMultimapAggGroupImplementation()));
}

if (featuresConfig.getLimitNumberOfGroupsForKHyperLogLogAggregations()) {
builder.function(new MergeKHyperLogLogWithLimitAggregationFunction(featuresConfig.getKHyperLogLogAggregationGroupNumberLimit()));
builder.function(new KHyperLogLogWithLimitAggregationFunction(featuresConfig.getKHyperLogLogAggregationGroupNumberLimit()));
if (functionsConfig.getLimitNumberOfGroupsForKHyperLogLogAggregations()) {
builder.function(new MergeKHyperLogLogWithLimitAggregationFunction(functionsConfig.getKHyperLogLogAggregationGroupNumberLimit()));
builder.function(new KHyperLogLogWithLimitAggregationFunction(functionsConfig.getKHyperLogLogAggregationGroupNumberLimit()));
}
else {
builder.aggregates(MergeKHyperLogLogAggregationFunction.class);
Expand Down
14 changes: 11 additions & 3 deletions presto-main/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
import com.facebook.presto.spi.function.SqlInvokedFunction;
import com.facebook.presto.sql.analyzer.FeaturesConfig;
import com.facebook.presto.sql.analyzer.FunctionAndTypeResolver;
import com.facebook.presto.sql.analyzer.FunctionsConfig;
import com.facebook.presto.sql.analyzer.TypeSignatureProvider;
import com.facebook.presto.sql.gen.CacheStatsMBean;
import com.facebook.presto.sql.tree.QualifiedName;
Expand Down Expand Up @@ -134,12 +135,13 @@ public FunctionAndTypeManager(
TransactionManager transactionManager,
BlockEncodingSerde blockEncodingSerde,
FeaturesConfig featuresConfig,
FunctionsConfig functionsConfig,
HandleResolver handleResolver,
Set<Type> types)
{
this.transactionManager = requireNonNull(transactionManager, "transactionManager is null");
this.blockEncodingSerde = requireNonNull(blockEncodingSerde, "blockEncodingSerde is null");
this.builtInTypeAndFunctionNamespaceManager = new BuiltInTypeAndFunctionNamespaceManager(blockEncodingSerde, featuresConfig, types, this);
this.builtInTypeAndFunctionNamespaceManager = new BuiltInTypeAndFunctionNamespaceManager(blockEncodingSerde, functionsConfig, types, this);
this.functionNamespaceManagers.put(DEFAULT_NAMESPACE.getCatalogName(), builtInTypeAndFunctionNamespaceManager);
this.functionInvokerProvider = new FunctionInvokerProvider(this);
this.handleResolver = requireNonNull(handleResolver, "handleResolver is null");
Expand All @@ -152,13 +154,19 @@ public FunctionAndTypeManager(
.build(CacheLoader.from(key -> resolveBuiltInFunction(key.functionName, fromTypeSignatures(key.parameterTypes))));
this.cacheStatsMBean = new CacheStatsMBean(functionCache);
this.functionSignatureMatcher = new FunctionSignatureMatcher(this);
this.typeCoercer = new TypeCoercer(featuresConfig, this);
this.typeCoercer = new TypeCoercer(functionsConfig, this);
this.nativeExecution = featuresConfig.isNativeExecutionEnabled();
}

public static FunctionAndTypeManager createTestFunctionAndTypeManager()
{
return new FunctionAndTypeManager(createTestTransactionManager(), new BlockEncodingManager(), new FeaturesConfig(), new HandleResolver(), ImmutableSet.of());
return new FunctionAndTypeManager(
createTestTransactionManager(),
new BlockEncodingManager(),
new FeaturesConfig(),
new FunctionsConfig(),
new HandleResolver(),
ImmutableSet.of());
}

public FunctionAndTypeResolver getFunctionAndTypeResolver()
Expand Down
23 changes: 17 additions & 6 deletions presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
import com.facebook.presto.spi.statistics.TableStatistics;
import com.facebook.presto.spi.statistics.TableStatisticsMetadata;
import com.facebook.presto.sql.analyzer.FeaturesConfig;
import com.facebook.presto.sql.analyzer.FunctionsConfig;
import com.facebook.presto.sql.analyzer.TypeSignatureProvider;
import com.facebook.presto.sql.planner.PartitioningHandle;
import com.facebook.presto.transaction.TransactionManager;
Expand Down Expand Up @@ -210,24 +211,34 @@ public static MetadataManager createTestMetadataManager()

public static MetadataManager createTestMetadataManager(FeaturesConfig featuresConfig)
{
return createTestMetadataManager(new CatalogManager(), featuresConfig);
return createTestMetadataManager(new CatalogManager(), featuresConfig, new FunctionsConfig());
}

public static MetadataManager createTestMetadataManager(FunctionsConfig functionsConfig)
{
return createTestMetadataManager(new CatalogManager(), new FeaturesConfig(), functionsConfig);
}

public static MetadataManager createTestMetadataManager(CatalogManager catalogManager)
{
return createTestMetadataManager(catalogManager, new FeaturesConfig());
return createTestMetadataManager(catalogManager, new FeaturesConfig(), new FunctionsConfig());
}

public static MetadataManager createTestMetadataManager(CatalogManager catalogManager, FeaturesConfig featuresConfig, FunctionsConfig functionsConfig)
{
return createTestMetadataManager(createTestTransactionManager(catalogManager), featuresConfig, functionsConfig);
}

public static MetadataManager createTestMetadataManager(CatalogManager catalogManager, FeaturesConfig featuresConfig)
public static MetadataManager createTestMetadataManager(TransactionManager transactionManager)
{
return createTestMetadataManager(createTestTransactionManager(catalogManager), featuresConfig);
return createTestMetadataManager(transactionManager, new FeaturesConfig(), new FunctionsConfig());
}

public static MetadataManager createTestMetadataManager(TransactionManager transactionManager, FeaturesConfig featuresConfig)
public static MetadataManager createTestMetadataManager(TransactionManager transactionManager, FeaturesConfig featuresConfig, FunctionsConfig functionsConfig)
{
BlockEncodingManager blockEncodingManager = new BlockEncodingManager();
return new MetadataManager(
new FunctionAndTypeManager(transactionManager, blockEncodingManager, featuresConfig, new HandleResolver(), ImmutableSet.of()),
new FunctionAndTypeManager(transactionManager, blockEncodingManager, featuresConfig, functionsConfig, new HandleResolver(), ImmutableSet.of()),
blockEncodingManager,
new SessionPropertyManager(),
new SchemaPropertyManager(),
Expand Down
Loading

0 comments on commit cac6682

Please sign in to comment.