Support defining UDFs in RecordMetaData

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java

@@ -31,7 +31,9 @@
 import com.apple.foundationdb.record.metadata.UnnestedRecordType;
 import com.apple.foundationdb.record.metadata.expressions.KeyExpression;
 import com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression;
+import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
+import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry;
 import com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordPlanner;
 import com.apple.foundationdb.record.util.MapUtils;
 import com.google.common.base.Verify;
@@ -83,6 +85,8 @@ public class RecordMetaData implements RecordMetaDataProvider {
     @Nonnull
     private final Map<Object, SyntheticRecordType<?>> recordTypeKeyToSyntheticTypeMap;
     @Nonnull
+    private final Map<String, UserDefinedFunction> userDefinedFunctionMap;
+    @Nonnull
     private final Map<String, Index> indexes;
     @Nonnull
     private final Map<String, Index> universalIndexes;
@@ -112,6 +116,7 @@ protected RecordMetaData(@Nonnull RecordMetaData orig) {
                 Collections.unmodifiableMap(orig.indexes),
                 Collections.unmodifiableMap(orig.universalIndexes),
                 Collections.unmodifiableList(orig.formerIndexes),
+                Collections.unmodifiableMap(orig.userDefinedFunctionMap),
                 orig.splitLongRecords,
                 orig.storeRecordVersions,
                 orig.version,
@@ -131,6 +136,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor,
                              @Nonnull Map<String, Index> indexes,
                              @Nonnull Map<String, Index> universalIndexes,
                              @Nonnull List<FormerIndex> formerIndexes,
+                             @Nonnull Map<String, UserDefinedFunction> userDefinedFunctionMap,
                              boolean splitLongRecords,
                              boolean storeRecordVersions,
                              int version,
@@ -147,6 +153,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor,
         this.indexes = indexes;
         this.universalIndexes = universalIndexes;
         this.formerIndexes = formerIndexes;
+        this.userDefinedFunctionMap = userDefinedFunctionMap;
         this.splitLongRecords = splitLongRecords;
         this.storeRecordVersions = storeRecordVersions;
         this.version = version;
@@ -692,7 +699,9 @@ public RecordMetaDataProto.MetaData toProto(@Nullable Descriptors.FileDescriptor
             builder.addFormerIndexes(formerIndex.toProto());
         }
 
-        // Add in the final options.
+        PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE,
+                PlanHashable.CURRENT_FOR_CONTINUATION);
+        builder.addAllUserDefinedFunctions(userDefinedFunctionMap.values().stream().map(func -> func.toProto(serializationContext)).collect(Collectors.toList()));
         builder.setSplitLongRecords(splitLongRecords);
         builder.setStoreRecordVersions(storeRecordVersions);
         builder.setVersion(version);

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java

@@ -42,6 +42,9 @@
 import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistry;
 import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl;
 import com.apple.foundationdb.record.provider.foundationdb.MetaDataProtoEditor;
+import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction;
+import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry;
+import com.apple.foundationdb.record.query.plan.serialization.PlanSerialization;
 import com.google.common.base.Verify;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
@@ -110,6 +113,8 @@ public class RecordMetaDataBuilder implements RecordMetaDataProvider {
     @Nonnull
     private final Map<String, SyntheticRecordTypeBuilder<?>> syntheticRecordTypes;
     @Nonnull
+    private final Map<String, UserDefinedFunction> userDefinedFunctionMap;
+    @Nonnull
     private final Map<String, Index> indexes;
     @Nonnull
     private final Map<String, Index> universalIndexes;
@@ -144,6 +149,7 @@ public class RecordMetaDataBuilder implements RecordMetaDataProvider {
         indexMaintainerRegistry = IndexMaintainerRegistryImpl.instance();
         evolutionValidator = MetaDataEvolutionValidator.getDefaultInstance();
         syntheticRecordTypes = new HashMap<>();
+        userDefinedFunctionMap = new HashMap<>();
     }
 
     private void processSchemaOptions(boolean processExtensionOptions) {
@@ -222,6 +228,11 @@ private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDa
                 typeBuilder.setRecordTypeKey(LiteralKeyExpression.fromProtoValue(typeProto.getExplicitKey()));
             }
         }
+        for (RecordMetaDataProto.PUserDefinedFunction function: metaDataProto.getUserDefinedFunctionsList()) {
+            UserDefinedFunction func = (UserDefinedFunction)PlanSerialization.dispatchFromProtoContainer(new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE,
+                    PlanHashable.CURRENT_FOR_CONTINUATION), function);
+            userDefinedFunctionMap.put(func.getFunctionName(), func);
+        }
         if (metaDataProto.hasSplitLongRecords()) {
             splitLongRecords = metaDataProto.getSplitLongRecords();
         }
@@ -1179,6 +1190,14 @@ public void addFormerIndex(@Nonnull FormerIndex formerIndex) {
         formerIndexes.add(formerIndex);
     }
 
+    public void addUserDefinedFunction(@Nonnull UserDefinedFunction userDefinedFunction) {
+        userDefinedFunctionMap.put(userDefinedFunction.getFunctionName(), userDefinedFunction);
+    }
+
+    public void addUserDefinedFunctions(@Nonnull Iterable<? extends UserDefinedFunction> functions) {
+        functions.forEach(this::addUserDefinedFunction);
+    }
+
     public boolean isSplitLongRecords() {
         return splitLongRecords;
     }
@@ -1420,7 +1439,7 @@ public RecordMetaData build(boolean validate) {
         Map<Object, SyntheticRecordType<?>> recordTypeKeyToSyntheticRecordTypeMap = Maps.newHashMapWithExpectedSize(syntheticRecordTypes.size());
         RecordMetaData metaData = new RecordMetaData(recordsDescriptor, getUnionDescriptor(), unionFields,
                 builtRecordTypes, builtSyntheticRecordTypes, recordTypeKeyToSyntheticRecordTypeMap,
-                indexes, universalIndexes, formerIndexes,
+                indexes, universalIndexes, formerIndexes, userDefinedFunctionMap,
                 splitLongRecords, storeRecordVersions, version, subspaceKeyCounter, usesSubspaceKeyCounter, recordCountKey, localFileDescriptor != null);
         for (RecordTypeBuilder recordTypeBuilder : recordTypes.values()) {
             KeyExpression primaryKey = recordTypeBuilder.getPrimaryKey();

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Key.java

@@ -22,7 +22,7 @@
 
 import com.apple.foundationdb.annotation.API;
 import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings;
-import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
 import com.apple.foundationdb.record.logging.LogMessageKeys;
 import com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression;
 import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression;
@@ -392,21 +392,21 @@ public static class Evaluated {
          * Values used in index keys in place of missing fields.
          */
         public enum NullStandin {
-            NULL(RecordMetaDataProto.Field.NullInterpretation.NOT_UNIQUE), // Missing field here skips uniqueness checks.
-            NULL_UNIQUE(RecordMetaDataProto.Field.NullInterpretation.UNIQUE), // Missing field here like ordinary value, but null, for uniqueness.
-            NOT_NULL(RecordMetaDataProto.Field.NullInterpretation.NOT_NULL); // Missing field has type's ordinary default value.
+            NULL(RecordKeyExpressionProto.Field.NullInterpretation.NOT_UNIQUE), // Missing field here skips uniqueness checks.
+            NULL_UNIQUE(RecordKeyExpressionProto.Field.NullInterpretation.UNIQUE), // Missing field here like ordinary value, but null, for uniqueness.
+            NOT_NULL(RecordKeyExpressionProto.Field.NullInterpretation.NOT_NULL); // Missing field has type's ordinary default value.
 
-            private RecordMetaDataProto.Field.NullInterpretation proto;
+            private RecordKeyExpressionProto.Field.NullInterpretation proto;
 
-            NullStandin(RecordMetaDataProto.Field.NullInterpretation nullInterpretation) {
+            NullStandin(RecordKeyExpressionProto.Field.NullInterpretation nullInterpretation) {
                 proto = nullInterpretation;
             }
 
-            public RecordMetaDataProto.Field.NullInterpretation toProto() {
+            public RecordKeyExpressionProto.Field.NullInterpretation toProto() {
                 return proto;
             }
 
-            public static NullStandin valueOf(RecordMetaDataProto.Field.NullInterpretation nullInterpretation) throws KeyExpression.DeserializationException {
+            public static NullStandin valueOf(RecordKeyExpressionProto.Field.NullInterpretation nullInterpretation) throws KeyExpression.DeserializationException {
                 switch (nullInterpretation) {
                     case NOT_UNIQUE:
                         return NULL;

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java

@@ -23,7 +23,7 @@
 import com.apple.foundationdb.annotation.API;
 import com.apple.foundationdb.record.ObjectPlanHash;
 import com.apple.foundationdb.record.PlanHashable;
-import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
 import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
@@ -63,7 +63,7 @@ private DimensionsKeyExpression(@Nonnull final KeyExpression wholeKey,
         this.dimensionsSize = dimensionsSize;
     }
 
-    DimensionsKeyExpression(@Nonnull final RecordMetaDataProto.Dimensions dimensions) throws DeserializationException {
+    DimensionsKeyExpression(@Nonnull final RecordKeyExpressionProto.Dimensions dimensions) throws DeserializationException {
         this(KeyExpression.fromProto(dimensions.getWholeKey()), dimensions.getPrefixSize(), dimensions.getDimensionsSize());
     }
 
@@ -114,8 +114,8 @@ public boolean needsCopyingToPartialRecord() {
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.Dimensions toProto() throws SerializationException {
-        final RecordMetaDataProto.Dimensions.Builder builder = RecordMetaDataProto.Dimensions.newBuilder();
+    public RecordKeyExpressionProto.Dimensions toProto() throws SerializationException {
+        final RecordKeyExpressionProto.Dimensions.Builder builder = RecordKeyExpressionProto.Dimensions.newBuilder();
         builder.setWholeKey(getWholeKey().toKeyExpression());
         builder.setPrefixSize(prefixSize);
         builder.setDimensionsSize(dimensionsSize);
@@ -124,8 +124,8 @@ public RecordMetaDataProto.Dimensions toProto() throws SerializationException {
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.KeyExpression toKeyExpression() {
-        return RecordMetaDataProto.KeyExpression.newBuilder().setDimensions(toProto()).build();
+    public RecordKeyExpressionProto.KeyExpression toKeyExpression() {
+        return RecordKeyExpressionProto.KeyExpression.newBuilder().setDimensions(toProto()).build();
     }
 
     @Nonnull

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java

@@ -23,7 +23,7 @@
 import com.apple.foundationdb.annotation.API;
 import com.apple.foundationdb.record.ObjectPlanHash;
 import com.apple.foundationdb.record.PlanHashable;
-import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
 import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
@@ -45,8 +45,8 @@ public class EmptyKeyExpression extends BaseKeyExpression implements KeyExpressi
     private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Empty-Key-Expression");
 
     public static final EmptyKeyExpression EMPTY = new EmptyKeyExpression();
-    public static final RecordMetaDataProto.KeyExpression EMPTY_PROTO =
-            RecordMetaDataProto.KeyExpression.newBuilder().setEmpty(EMPTY.toProto()).build();
+    public static final RecordKeyExpressionProto.KeyExpression EMPTY_PROTO =
+            RecordKeyExpressionProto.KeyExpression.newBuilder().setEmpty(EMPTY.toProto()).build();
 
     private EmptyKeyExpression() {
         // nothing to initialize
@@ -75,8 +75,8 @@ public int getColumnSize() {
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.Empty toProto() throws SerializationException {
-        return RecordMetaDataProto.Empty.getDefaultInstance();
+    public RecordKeyExpressionProto.Empty toProto() throws SerializationException {
+        return RecordKeyExpressionProto.Empty.getDefaultInstance();
     }
 
     @Nonnull
@@ -87,7 +87,7 @@ public <S extends KeyExpressionVisitor.State, R> R expand(@Nonnull final KeyExpr
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.KeyExpression toKeyExpression() {
+    public RecordKeyExpressionProto.KeyExpression toKeyExpression() {
         return EMPTY_PROTO;
     }
 

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FieldKeyExpression.java

@@ -24,7 +24,7 @@
 import com.apple.foundationdb.record.ObjectPlanHash;
 import com.apple.foundationdb.record.PlanHashable;
 import com.apple.foundationdb.record.RecordCoreException;
-import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
 import com.apple.foundationdb.record.logging.LogMessageKeys;
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
@@ -71,7 +71,7 @@ public FieldKeyExpression(@Nonnull String fieldName, @Nonnull FanType fanType, @
         this.nullStandin = nullStandin;
     }
 
-    public FieldKeyExpression(@Nonnull RecordMetaDataProto.Field field) throws DeserializationException {
+    public FieldKeyExpression(@Nonnull RecordKeyExpressionProto.Field field) throws DeserializationException {
         if (!field.hasFieldName()) {
             throw new DeserializationException("Serialized Field is missing field name");
         }
@@ -193,8 +193,8 @@ public int getColumnSize() {
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.Field toProto() throws SerializationException {
-        return RecordMetaDataProto.Field.newBuilder()
+    public RecordKeyExpressionProto.Field toProto() throws SerializationException {
+        return RecordKeyExpressionProto.Field.newBuilder()
                 .setFieldName(fieldName)
                 .setFanType(fanType.toProto())
                 .setNullInterpretation(nullStandin.toProto())
@@ -203,8 +203,8 @@ public RecordMetaDataProto.Field toProto() throws SerializationException {
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.KeyExpression toKeyExpression() {
-        return RecordMetaDataProto.KeyExpression.newBuilder().setField(toProto()).build();
+    public RecordKeyExpressionProto.KeyExpression toKeyExpression() {
+        return RecordKeyExpressionProto.KeyExpression.newBuilder().setField(toProto()).build();
     }
 
     @Nonnull

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java

@@ -25,7 +25,7 @@
 import com.apple.foundationdb.record.PlanHashable;
 import com.apple.foundationdb.record.RecordCoreArgumentException;
 import com.apple.foundationdb.record.RecordCoreException;
-import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
 import com.apple.foundationdb.record.logging.LogMessageKeys;
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
@@ -232,7 +232,7 @@ public boolean equalsAtomic(AtomKeyExpression other) {
      *   implementation, or the arguments provided are not suitable for the function.
      */
     @Nonnull
-    public static FunctionKeyExpression fromProto(RecordMetaDataProto.Function function) throws DeserializationException {
+    public static FunctionKeyExpression fromProto(RecordKeyExpressionProto.Function function) throws DeserializationException {
         try {
             return create(function.getName(), KeyExpression.fromProto(function.getArguments()));
         } catch (RecordCoreException e) {
@@ -242,17 +242,17 @@ public static FunctionKeyExpression fromProto(RecordMetaDataProto.Function funct
 
     @Nonnull
     @Override
-    public final RecordMetaDataProto.Function toProto() throws SerializationException {
-        RecordMetaDataProto.Function.Builder builder = RecordMetaDataProto.Function.newBuilder()
+    public final RecordKeyExpressionProto.Function toProto() throws SerializationException {
+        RecordKeyExpressionProto.Function.Builder builder = RecordKeyExpressionProto.Function.newBuilder()
                 .setName(getName());
         builder.setArguments(getArguments().toKeyExpression());
         return builder.build();
     }
 
     @Nonnull
     @Override
-    public final RecordMetaDataProto.KeyExpression toKeyExpression() {
-        return RecordMetaDataProto.KeyExpression.newBuilder().setFunction(toProto()).build();
+    public final RecordKeyExpressionProto.KeyExpression toKeyExpression() {
+        return RecordKeyExpressionProto.KeyExpression.newBuilder().setFunction(toProto()).build();
     }
 
     @Nonnull

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java

@@ -23,7 +23,7 @@
 import com.apple.foundationdb.annotation.API;
 import com.apple.foundationdb.record.ObjectPlanHash;
 import com.apple.foundationdb.record.PlanHashable;
-import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
 import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
@@ -53,7 +53,7 @@ public GroupingKeyExpression(@Nonnull KeyExpression wholeKey, int groupedCount)
         this.groupedCount = groupedCount;
     }
 
-    public GroupingKeyExpression(@Nonnull RecordMetaDataProto.Grouping grouping) throws DeserializationException {
+    public GroupingKeyExpression(@Nonnull RecordKeyExpressionProto.Grouping grouping) throws DeserializationException {
         this(KeyExpression.fromProto(grouping.getWholeKey()), grouping.getGroupedCount());
     }
 
@@ -100,17 +100,17 @@ public boolean needsCopyingToPartialRecord() {
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.Grouping toProto() throws SerializationException {
-        final RecordMetaDataProto.Grouping.Builder builder = RecordMetaDataProto.Grouping.newBuilder();
+    public RecordKeyExpressionProto.Grouping toProto() throws SerializationException {
+        final RecordKeyExpressionProto.Grouping.Builder builder = RecordKeyExpressionProto.Grouping.newBuilder();
         builder.setWholeKey(getWholeKey().toKeyExpression());
         builder.setGroupedCount(groupedCount);
         return builder.build();
     }
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.KeyExpression toKeyExpression() {
-        return RecordMetaDataProto.KeyExpression.newBuilder().setGrouping(toProto()).build();
+    public RecordKeyExpressionProto.KeyExpression toKeyExpression() {
+        return RecordKeyExpressionProto.KeyExpression.newBuilder().setGrouping(toProto()).build();
     }
 
     @Nonnull