Merge remote-tracking branch 'upstream/main' into uuid_support

Author: g31pranjal

.github/workflows/nightly.yml

@@ -3,6 +3,7 @@ name: Nightly
 on:
   schedule:
     - cron: "0 7 * * 1-5"
+  workflow_dispatch:
 
 jobs:
   gradle:

build.gradle

@@ -279,6 +279,7 @@ subprojects {
                 sourceSets["test"].resources.srcDirs.stream().flatMap( resourceDir ->
                     fileTree(dir: resourceDir).matching {
                         include '**/*.yamsql'
+                        exclude '/initial-version/*'
                         exclude '/supported-version/*'
                     }.stream()
                 ).forEach { yamsql ->

docs/sphinx/source/ReleaseNotes.md

@@ -15,6 +15,70 @@ Builds and releases have been moved to a new CI system. This includes the resump
 
 Users performing online updates are encouraged to update from [4.0.559.4](#405594). The continuations of some queries have changed in ways that may break if continued on other 4.0 builds. See: [Issue #3093](https://github.com/FoundationDB/fdb-record-layer/issues/3093), [PR #3092](https://github.com/FoundationDB/fdb-record-layer/pull/3092) fixing the issue, and [PR #3108](https://github.com/FoundationDB/fdb-record-layer/issues/3108) preparing 4.0.559.4 to accept newer continuations.
 
+### 4.1.9.0
+
+<h4> New Features </h4>
+
+* Introduce a new `Option` to set `asyncToSync` timeout. - [PR #3205](https://github.com/FoundationDB/fdb-record-layer/pull/3205)
+* `RecordQueryPlanComplexityException` now includes only a shortened plan string as a log message key - [PR #3168](https://github.com/FoundationDB/fdb-record-layer/pull/3168)
+<h4> Bug Fixes </h4>
+
+* Fix deserialization of continuations for queries that contain LIKE - [PR #3207](https://github.com/FoundationDB/fdb-record-layer/pull/3207)
+* `EXISTS` queries now honor continuation from previous runs to avoid infinite loops - [PR #3221](https://github.com/FoundationDB/fdb-record-layer/pull/3221)
+* `RecordQueryExplodePlan` plan adheres to `skip` and `limit` requirements - [PR #3230](https://github.com/FoundationDB/fdb-record-layer/pull/3230)
+* Prevent caching `INSERT` statements - [PR #3212](https://github.com/FoundationDB/fdb-record-layer/pull/3212)
+* Fix unstable hash for NULL literal - [PR #3222](https://github.com/FoundationDB/fdb-record-layer/pull/3222)
+* Fix continuation deserialization errors by deserializing streaming aggregate plans in the correct order - [PR #3215](https://github.com/FoundationDB/fdb-record-layer/pull/3215)
+* Serialize missing field `createDefaultIfEmpty` into streaming aggregate plan continuations - [PR #3211](https://github.com/FoundationDB/fdb-record-layer/pull/3211)
+* Fix instability in the query planner - [PR #3185](https://github.com/FoundationDB/fdb-record-layer/pull/3185)
+* Adjust serialization of `VersionValue` to make continuations compatible with versions older than 4.0.564.0 - [PR #3179](https://github.com/FoundationDB/fdb-record-layer/pull/3179)
+* Fixes off-by-one error that could result in streaming aggregate plans returning incorrect results when resumed from a continuation - [PR #3112](https://github.com/FoundationDB/fdb-record-layer/pull/3112)
+<h4> Dependency Updates </h4>
+
+* Upgrade protobuf plugin from 0.8.19 to 0.9.4 - [PR #3101](https://github.com/FoundationDB/fdb-record-layer/pull/3101)
+
+<details>
+<summary>
+
+<h4> Build/Test/Documentation Improvements (click to expand) </h4>
+
+</summary>
+
+* Enable force continuations mode on all `YamlIntegrationTest`s - [PR #3228](https://github.com/FoundationDB/fdb-record-layer/pull/3228)
+* Write PMD output to consoleOutput during build - [PR #3223](https://github.com/FoundationDB/fdb-record-layer/pull/3223)
+* Create a yaml test config that runs two servers with the same version - [PR #3217](https://github.com/FoundationDB/fdb-record-layer/pull/3217)
+* Test concurrent indexing sessions without sync lock - [PR #3200](https://github.com/FoundationDB/fdb-record-layer/pull/3200)
+* Yaml tests: Only allow forced continuations on SELECT queries - [PR #3201](https://github.com/FoundationDB/fdb-record-layer/pull/3201)
+* Adds commands to the YAML framework to assert on changing query behavior - [PR #3188](https://github.com/FoundationDB/fdb-record-layer/pull/3188)
+* Run nightly variations of yaml-tests in nightly workflow - [PR #3202](https://github.com/FoundationDB/fdb-record-layer/pull/3202)
+* Run the PR label check whenever the HEAD commit changes to ensure it is applied to the final commit of each PR - [PR #3195](https://github.com/FoundationDB/fdb-record-layer/pull/3195)
+* Change netty logs to INFO when running tests - [PR #3196](https://github.com/FoundationDB/fdb-record-layer/pull/3196)
+* Make prepared.yamsql run against 4.0.559.6 as the earliest - [PR #3187](https://github.com/FoundationDB/fdb-record-layer/pull/3187)
+* Add protection against test running into an infinite loop - [PR #3186](https://github.com/FoundationDB/fdb-record-layer/pull/3186)
+* Add version information to yaml test exceptions - [PR #3189](https://github.com/FoundationDB/fdb-record-layer/pull/3189)
+* Generate release notes automatically from pull requests - [PR #3174](https://github.com/FoundationDB/fdb-record-layer/pull/3174)
+* Fix link to documentation in README - [PR #3184](https://github.com/FoundationDB/fdb-record-layer/pull/3184)
+* Yaml tests: Add protection against infinite continuation loops - [PR #3182](https://github.com/FoundationDB/fdb-record-layer/pull/3182)
+* Enable a few tests where the force continuation option does not fail anymore - [PR #3176](https://github.com/FoundationDB/fdb-record-layer/pull/3176)
+* Improve multi-server connection testing - [PR #3170](https://github.com/FoundationDB/fdb-record-layer/pull/3170)
+* Enforce all PRs are labeled for Release Note generation - [PR #3181](https://github.com/FoundationDB/fdb-record-layer/pull/3181)
+* Cache FDB packages during the build and avoid checking them in - [PR #3164](https://github.com/FoundationDB/fdb-record-layer/pull/3164)
+* Switch documentation site to sphinx-based solution - [PR #3078](https://github.com/FoundationDB/fdb-record-layer/pull/3078)
+
+</details>
+
+
+**[Full Changelog (4.1.8.0...4.1.9.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.1.8.0...4.1.9.0)**
+
+#### Mixed Mode Test Results
+
+Mixed mode testing run against the following previous versions:
+
+❌`4.0.559.1`, ❌`4.0.559.2`, ❌`4.0.559.3`, ✅`4.0.559.4`, ✅`4.0.559.6`, ❌`4.0.561.0`, ❌`4.0.562.0`, ❌`4.0.564.0`, ❌`4.0.565.0`, ❌`4.0.566.0`, ❌`4.0.567.0`, ❌`4.0.568.0`, ❌`4.0.569.0`, ❌`4.0.570.0`, ❌`4.0.571.0`, ❌`4.0.572.0`, ❌`4.0.573.0`, ❌`4.0.574.0`, ❌`4.0.575.0`, ❌`4.1.4.0`, ✅`4.1.5.0`, ✅`4.1.6.0`, ✅`4.1.8.0`
+
+[See full test run](https://github.com/FoundationDB/fdb-record-layer/actions/runs/13729472581)
+
+
 ### 4.1.8.0
 
 <h4> New Features </h4>

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordCursor.java

@@ -59,6 +59,7 @@
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
@@ -884,24 +885,57 @@ static <T> RecordCursor<T> fromFuture(@Nonnull CompletableFuture<T> future) {
 
     @Nonnull
     static <T> RecordCursor<T> fromFuture(@Nonnull Executor executor, @Nonnull CompletableFuture<T> future) {
-        return new FutureCursor<>(executor, future);
+        return fromFuture(executor, future, null);
     }
 
     /**
      * Get a new cursor that has the contents of the given future as its only record.
      * If the continuation is nonnull, return an empty cursor since a {@link FutureCursor} returns only a single record.
+     * Note that in general, this can lead to futures being started that are not properly waited on,
+     * so it is generally advised that the user use {@link #fromFuture(Executor, Supplier, byte[])} instead
+     * of this one. See also the warning on {@link #fromFuture(Executor, Supplier, byte[])} for a word
+     * about if the work powering the underlying future has more complicated state that should be
+     * included in the cursor's continuation.
+     *
      * @param executor an executor for executing the future
      * @param future a future that completes to the only element of the cursor
      * @param continuation a continuation from a future cursor to determine whether the cursor has a single element or no elements
      * @param <T> the result type of the future
      * @return a new cursor producing the contents of {@code future} if the continuation is nonnull and an empty cursor otherwise
+     * @see #fromFuture(Executor, Supplier, byte[])
      */
     @Nonnull
     static <T> RecordCursor<T> fromFuture(@Nonnull Executor executor, @Nonnull CompletableFuture<T> future, @Nullable byte[] continuation) {
+        return fromFuture(executor, () -> future, continuation);
+    }
+
+    /**
+     * Get a new cursor that has the contents of the lazily-evaluated future as its only record.
+     * If the continuation is nonnull, return an empty cursor since a {@link FutureCursor} returns only a single record.
+     * In that case, the {@code futureSupplier} will not be evaluated to avoid queuing up asynchronous work
+     * that is not guaranteed to have anyone wait on. For that reason, this should generally be preferred
+     * to {@link #fromFuture(Executor, CompletableFuture, byte[])}.
+     *
+     * <p>
+     * <em>Warning:</em> Using a {@link FutureCursor} here may lead to incorrect results if the underlying
+     * future is backed by something that has some kind of state that should be included in the continuation.
+     * For example, if the future is backed by a cursor that may stop for an out-of-band {@link NoNextReason},
+     * it may be desirable to include the progress included in the underlying cursor in the final continuation.
+     * That is not done by this class, so something more bespoke may be required.
+     * </p>
+     *
+     * @param executor an executor for executing the future
+     * @param futureSupplier a supplier of a future that completes to the only element of the cursor
+     * @param continuation a continuation from a future cursor to determine whether the cursor has a single element or no elements
+     * @param <T> the result type of the future
+     * @return a new cursor producing the contents of {@code future} if the continuation is nonnull and an empty cursor otherwise
+     */
+    @Nonnull
+    static <T> RecordCursor<T> fromFuture(@Nonnull Executor executor, @Nonnull Supplier<CompletableFuture<T>> futureSupplier, @Nullable byte[] continuation) {
         if (continuation == null) {
-            return new FutureCursor<>(executor, future);
+            return new FutureCursor<>(executor, futureSupplier.get());
         } else {
-            return RecordCursor.empty();
+            return RecordCursor.empty(executor);
         }
     }
 

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/cursors/FutureCursor.java

@@ -50,6 +50,15 @@ public class FutureCursor<T> implements RecordCursor<T> {
     @Nonnull
     private static final RecordCursorContinuation notDoneContinuation = ByteArrayContinuation.fromNullable(new byte[] {0});
 
+    /**
+     * Internal constructor. Users should generally call
+     * {@link RecordCursor#fromFuture(Executor, java.util.function.Supplier, byte[])}
+     * in order to ensure that continuations from this cursor are properly handled.
+     *
+     * @param executor the executor to associate with this cursor
+     * @param future a future that when completed will provide the single element returned by this cursor
+     */
+    @API(API.Status.INTERNAL)
     public FutureCursor(@Nonnull Executor executor, @Nonnull CompletableFuture<T> future) {
         this.executor = executor;
         this.future = future;

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java

@@ -1138,6 +1138,16 @@ public Null fromProto(@Nonnull final PlanSerializationContext serializationConte
                 return Null.fromProto(serializationContext, nullTypeProto);
             }
         }
+
+        @Override
+        public int hashCode() {
+            return getTypeCode().name().hashCode();
+        }
+
+        @Override
+        public boolean equals(final Object other) {
+            return other instanceof Null;
+        }
     }
 
     /**
@@ -1227,6 +1237,16 @@ public None fromProto(@Nonnull final PlanSerializationContext serializationConte
                 return None.fromProto(serializationContext, noneTypeProto);
             }
         }
+
+        @Override
+        public int hashCode() {
+            return getTypeCode().name().hashCode();
+        }
+
+        @Override
+        public boolean equals(final Object other) {
+            return other instanceof None;
+        }
     }
 
     /**

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/PatternForLikeValue.java

@@ -188,7 +188,7 @@ public PValue toValueProto(@Nonnull final PlanSerializationContext serialization
     public static PatternForLikeValue fromProto(@Nonnull final PlanSerializationContext serializationContext,
                                                 @Nonnull final PPatternForLikeValue patternForLikeValueProto) {
         return new PatternForLikeValue(Value.fromValueProto(serializationContext, Objects.requireNonNull(patternForLikeValueProto.getPatternChild())),
-                Value.fromValueProto(serializationContext, Objects.requireNonNull(patternForLikeValueProto.getPatternChild())));
+                Value.fromValueProto(serializationContext, Objects.requireNonNull(patternForLikeValueProto.getEscapeChild())));
     }
 
     @Nonnull

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java

@@ -88,7 +88,8 @@ public <M extends Message> RecordCursor<QueryResult> executePlan(@Nonnull final
                                                                      @Nonnull final ExecuteProperties executeProperties) {
         final var result = collectionValue.eval(store, context);
         return RecordCursor.fromList(result == null ? ImmutableList.of() : (List<?>)result, continuation)
-                .map(QueryResult::ofComputed);
+                .map(QueryResult::ofComputed)
+                .skipThenLimit(executeProperties.getSkip(), executeProperties.getReturnedRowLimit());
     }
 
     @Nonnull

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryFirstOrDefaultPlan.java

@@ -28,12 +28,10 @@
 import com.apple.foundationdb.record.PlanHashable;
 import com.apple.foundationdb.record.PlanSerializationContext;
 import com.apple.foundationdb.record.RecordCursor;
-import com.apple.foundationdb.record.cursors.FutureCursor;
 import com.apple.foundationdb.record.planprotos.PRecordQueryFirstOrDefaultPlan;
 import com.apple.foundationdb.record.planprotos.PRecordQueryPlan;
 import com.apple.foundationdb.record.provider.common.StoreTimer;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase;
-import com.apple.foundationdb.record.query.plan.explain.ExplainPlanVisitor;
 import com.apple.foundationdb.record.query.plan.cascades.AliasMap;
 import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier;
 import com.apple.foundationdb.record.query.plan.cascades.Memoizer;
@@ -47,6 +45,7 @@
 import com.apple.foundationdb.record.query.plan.cascades.values.DerivedValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
 import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap;
+import com.apple.foundationdb.record.query.plan.explain.ExplainPlanVisitor;
 import com.google.auto.service.AutoService;
 import com.google.common.base.Verify;
 import com.google.common.collect.ImmutableList;
@@ -96,10 +95,17 @@ public <M extends Message> RecordCursor<QueryResult> executePlan(@Nonnull final
                                                                      @Nonnull final EvaluationContext context,
                                                                      @Nullable final byte[] continuation,
                                                                      @Nonnull final ExecuteProperties executeProperties) {
-        return new FutureCursor<>(store.getExecutor(),
-                getChild().executePlan(store, context, continuation, executeProperties).first()
-                        .thenApply(resultOptional ->
-                                resultOptional.orElseGet(() -> QueryResult.ofComputed(onEmptyResultValue.eval(store, context)))));
+        // Note that a null child continuation is always handed to the child cursor below.
+        // This is because the returned FutureCursor only ever returns a single value, and so if
+        // if that lambda is called, it indicates that the original continuation is null, and we
+        // are starting the plan from the beginning. That this doesn't handle the inner cursor
+        // halting with an out-of-band no-next-reasons, which currently is treated the same way
+        // as the inner cursor being empty.
+        // See: https://github.com/FoundationDB/fdb-record-layer/issues/3220
+        return RecordCursor.fromFuture(store.getExecutor(),
+                () -> getChild().executePlan(store, context, null, executeProperties).first()
+                        .thenApply(resultOptional -> resultOptional.orElseGet(() -> QueryResult.ofComputed(onEmptyResultValue.eval(store, context)))),
+                continuation);
     }
 
     @Override