Skip to content

Commit 6c0597e

Browse files
authored
Implementation of grouping size aggregator (#3577)
This implementation is creating a flavor of the `SizeStatisticsCollectorCursor` that can return sizes aggregated by sub-subspaces. This relies on the original implementation but had to go into a separate class as the generic type of the cursor has changed. Eventually, the original cursor can be replaced by this one with the aggregation size being "0". For backwards compatibility, the original class was marked as "deprecated". The original `SizeStatisticsResults` was marked as deprecated and a new (non-embedded version) of it was added. Initially, I thought that we can make a backwards-compatible change to the `SizeStatisticsCollectorCursor` class, but that proved more complicated with the changes to the result type, so I decided to go with a new implementation instead. Resolves #3576
1 parent 7119e7e commit 6c0597e

File tree

8 files changed

+1706
-1
lines changed

8 files changed

+1706
-1
lines changed

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/cursors/SizeStatisticsCollectorCursor.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,9 @@
6565
* for all keys and values in the specified subspace. Any cursor results emitted prior to that are a manifestation of
6666
* hitting an execution limit while scanning the subspace. These results provide a continuation that can be used
6767
* to resume aggregating the remaining keys and values of the subspace.
68+
* Deprecated, use the {@link SizeStatisticsGroupingCursor} instead.
6869
*/
69-
@API(API.Status.EXPERIMENTAL)
70+
@API(API.Status.DEPRECATED)
7071
public class SizeStatisticsCollectorCursor implements RecordCursor<SizeStatisticsCollectorCursor.SizeStatisticsResults> {
7172
@Nonnull
7273
private final SubspaceProvider subspaceProvider;
@@ -318,7 +319,9 @@ public static SizeStatisticsCollectorCursor ofSubspace(@Nonnull Subspace subspac
318319

319320
/**
320321
* Encapsulates the distribution statistics returned by a SizeStatisticsCollectorCursor.
322+
* This is deprecated. Use the {@link com.apple.foundationdb.record.provider.foundationdb.cursors.SizeStatisticsResults} instead
321323
*/
324+
@API(API.Status.DEPRECATED)
322325
public static class SizeStatisticsResults {
323326
private long keyCount;
324327
private long keySize;
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* SizeStatisticsGroupedResults.java
3+
*
4+
* This source file is part of the FoundationDB open source project
5+
*
6+
* Copyright 2015-2025 Apple Inc. and the FoundationDB project authors
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
package com.apple.foundationdb.record.provider.foundationdb.cursors;
22+
23+
import com.apple.foundationdb.annotation.API;
24+
import com.apple.foundationdb.tuple.Tuple;
25+
26+
import javax.annotation.Nonnull;
27+
28+
/**
29+
* The result of a grouped size calculation.
30+
* The result is aggregated for a subspace whose key suffix is specified below, within the containing subspace.
31+
*/
32+
@API(API.Status.EXPERIMENTAL)
33+
public class SizeStatisticsGroupedResults {
34+
private final Tuple aggregationKey;
35+
private final SizeStatisticsResults stats;
36+
37+
public SizeStatisticsGroupedResults(@Nonnull final Tuple aggregationKey, @Nonnull SizeStatisticsResults stats) {
38+
this.aggregationKey = aggregationKey;
39+
this.stats = stats;
40+
}
41+
42+
/**
43+
* The Tuple that represents the key whose stats are aggregated.
44+
* @return the Tuple representing the subspace key aggregated
45+
*/
46+
@Nonnull
47+
public Tuple getAggregationKey() {
48+
return aggregationKey;
49+
}
50+
51+
/**
52+
* The aggregated stats.
53+
* @return the statistics collected for the subspace
54+
*/
55+
@Nonnull
56+
public SizeStatisticsResults getStats() {
57+
return stats;
58+
}
59+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
/*
2+
* SizeStatisticsGroupingContinuation.java
3+
*
4+
* This source file is part of the FoundationDB open source project
5+
*
6+
* Copyright 2015-2025 Apple Inc. and the FoundationDB project authors
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
package com.apple.foundationdb.record.provider.foundationdb.cursors;
22+
23+
import com.apple.foundationdb.KeyValue;
24+
import com.apple.foundationdb.record.RecordCursorContinuation;
25+
import com.apple.foundationdb.record.RecordCursorEndContinuation;
26+
import com.apple.foundationdb.record.RecordCursorProto;
27+
import com.apple.foundationdb.record.RecordCursorResult;
28+
import com.apple.foundationdb.tuple.Tuple;
29+
import com.google.protobuf.ByteString;
30+
31+
import javax.annotation.Nonnull;
32+
import javax.annotation.Nullable;
33+
34+
/**
35+
* A continuation for the {@link SizeStatisticsGroupingCursor}.
36+
*/
37+
class SizeStatisticsGroupingContinuation implements RecordCursorContinuation {
38+
/**
39+
* A continuation for the case where we have one last result to send, after which the cursor would be done.
40+
*/
41+
public static final SizeStatisticsGroupingContinuation LAST_RESULT_CONTINUATION = new SizeStatisticsGroupingContinuation();
42+
43+
/** Whether the continuation is a marker for the last result being sent before the cursor is done. */
44+
private final boolean lastResultContinuation;
45+
@Nullable
46+
private RecordCursorContinuation innerContinuation;
47+
@Nullable
48+
private SizeStatisticsResults partialResults;
49+
private Tuple currentGroupingKey;
50+
@Nullable
51+
private byte[] cachedBytes;
52+
@Nullable
53+
private ByteString cachedByteString;
54+
55+
/**
56+
* The inner cursor is done, and we sent the last result, cannot continue afterward.
57+
*/
58+
private SizeStatisticsGroupingContinuation() {
59+
lastResultContinuation = true;
60+
}
61+
62+
/**
63+
* The inner cursor still has results, we can continue (e.g. group break).
64+
*/
65+
SizeStatisticsGroupingContinuation(@Nonnull RecordCursorResult<KeyValue> currentKvResult,
66+
@Nonnull SizeStatisticsResults partialResults,
67+
@Nonnull Tuple currentGroupingKey) {
68+
lastResultContinuation = false;
69+
this.innerContinuation = currentKvResult.getContinuation();
70+
this.partialResults = partialResults.copy(); //cache an immutable snapshot of the partial aggregate state
71+
this.currentGroupingKey = currentGroupingKey;
72+
}
73+
74+
/**
75+
* Whether the given protobuf is the same as the "LAST_RESULT_CONTINUATION".
76+
* @param statsContinuation the incoming protobuf
77+
* @return TRUE if the protobuf marks a continuation with last result
78+
*/
79+
public static boolean isLastResultContinuation(final RecordCursorProto.SizeStatisticsGroupingContinuation statsContinuation) {
80+
// Default to false if unset
81+
return (statsContinuation.getLastResultContinuation());
82+
}
83+
84+
public boolean isLastResultContinuation() {
85+
return lastResultContinuation;
86+
}
87+
88+
@Nullable
89+
@Override
90+
public byte[] toBytes() {
91+
// form bytes exactly once
92+
if (this.cachedBytes == null) {
93+
this.cachedBytes = toByteString().toByteArray();
94+
}
95+
return cachedBytes;
96+
}
97+
98+
@Override
99+
@Nonnull
100+
public ByteString toByteString() {
101+
if (cachedByteString == null) {
102+
final RecordCursorProto.SizeStatisticsGroupingContinuation.Builder builder = RecordCursorProto.SizeStatisticsGroupingContinuation.newBuilder();
103+
builder.setLastResultContinuation(lastResultContinuation);
104+
if (innerContinuation != null) {
105+
builder.setUnderlyingContinuation(innerContinuation.toByteString());
106+
}
107+
if (partialResults != null) {
108+
builder.setPartialResults(partialResults.toProto());
109+
}
110+
if (currentGroupingKey != null) {
111+
builder.setCurrentGroupingKey(ByteString.copyFrom(currentGroupingKey.pack()));
112+
}
113+
cachedByteString = builder.build().toByteString();
114+
}
115+
return cachedByteString;
116+
}
117+
118+
/**
119+
* Whether this continuation is the end of the stream.
120+
* This is always false, since the {@link RecordCursorEndContinuation#END} would be used once the cursor is exhausted.
121+
* @return TRUE if the cursor is exhausted
122+
*/
123+
@Override
124+
public boolean isEnd() {
125+
return false;
126+
}
127+
}

0 commit comments

Comments
 (0)