Skip to content

Commit 8ed4400

Browse files
stevenschlanskerstevenschlansker
committed
feat(java): introduce new 'compact' row encoding
compact encoding will store fixed sized fields inline, relaxes alignment considerations to preserve alignment where possible but using space more effectively
1 parent 80784a7 commit 8ed4400

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+3719
-674
lines changed

‎java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,8 @@ public CodecBuilder(CodegenContext ctx, TypeRef<?> beanType) {
122122
.forEach(ctx::reserveName);
123123
}
124124

125+
public abstract String codecClassName(Class<?> cls);
126+
125127
/** Generate codec class code. */
126128
public abstract String genCode();
127129

‎java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
* <li>additional binary compare, swap, and copy methods.
4141
* <li>little-endian access.
4242
* <li>independent read/write index.
43-
* <li>variant int/long encoding.
43+
* <li>varint int/long encoding.
4444
* <li>aligned int/long encoding.
4545
* </ul>
4646
*

‎java/fory-core/src/main/java/org/apache/fory/util/ExceptionUtils.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,4 +71,13 @@ public static RuntimeException handleReadFailed(Fory fory, Throwable t) {
7171
}
7272

7373
public static void ignore(Object... args) {}
74+
75+
public static RuntimeException throwAnyway(Throwable t) {
76+
throw ExceptionUtils.<RuntimeException>throwEvadingChecks(t);
77+
}
78+
79+
@SuppressWarnings({"unchecked", "TypeParameterUnusedInFormals"})
80+
private static <E extends Throwable> E throwEvadingChecks(Throwable throwable) throws E {
81+
throw (E) throwable;
82+
}
7483
}

‎java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayCodecBuilder.java‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.fory.format.encoder;
21+
22+
import static org.apache.fory.type.TypeUtils.getRawType;
23+
24+
import java.lang.invoke.MethodHandle;
25+
import java.lang.invoke.MethodHandles;
26+
import java.lang.invoke.MethodType;
27+
import java.util.Collection;
28+
import java.util.HashSet;
29+
import java.util.Set;
30+
import java.util.function.Function;
31+
import java.util.function.Supplier;
32+
import org.apache.arrow.vector.types.pojo.Field;
33+
import org.apache.fory.format.row.binary.writer.BinaryArrayWriter;
34+
import org.apache.fory.format.type.DataTypes;
35+
import org.apache.fory.format.type.TypeInference;
36+
import org.apache.fory.reflect.TypeRef;
37+
import org.apache.fory.type.TypeUtils;
38+
import org.apache.fory.util.ExceptionUtils;
39+
40+
public class ArrayCodecBuilder<C extends Collection<?>>
41+
extends BaseCodecBuilder<ArrayCodecBuilder<C>> {
42+
43+
private final TypeRef<C> collectionType;
44+
private final Field elementField;
45+
46+
ArrayCodecBuilder(final TypeRef<C> collectionType) {
47+
super(TypeInference.inferSchema(collectionType, false));
48+
this.collectionType = collectionType;
49+
elementField = DataTypes.fieldOfSchema(schema, 0);
50+
}
51+
52+
public Supplier<ArrayEncoder<C>> build() {
53+
final Function<BinaryArrayWriter, ArrayEncoder<C>> arrayEncoderFactory = buildWithWriter();
54+
return new Supplier<ArrayEncoder<C>>() {
55+
@Override
56+
public ArrayEncoder<C> get() {
57+
final BinaryArrayWriter writer = codecFormat.newArrayWriter(elementField);
58+
return new BufferResettingArrayEncoder<>(
59+
initialBufferSize, writer, arrayEncoderFactory.apply(writer));
60+
}
61+
};
62+
}
63+
64+
Function<BinaryArrayWriter, ArrayEncoder<C>> buildWithWriter() {
65+
loadArrayInnerCodecs();
66+
final Function<BinaryArrayWriter, GeneratedArrayEncoder> generatedEncoderFactory =
67+
generatedEncoderFactory();
68+
return new Function<BinaryArrayWriter, ArrayEncoder<C>>() {
69+
@Override
70+
public ArrayEncoder<C> apply(final BinaryArrayWriter writer) {
71+
return new BaseArrayEncoder<>(writer, generatedEncoderFactory.apply(writer));
72+
}
73+
};
74+
}
75+
76+
private void loadArrayInnerCodecs() {
77+
final Set<TypeRef<?>> set = new HashSet<>();
78+
Encoders.findBeanToken(collectionType, set);
79+
if (set.isEmpty()) {
80+
throw new IllegalArgumentException("can not find bean class.");
81+
}
82+
83+
for (final TypeRef<?> tt : set) {
84+
Encoders.loadOrGenRowCodecClass(getRawType(tt), codecFormat);
85+
}
86+
}
87+
88+
Function<BinaryArrayWriter, GeneratedArrayEncoder> generatedEncoderFactory() {
89+
final TypeRef<?> elementType = TypeUtils.getElementType(collectionType);
90+
final Class<?> arrayCodecClass =
91+
Encoders.loadOrGenArrayCodecClass(collectionType, elementType, codecFormat);
92+
93+
final MethodHandle constructorHandle;
94+
try {
95+
final var constructor =
96+
arrayCodecClass.asSubclass(GeneratedArrayEncoder.class).getConstructor(Object[].class);
97+
constructorHandle =
98+
MethodHandles.lookup()
99+
.unreflectConstructor(constructor)
100+
.asType(MethodType.methodType(GeneratedArrayEncoder.class, Object[].class));
101+
} catch (final NoSuchMethodException | IllegalAccessException e) {
102+
throw new EncoderException(
103+
"Failed to construct array codec for "
104+
+ collectionType
105+
+ " with element class "
106+
+ elementType,
107+
e);
108+
}
109+
return new Function<BinaryArrayWriter, GeneratedArrayEncoder>() {
110+
@Override
111+
public GeneratedArrayEncoder apply(final BinaryArrayWriter writer) {
112+
final Object[] references = {writer.getField(), writer, fory};
113+
try {
114+
return (GeneratedArrayEncoder) constructorHandle.invokeExact(references);
115+
} catch (final Throwable t) {
116+
throw ExceptionUtils.throwAnyway(t);
117+
}
118+
}
119+
};
120+
}
121+
}

‎java/fory-format/src/main/java/org/apache/fory/format/encoder/ArrayEncoderBuilder.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import org.apache.fory.codegen.Expression;
3232
import org.apache.fory.codegen.ExpressionUtils;
3333
import org.apache.fory.format.row.binary.BinaryArray;
34-
import org.apache.fory.format.row.binary.writer.BinaryArrayWriter;
3534
import org.apache.fory.format.type.TypeInference;
3635
import org.apache.fory.logging.Logger;
3736
import org.apache.fory.logging.LoggerFactory;
@@ -104,13 +103,13 @@ public String genCode() {
104103
"arrayWriter",
105104
ROOT_ARRAY_WRITER_NAME,
106105
"arrayWriterType",
107-
ctx.type(BinaryArrayWriter.class),
106+
arrayWriterType(),
108107
"fory",
109108
FORY_NAME,
110109
"foryType",
111110
ctx.type(Fory.class));
112111
ctx.addField(ctx.type(Field.class), FIELD_NAME);
113-
ctx.addField(ctx.type(BinaryArrayWriter.class), ROOT_ARRAY_WRITER_NAME);
112+
ctx.addField(ctx.type(arrayWriterType()), ROOT_ARRAY_WRITER_NAME);
114113
ctx.addField(ctx.type(Fory.class), FORY_NAME);
115114

116115
Expression encodeExpr = buildEncodeExpression();
@@ -136,7 +135,7 @@ public String genCode() {
136135
@Override
137136
public Expression buildEncodeExpression() {
138137
Expression.Reference arrayWriter =
139-
new Expression.Reference(ROOT_ARRAY_WRITER_NAME, arrayWriterTypeToken, false);
138+
new Expression.Reference(ROOT_ARRAY_WRITER_NAME, arrayWriterType(), false);
140139
Expression.ListExpression expressions = new Expression.ListExpression();
141140

142141
Expression.Reference inputObject =
@@ -148,7 +147,7 @@ public Expression buildEncodeExpression() {
148147

149148
Expression.Reference fieldExpr = new Expression.Reference(FIELD_NAME, ARROW_FIELD_TYPE, false);
150149
Expression listExpression =
151-
serializeForArrayByWriter(array, arrayWriter, arrayToken, fieldExpr);
150+
serializeForArrayByWriter(array, arrayWriter, arrayToken, null, fieldExpr);
152151

153152
expressions.add(listExpression);
154153

‎java/fory-format/src/main/java/org/apache/fory/format/encoder/BaseArrayEncoder.java‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.fory.format.encoder;
21+
22+
import org.apache.arrow.vector.types.pojo.Field;
23+
import org.apache.fory.format.row.binary.BinaryArray;
24+
import org.apache.fory.format.row.binary.writer.BinaryArrayWriter;
25+
import org.apache.fory.memory.MemoryBuffer;
26+
import org.apache.fory.memory.MemoryUtils;
27+
28+
class BaseArrayEncoder<T> implements ArrayEncoder<T> {
29+
private final BinaryArrayWriter writer;
30+
private final GeneratedArrayEncoder codec;
31+
32+
BaseArrayEncoder(final BinaryArrayWriter writer, final GeneratedArrayEncoder codec) {
33+
this.writer = writer;
34+
this.codec = codec;
35+
}
36+
37+
@Override
38+
public Field field() {
39+
return writer.getField();
40+
}
41+
42+
@SuppressWarnings("unchecked")
43+
@Override
44+
public T fromArray(final BinaryArray array) {
45+
return (T) codec.fromArray(array);
46+
}
47+
48+
@Override
49+
public BinaryArray toArray(final T obj) {
50+
return codec.toArray(obj);
51+
}
52+
53+
@Override
54+
public T decode(final MemoryBuffer buffer) {
55+
return decode(buffer, buffer.readInt32());
56+
}
57+
58+
@Override
59+
public T decode(final byte[] bytes) {
60+
return decode(MemoryUtils.wrap(bytes), bytes.length);
61+
}
62+
63+
T decode(final MemoryBuffer buffer, final int size) {
64+
final BinaryArray array = writer.newArray();
65+
final int readerIndex = buffer.readerIndex();
66+
array.pointTo(buffer, readerIndex, size);
67+
buffer.readerIndex(readerIndex + size);
68+
return fromArray(array);
69+
}
70+
71+
@Override
72+
public T decodeRemaining(final MemoryBuffer buffer) {
73+
return decode(buffer, buffer.remaining());
74+
}
75+
76+
@Override
77+
public byte[] encode(final T obj) {
78+
final BinaryArray array = toArray(obj);
79+
return writer.getBuffer().getBytes(0, array.getSizeInBytes());
80+
}
81+
82+
@Override
83+
public void encode(final MemoryBuffer buffer, final T obj) {
84+
final MemoryBuffer prevBuffer = writer.getBuffer();
85+
final int writerIndex = buffer.writerIndex();
86+
buffer.writeInt32(-1);
87+
try {
88+
writer.setBuffer(buffer);
89+
final BinaryArray array = toArray(obj);
90+
final int size = buffer.writerIndex() - writerIndex - 4;
91+
assert size == array.getSizeInBytes();
92+
buffer.putInt32(writerIndex, size);
93+
} finally {
94+
writer.setBuffer(prevBuffer);
95+
}
96+
}
97+
98+
@Override
99+
public int bareEncode(final MemoryBuffer buffer, final T obj) {
100+
final MemoryBuffer prevBuffer = writer.getBuffer();
101+
final int writerIndex = buffer.writerIndex();
102+
try {
103+
writer.setBuffer(buffer);
104+
final BinaryArray array = toArray(obj);
105+
final int size = buffer.writerIndex() - writerIndex;
106+
assert size == array.getSizeInBytes();
107+
return size;
108+
} finally {
109+
writer.setBuffer(prevBuffer);
110+
}
111+
}
112+
}

0 commit comments

Comments
 (0)