Skip to content

Commit 141e947

Browse files
staticlibsstaticlibs
committed
Appender rework (1.3)
This is a backport of the PR duckdb#295 to `v1.3-ossivalis` stable branch. This change implements the access to the Appender interface from Java with the following features: - C API is used to access the native Appender - necessary C API calls are exposed to Java using JNI wrappers as thin as possible - Java calls mirror corresponding C API calls 1 to 1 - the data chunk interface of the Appender API is used: vector data is exposed as a direct ByteBuffer, all primitive appended values are written to this buffer from Java without going through JNI + C API ( that is still used for some types with calls like: `duckdb_vector_assign_string_element_len`) - Java-side Appender/DataChunk/Vector data structures follow the Go lang's Appender implementation closely (with nested arrays initialization etc) - Java Appender usage is made thread-safe for concurrent Appender or Connection closure; `append()` calls are remained not thread-safe (to minimize the overhead), but their usage cannot cause the JNI-side crash - Java API of the new Appender is modeled after the `java.lang.StringBuilder` class and intended to be used with method chaining - support primitive arrays (one and two dimensional) - support for `NULL` and `DEFAULT` values - type checks between Java types and DB column types are enforced Note: previous version of the Appender (that internally creates `Value` instances and appends them one by one) is still available for compatibility. It can be created using `Connection#createSingleValueAppender` method. It is marked as 'deprecated' and intended to be removed in future versions. Testing: existing Appender test suite is extended and adapted to new Appender API. Fixes: duckdb#84, duckdb#100, duckdb#110, duckdb#139, duckdb#157, duckdb#163, duckdb#219, duckdb#249
1 parent f9d3470 commit 141e947

39 files changed

+5958
-633
lines changed

‎CMakeLists.txt

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -549,7 +549,7 @@ endif()
549549

550550
file(GLOB_RECURSE JAVA_SRC_FILES src/main/java/org/duckdb/*.java)
551551
file(GLOB_RECURSE JAVA_TEST_FILES src/test/java/org/duckdb/*.java)
552-
set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g)
552+
set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g -Xlint:all)
553553

554554
add_jar(duckdb_jdbc ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
555555
MANIFEST META-INF/MANIFEST.MF
@@ -566,6 +566,12 @@ else()
566566
endif()
567567

568568
add_library(duckdb_java SHARED
569+
src/jni/bindings_appender.cpp
570+
src/jni/bindings_common.cpp
571+
src/jni/bindings_data_chunk.cpp
572+
src/jni/bindings_logical_type.cpp
573+
src/jni/bindings_validity.cpp
574+
src/jni/bindings_vector.cpp
569575
src/jni/config.cpp
570576
src/jni/duckdb_java.cpp
571577
src/jni/functions.cpp

‎CMakeLists.txt.in

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ endif()
8383

8484
file(GLOB_RECURSE JAVA_SRC_FILES src/main/java/org/duckdb/*.java)
8585
file(GLOB_RECURSE JAVA_TEST_FILES src/test/java/org/duckdb/*.java)
86-
set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g)
86+
set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g -Xlint:all)
8787

8888
add_jar(duckdb_jdbc ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
8989
MANIFEST META-INF/MANIFEST.MF
@@ -100,6 +100,12 @@ else()
100100
endif()
101101

102102
add_library(duckdb_java SHARED
103+
src/jni/bindings_appender.cpp
104+
src/jni/bindings_common.cpp
105+
src/jni/bindings_data_chunk.cpp
106+
src/jni/bindings_logical_type.cpp
107+
src/jni/bindings_validity.cpp
108+
src/jni/bindings_vector.cpp
103109
src/jni/config.cpp
104110
src/jni/duckdb_java.cpp
105111
src/jni/functions.cpp

‎duckdb_java.def

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,50 @@ Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1set_1catalog
4747
Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1set_1schema
4848
Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1startup
4949

50+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1size
51+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1logical_1type
52+
Java_org_duckdb_DuckDBBindings_duckdb_1get_1type_1id
53+
Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1width
54+
Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1scale
55+
Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1internal_1type
56+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1list_1type
57+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1array_1type
58+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1struct_1type
59+
Java_org_duckdb_DuckDBBindings_duckdb_1struct_1type_1child_1count
60+
Java_org_duckdb_DuckDBBindings_duckdb_1array_1type_1array_1size
61+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1logical_1type
62+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1vector
63+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1vector
64+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1column_1type
65+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1data
66+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1validity
67+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1ensure_1validity_1writable
68+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1assign_1string_1element_1len
69+
Java_org_duckdb_DuckDBBindings_duckdb_1validity_1row_1is_1valid
70+
Java_org_duckdb_DuckDBBindings_duckdb_1validity_1set_1row_1validity
71+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1child
72+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1size
73+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1set_1size
74+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1reserve
75+
Java_org_duckdb_DuckDBBindings_duckdb_1struct_1vector_1get_1child
76+
Java_org_duckdb_DuckDBBindings_duckdb_1array_1vector_1get_1child
77+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1data_1chunk
78+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1data_1chunk
79+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1reset
80+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1column_1count
81+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1vector
82+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1size
83+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1set_1size
84+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1create_1ext
85+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1error
86+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1flush
87+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1close
88+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1destroy
89+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1count
90+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1type
91+
Java_org_duckdb_DuckDBBindings_duckdb_1append_1data_1chunk
92+
Java_org_duckdb_DuckDBBindings_duckdb_1append_1default_1to_1chunk
93+
5094
duckdb_adbc_init
5195
duckdb_add_aggregate_function_to_set
5296
duckdb_add_replacement_scan

‎duckdb_java.exp

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,50 @@ _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1set_1catalog
4444
_Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1set_1schema
4545
_Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1startup
4646

47+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1size
48+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1logical_1type
49+
_Java_org_duckdb_DuckDBBindings_duckdb_1get_1type_1id
50+
_Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1width
51+
_Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1scale
52+
_Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1internal_1type
53+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1list_1type
54+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1array_1type
55+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1struct_1type
56+
_Java_org_duckdb_DuckDBBindings_duckdb_1struct_1type_1child_1count
57+
_Java_org_duckdb_DuckDBBindings_duckdb_1array_1type_1array_1size
58+
_Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1logical_1type
59+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1vector
60+
_Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1vector
61+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1column_1type
62+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1data
63+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1validity
64+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1ensure_1validity_1writable
65+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1assign_1string_1element_1len
66+
_Java_org_duckdb_DuckDBBindings_duckdb_1validity_1row_1is_1valid
67+
_Java_org_duckdb_DuckDBBindings_duckdb_1validity_1set_1row_1validity
68+
_Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1child
69+
_Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1size
70+
_Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1set_1size
71+
_Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1reserve
72+
_Java_org_duckdb_DuckDBBindings_duckdb_1struct_1vector_1get_1child
73+
_Java_org_duckdb_DuckDBBindings_duckdb_1array_1vector_1get_1child
74+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1data_1chunk
75+
_Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1data_1chunk
76+
_Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1reset
77+
_Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1column_1count
78+
_Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1vector
79+
_Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1size
80+
_Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1set_1size
81+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1create_1ext
82+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1error
83+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1flush
84+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1close
85+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1destroy
86+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1count
87+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1type
88+
_Java_org_duckdb_DuckDBBindings_duckdb_1append_1data_1chunk
89+
_Java_org_duckdb_DuckDBBindings_duckdb_1append_1default_1to_1chunk
90+
4791
_duckdb_adbc_init
4892
_duckdb_add_aggregate_function_to_set
4993
_duckdb_add_replacement_scan

‎duckdb_java.map

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,50 @@ DUCKDB_JAVA {
4646
Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1set_1schema;
4747
Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1startup;
4848

49+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1size;
50+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1logical_1type;
51+
Java_org_duckdb_DuckDBBindings_duckdb_1get_1type_1id;
52+
Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1width;
53+
Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1scale;
54+
Java_org_duckdb_DuckDBBindings_duckdb_1decimal_1internal_1type;
55+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1list_1type;
56+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1array_1type;
57+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1struct_1type;
58+
Java_org_duckdb_DuckDBBindings_duckdb_1struct_1type_1child_1count;
59+
Java_org_duckdb_DuckDBBindings_duckdb_1array_1type_1array_1size;
60+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1logical_1type;
61+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1vector;
62+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1vector;
63+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1column_1type;
64+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1data;
65+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1validity;
66+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1ensure_1validity_1writable;
67+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1assign_1string_1element_1len;
68+
Java_org_duckdb_DuckDBBindings_duckdb_1validity_1row_1is_1valid;
69+
Java_org_duckdb_DuckDBBindings_duckdb_1validity_1set_1row_1validity;
70+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1child;
71+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1size;
72+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1set_1size;
73+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1reserve;
74+
Java_org_duckdb_DuckDBBindings_duckdb_1struct_1vector_1get_1child;
75+
Java_org_duckdb_DuckDBBindings_duckdb_1array_1vector_1get_1child;
76+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1data_1chunk;
77+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1data_1chunk;
78+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1reset;
79+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1column_1count;
80+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1vector;
81+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1size;
82+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1set_1size;
83+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1create_1ext;
84+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1error;
85+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1flush;
86+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1close;
87+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1destroy;
88+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1count;
89+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1type;
90+
Java_org_duckdb_DuckDBBindings_duckdb_1append_1data_1chunk;
91+
Java_org_duckdb_DuckDBBindings_duckdb_1append_1default_1to_1chunk;
92+
4993
duckdb_adbc_init;
5094
duckdb_add_aggregate_function_to_set;
5195
duckdb_add_replacement_scan;

‎src/jni/bindings.hpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#pragma once
2+
3+
extern "C" {
4+
#include "duckdb.h"
5+
}
6+
7+
#include "org_duckdb_DuckDBBindings.h"
8+
9+
duckdb_logical_type logical_type_buf_to_logical_type(JNIEnv *env, jobject logical_type_buf);
10+
11+
duckdb_data_chunk chunk_buf_to_chunk(JNIEnv *env, jobject chunk_buf);

0 commit comments

Comments
 (0)