Skip to content

Commit a02716b

Browse files
staticlibsstaticlibs
committed
Appender rework
Note: this PR is still a draft, please do not merge. This change implements the access to Appender interface from Java with the following features: - C API is used to access the native Appender - necessary C API calls are exposed to Java using JNI wrappers as thin as possible - Java calls mirror corresponding C API calls 1 to 1 - the data chunk interface of the Appender API is used: vector data is exposed as a direct ByteBuffer, all primitive appended values are written to this buffer from Java without going through JNI + C API ( that is still used for some types with calls like: `duckdb_vector_assign_string_element_len`) - Java-side Appender/DataChunk/Vector data structures follow the Go lang's Appender implementation closely (with nested lists/structs initializatiion etc) - Java Appender usage is made thread-safe for concurrent Appender or Connection closure; `append()` calls are remained not thread-safe (to minimize the overhead), but their usage cannot cause the JNI-side crash` Testing: existing Appender test suite is used and adapted to extended Appender API. Fixes: todo: list all related GH issues
1 parent 53f0fc2 commit a02716b

39 files changed

+3828
-608
lines changed

‎CMakeLists.txt

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -553,7 +553,7 @@ endif()
553553

554554
file(GLOB_RECURSE JAVA_SRC_FILES src/main/java/org/duckdb/*.java)
555555
file(GLOB_RECURSE JAVA_TEST_FILES src/test/java/org/duckdb/*.java)
556-
set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g)
556+
set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g -Xlint:all)
557557

558558
add_jar(duckdb_jdbc ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
559559
MANIFEST META-INF/MANIFEST.MF
@@ -570,6 +570,12 @@ else()
570570
endif()
571571

572572
add_library(duckdb_java SHARED
573+
src/jni/bindings_appender.cpp
574+
src/jni/bindings_common.cpp
575+
src/jni/bindings_data_chunk.cpp
576+
src/jni/bindings_logical_type.cpp
577+
src/jni/bindings_validity.cpp
578+
src/jni/bindings_vector.cpp
573579
src/jni/config.cpp
574580
src/jni/duckdb_java.cpp
575581
src/jni/functions.cpp

‎CMakeLists.txt.in

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ endif()
8383

8484
file(GLOB_RECURSE JAVA_SRC_FILES src/main/java/org/duckdb/*.java)
8585
file(GLOB_RECURSE JAVA_TEST_FILES src/test/java/org/duckdb/*.java)
86-
set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g)
86+
set(CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 -g -Xlint:all)
8787

8888
add_jar(duckdb_jdbc ${JAVA_SRC_FILES} META-INF/services/java.sql.Driver
8989
MANIFEST META-INF/MANIFEST.MF
@@ -100,6 +100,12 @@ else()
100100
endif()
101101

102102
add_library(duckdb_java SHARED
103+
src/jni/bindings_appender.cpp
104+
src/jni/bindings_common.cpp
105+
src/jni/bindings_data_chunk.cpp
106+
src/jni/bindings_logical_type.cpp
107+
src/jni/bindings_validity.cpp
108+
src/jni/bindings_vector.cpp
103109
src/jni/config.cpp
104110
src/jni/duckdb_java.cpp
105111
src/jni/functions.cpp

‎duckdb_java.def

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,47 @@ Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1set_1catalog
4747
Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1set_1schema
4848
Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1startup
4949

50+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1size
51+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1logical_1type
52+
Java_org_duckdb_DuckDBBindings_duckdb_1get_1type_1id
53+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1list_1type
54+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1array_1type
55+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1struct_1type
56+
Java_org_duckdb_DuckDBBindings_duckdb_1struct_1type_1child_1count
57+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1logical_1type
58+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1vector
59+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1vector
60+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1column_1type
61+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1data
62+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1validity
63+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1ensure_1validity_1writable
64+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1assign_1string_1element
65+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1assign_1string_1element_1len
66+
Java_org_duckdb_DuckDBBindings_duckdb_1validity_1row_1is_1valid
67+
Java_org_duckdb_DuckDBBindings_duckdb_1validity_1set_1row_1validity
68+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1child
69+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1size
70+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1set_1size
71+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1reserve
72+
Java_org_duckdb_DuckDBBindings_duckdb_1struct_1vector_1get_1child
73+
Java_org_duckdb_DuckDBBindings_duckdb_1array_1vector_1get_1child
74+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1data_1chunk
75+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1data_1chunk
76+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1reset
77+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1column_1count
78+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1vector
79+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1size
80+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1set_1size
81+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1create_1ext
82+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1error
83+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1flush
84+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1close
85+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1destroy
86+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1count
87+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1type
88+
Java_org_duckdb_DuckDBBindings_duckdb_1append_1data_1chunk
89+
Java_org_duckdb_DuckDBBindings_duckdb_1append_1default_1to_1chunk
90+
5091
duckdb_adbc_init
5192
duckdb_add_aggregate_function_to_set
5293
duckdb_add_replacement_scan

‎duckdb_java.exp

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,47 @@ _Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1set_1catalog
4444
_Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1set_1schema
4545
_Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1startup
4646

47+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1size
48+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1logical_1type
49+
_Java_org_duckdb_DuckDBBindings_duckdb_1get_1type_1id
50+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1list_1type
51+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1array_1type
52+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1struct_1type
53+
_Java_org_duckdb_DuckDBBindings_duckdb_1struct_1type_1child_1count
54+
_Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1logical_1type
55+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1vector
56+
_Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1vector
57+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1column_1type
58+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1data
59+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1validity
60+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1ensure_1validity_1writable
61+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1assign_1string_1element
62+
_Java_org_duckdb_DuckDBBindings_duckdb_1vector_1assign_1string_1element_1len
63+
_Java_org_duckdb_DuckDBBindings_duckdb_1validity_1row_1is_1valid
64+
_Java_org_duckdb_DuckDBBindings_duckdb_1validity_1set_1row_1validity
65+
_Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1child
66+
_Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1size
67+
_Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1set_1size
68+
_Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1reserve
69+
_Java_org_duckdb_DuckDBBindings_duckdb_1struct_1vector_1get_1child
70+
_Java_org_duckdb_DuckDBBindings_duckdb_1array_1vector_1get_1child
71+
_Java_org_duckdb_DuckDBBindings_duckdb_1create_1data_1chunk
72+
_Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1data_1chunk
73+
_Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1reset
74+
_Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1column_1count
75+
_Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1vector
76+
_Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1size
77+
_Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1set_1size
78+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1create_1ext
79+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1error
80+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1flush
81+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1close
82+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1destroy
83+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1count
84+
_Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1type
85+
_Java_org_duckdb_DuckDBBindings_duckdb_1append_1data_1chunk
86+
_Java_org_duckdb_DuckDBBindings_duckdb_1append_1default_1to_1chunk
87+
4788
_duckdb_adbc_init
4889
_duckdb_add_aggregate_function_to_set
4990
_duckdb_add_replacement_scan

‎duckdb_java.map

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,47 @@ DUCKDB_JAVA {
4646
Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1set_1schema;
4747
Java_org_duckdb_DuckDBNative_duckdb_1jdbc_1startup;
4848

49+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1size;
50+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1logical_1type;
51+
Java_org_duckdb_DuckDBBindings_duckdb_1get_1type_1id;
52+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1list_1type;
53+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1array_1type;
54+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1struct_1type;
55+
Java_org_duckdb_DuckDBBindings_duckdb_1struct_1type_1child_1count;
56+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1logical_1type;
57+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1vector;
58+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1vector;
59+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1column_1type;
60+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1data;
61+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1get_1validity;
62+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1ensure_1validity_1writable;
63+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1assign_1string_1element;
64+
Java_org_duckdb_DuckDBBindings_duckdb_1vector_1assign_1string_1element_1len;
65+
Java_org_duckdb_DuckDBBindings_duckdb_1validity_1row_1is_1valid;
66+
Java_org_duckdb_DuckDBBindings_duckdb_1validity_1set_1row_1validity;
67+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1child;
68+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1get_1size;
69+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1set_1size;
70+
Java_org_duckdb_DuckDBBindings_duckdb_1list_1vector_1reserve;
71+
Java_org_duckdb_DuckDBBindings_duckdb_1struct_1vector_1get_1child;
72+
Java_org_duckdb_DuckDBBindings_duckdb_1array_1vector_1get_1child;
73+
Java_org_duckdb_DuckDBBindings_duckdb_1create_1data_1chunk;
74+
Java_org_duckdb_DuckDBBindings_duckdb_1destroy_1data_1chunk;
75+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1reset;
76+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1column_1count;
77+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1vector;
78+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1get_1size;
79+
Java_org_duckdb_DuckDBBindings_duckdb_1data_1chunk_1set_1size;
80+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1create_1ext;
81+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1error;
82+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1flush;
83+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1close;
84+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1destroy;
85+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1count;
86+
Java_org_duckdb_DuckDBBindings_duckdb_1appender_1column_1type;
87+
Java_org_duckdb_DuckDBBindings_duckdb_1append_1data_1chunk;
88+
Java_org_duckdb_DuckDBBindings_duckdb_1append_1default_1to_1chunk;
89+
4990
duckdb_adbc_init;
5091
duckdb_add_aggregate_function_to_set;
5192
duckdb_add_replacement_scan;

‎src/jni/bindings.hpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#pragma once
2+
3+
extern "C" {
4+
#include "duckdb.h"
5+
}
6+
7+
#include "org_duckdb_DuckDBBindings.h"
8+
9+
duckdb_logical_type logical_type_buf_to_logical_type(JNIEnv *env, jobject logical_type_buf);
10+
11+
duckdb_data_chunk chunk_buf_to_chunk(JNIEnv *env, jobject chunk_buf);

0 commit comments

Comments
 (0)