Async sender

ci/compile.yaml

@@ -4,10 +4,6 @@ steps:
       rustup default $(toolchain)
     condition: ne(variables['toolchain'], '')
     displayName: "Update and set Rust toolchain"
-  - script: |
-      python -m pip install --upgrade pip
-      pip install numpy
-    displayName: 'Install Python Dependencies'
   - script: cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DQUESTDB_TESTS_AND_EXAMPLES=ON
     env:
       JAVA_HOME: $(JAVA_HOME_11_X64)
@@ -16,11 +12,3 @@ steps:
     env:
       JAVA_HOME: $(JAVA_HOME_11_X64)
     displayName: "Make"
-  - script: cmake -S . -B build_CXX20 -DCMAKE_BUILD_TYPE=Release -DQUESTDB_TESTS_AND_EXAMPLES=ON -DCMAKE_CXX_STANDARD=20
-    env:
-      JAVA_HOME: $(JAVA_HOME_11_X64)
-    displayName: "Build Makefile with CMake"
-  - script: cmake --build build_CXX20 --config Release
-    env:
-      JAVA_HOME: $(JAVA_HOME_11_X64)
-    displayName: "Make"

ci/run_all_tests.py

@@ -36,26 +36,17 @@ def main():
     exe_suffix = '.exe' if platform.system() == 'Windows' else ''
     test_line_sender_path = next(iter(
         build_dir.glob(f'**/test_line_sender{exe_suffix}')))
-    build_cxx20_dir = pathlib.Path('build_CXX20')
-    test_line_sender_path_CXX20 = next(iter(
-        build_cxx20_dir.glob(f'**/test_line_sender{exe_suffix}')))
-
-    system_test_path = pathlib.Path('system_test') / 'test.py'
-    #qdb_v = '8.2.3'  # The version of QuestDB we'll test against.
 
     run_cmd('cargo', 'test',
             '--', '--nocapture', cwd='questdb-rs')
-    run_cmd('cargo', 'test', '--no-default-features', '--features=aws-lc-crypto,tls-native-certs',
+    run_cmd('cargo', 'test', '--no-default-features', '--features=aws-lc-crypto,tls-native-certs,sync-sender-tcp',
             '--', '--nocapture', cwd='questdb-rs')
-    run_cmd('cargo', 'test', '--no-default-features', '--features=ring-crypto,tls-native-certs,ilp-over-http',
+    run_cmd('cargo', 'test', '--no-default-features', '--features=ring-crypto,tls-native-certs,sync-sender',
             '--', '--nocapture', cwd='questdb-rs')
     run_cmd('cargo', 'test', '--features=almost-all-features',
             '--', '--nocapture', cwd='questdb-rs')
     run_cmd('cargo', 'test', cwd='questdb-rs-ffi')
     run_cmd(str(test_line_sender_path))
-    run_cmd(str(test_line_sender_path_CXX20))
-    #run_cmd('python3', str(system_test_path), 'run', '--versions', qdb_v, '-v')
-    run_cmd('python3', str(system_test_path), 'run', '--repo', './questdb', '-v')
 
 
 if __name__ == '__main__':

ci/run_tests_pipeline.yaml

@@ -54,17 +54,6 @@ stages:
               cd questdb-rs
               cargo build --examples --features almost-all-features
             displayName: "Build Rust examples"
-          ############################# temp for test begin #####################
-          - script: |
-              git clone --depth 1 https://github.com/questdb/questdb.git ./questdb
-            displayName: git clone questdb
-          - task: Maven@3
-            displayName: "Compile QuestDB"
-            inputs:
-              mavenPOMFile: "questdb/pom.xml"
-              jdkVersionOption: "1.11"
-              options: "-DskipTests -Pbuild-web-console"
-          ############################# temp for test end ##################### 
           - script: python3 ci/run_all_tests.py
             env:
               JAVA_HOME: $(JAVA_HOME_11_X64)
@@ -80,11 +69,6 @@ stages:
         timeoutInMinutes: 10
         steps:
           - checkout: self
-          - script: |
-              apt install clang-format
-              rustup component add clippy
-              rustup component add rustfmt
-            displayName: "Install clang-format, clippy and rustfmt"
           - script: |
               cd questdb-rs
               cargo fmt --all -- --check
@@ -101,9 +85,6 @@ stages:
               cd questdb-rs-ffi
               cargo clippy --all-targets --all-features -- -D warnings
             displayName: "questdb-rs-ffi: clippy"
-          - script: |
-              python3 ci/format_cpp.py --check
-            displayName: "C/C++ clang-format"
           - script: |
               cd system_test
               cd tls_proxy
@@ -114,26 +95,3 @@ stages:
               cd tls_proxy
               cargo clippy --all-targets --all-features -- -D warnings
             displayName: "tls_proxy: clippy"
-      - job: TestVsQuestDBMaster
-        displayName: "Vs QuestDB 'master'"
-        pool:
-          vmImage: 'ubuntu-latest'
-        timeoutInMinutes: 60
-        steps:
-          - checkout: self
-            fetchDepth: 1
-            lfs: false
-            submodules: false
-          - template: compile.yaml
-          - script: |
-              git clone --depth 1 https://github.com/questdb/questdb.git
-            displayName: git clone questdb
-          - task: Maven@3
-            displayName: "Compile QuestDB"
-            inputs:
-              mavenPOMFile: 'questdb/pom.xml'
-              jdkVersionOption: '1.11'
-              options: "-DskipTests -Pbuild-web-console"
-          - script: |
-              python3 system_test/test.py run --repo ./questdb -v
-            displayName: "integration test"

questdb-rs-ffi/Cargo.toml

@@ -20,7 +20,7 @@ features = [
     "insecure-skip-verify",
     "tls-webpki-certs",
     "tls-native-certs",
-    "ilp-over-http"
+    "sync-sender"
 ]
 
 [features]

questdb-rs/Cargo.toml

@@ -19,7 +19,7 @@ crate-type = ["lib"]
 
 [dependencies]
 libc = "0.2"
-socket2 = "0.5.5"
+socket2 = { version = "0.5.5", optional = true }
 dns-lookup = "2.0.4"
 base64ct = { version = "1.7", features = ["alloc"] }
 rustls-pemfile = "2.0.0"
@@ -30,12 +30,17 @@ ring = { version = "0.17.14", optional = true }
 rustls-pki-types = "1.0.1"
 rustls = { version = "0.23.25", default-features = false, features = ["logging", "std", "tls12"] }
 rustls-native-certs = { version = "0.8.1", optional = true }
-webpki-roots = { version = "0.26.8", default-features = false, optional = true }
+webpki-roots = { version = "1.0.1", default-features = false, optional = true }
 chrono = { version = "0.4.40", optional = true }
 
 # We need to limit the `ureq` version to 3.0.x since we use
 # the `ureq::unversioned` module which does not respect semantic versioning.
 ureq = { version = "3.0.10, <3.1.0", default-features = false, features = ["rustls-no-provider"], optional = true }
+
+tokio = { version = "1.45.1", default-features = false, features = ["net"], optional = true }
+tokio-rustls = { version = "0.26.2", default-features = false, optional = true }
+hyper = { version = "1.6.0", default-features = false, optional = true }
+
 serde_json = { version = "1", optional = true }
 questdb-confstr = "0.1.1"
 rand = { version = "0.9.0", optional = true }
@@ -54,43 +59,70 @@ indoc = "2"
 mio = { version = "1", features = ["os-poll", "net"] }
 chrono = "0.4.31"
 tempfile = "3"
-webpki-roots = "0.26.8"
+webpki-roots = "1.0.1"
 rstest = "0.25.0"
 
 [features]
-default = ["tls-webpki-certs", "ilp-over-http", "ring-crypto"]
+default = ["sync-sender", "tls-webpki-certs", "ring-crypto"]
+
+## Sync ILP/TCP + ILP/HTTP Sender
+sync-sender = ["sync-sender-tcp", "sync-sender-http"]
+
+## Sync ILP/TCP
+sync-sender-tcp = ["_sync-sender", "_sender-tcp", "dep:socket2"]
+
+## Sync ILP/HTTP
+sync-sender-http = ["_sync-sender", "_sender-http", "dep:ureq", "dep:serde_json", "dep:rand"]
+
+## Async ILP/TCP + ILP/HTTP Sender
+async-sender = ["_async-sender", "async-sender-tcp", "async-sender-http"]
+
+## Async ILP/TCP Sender
+async-sender-tcp = ["_async-sender", "_sender-tcp", "dep:tokio", "dep:tokio-rustls"]
+
+## Async ILP/HTTP Sender
+async-sender-http = ["_sender-http", "dep:tokio", "dep:hyper"]
 
-# Include support for ILP over HTTP.
-ilp-over-http = ["dep:ureq", "dep:serde_json", "dep:rand"]
+## Compatiblity alias.
+ilp-over-http = ["sync-sender-tcp"]
 
-# Allow use OS-provided root TLS certificates
+## Allow use OS-provided root TLS certificates
 tls-native-certs = ["dep:rustls-native-certs"]
 
-# Allow use of the `webpki-roots` crate to validate TLS certificates.
+## Allow use of the `webpki-roots` crate to validate TLS certificates.
 tls-webpki-certs = ["dep:webpki-roots"]
 
-# Use `aws-lc-rs` as the cryto library.
+## Use `aws-lc-rs` as the cryto library.
 aws-lc-crypto = ["dep:aws-lc-rs", "rustls/aws-lc-rs"]
 
-# Use `ring` as the crypto library.
+## Use `ring` as the crypto library.
 ring-crypto = ["dep:ring", "rustls/ring"]
 
-# Allow skipping verification of insecure certificates.
+## Allow skipping verification of insecure certificates.
 insecure-skip-verify = []
 
-# Enable code-generation in `build.rs` for additional tests.
+## Enable code-generation in `build.rs` for additional tests.
 json_tests = []
 
-# Enable methods to create timestamp objects from chrono::DateTime objects.
+## Enable methods to create timestamp objects from chrono::DateTime objects.
 chrono_timestamp = ["chrono"]
 
-# The `aws-lc-crypto` and `ring-crypto` features are mutually exclusive,
-# thus compiling with `--all-features` will not work.
-# Instead compile with `--features almost-all-features`.
-# This is useful for quickly running `cargo test` or `cargo clippy`.
+# Hidden derived features, used in code to enable-disable code sections. Don't use directly.
+_sender-tcp = [] # equiv of any(sync-sender-tcp, async-sender-tcp)
+_sender-http = [] # equiv of any(sync-sender-http, async-sender-http)
+_sync-sender = [] # equiv of any(sync-sender-tcp, sync-sender-http)
+_async-sender = [] # equiv of any(async-sender-tcp, async-sender-http)
+
+## Enable all cross-compatible features.
+## The `aws-lc-crypto` and `ring-crypto` features are mutually exclusive,
+## thus compiling with `--all-features` will not work.
+## Instead use `--features almost-all-features`.
+## This is useful for quickly running `cargo test` or `cargo clippy`.
 almost-all-features = [
+    "sync-sender",
+    "async-sender",
     "tls-webpki-certs",
-    "ilp-over-http",
+    "tls-native-certs",
     "ring-crypto",
     "insecure-skip-verify",
     "json_tests",
@@ -112,8 +144,8 @@ required-features = ["chrono_timestamp"]
 
 [[example]]
 name = "http"
-required-features = ["ilp-over-http", "ndarray"]
+required-features = ["sync-sender-http", "ndarray"]
 
 [[example]]
 name = "protocol_version"
-required-features = ["ilp-over-http", "ndarray"]
+required-features = ["sync-sender-http", "ndarray"]

questdb-rs/README.md

@@ -92,7 +92,7 @@ The crate provides several optional features to enable additional functionality.
 
 ### Default features
 
-- **ilp-over-http**: Enables ILP/HTTP support via the `ureq` crate for sending data over HTTP.
+- **sync-sender**: Enables sync Sender APIs for ILP/TCP and ILP/HTTP. HTTP is enabled via the [`ureq`](https://github.com/algesten/ureq) crate.
 - **tls-webpki-certs**: Uses the `webpki-roots` crate to validate TLS certificates.
 - **ring-crypto**: Uses the `ring` crate as the cryptography backend for TLS (default crypto backend).