feat: High performance pandas integration.

.gitignore

@@ -1,7 +1,14 @@
-src/questdb/ingress.html
 src/questdb/ingress.c
+src/questdb/*.html
 rustup-init.exe
 
+# Linux Perf profiles
+perf.data*
+perf/*.svg
+
+# Atheris Crash/OOM and other files
+fuzz-artifact/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

.vscode/settings.json

@@ -1,3 +1,7 @@
 {
-    "esbonio.sphinx.confDir": ""
+    "esbonio.sphinx.confDir": "",
+    "cmake.configureOnOpen": false,
+    "files.associations": {
+        "ingress_helper.h": "c"
+    }
 }

TODO.rst

@@ -6,8 +6,6 @@ TODO
 Build Tooling
 =============
 
-* **[HIGH]** Transition to Azure, move Linux arm to ARM pipeline without QEMU.
-
 * **[MEDIUM]** Automate Apple Silicon as part of CI.
 
 * **[LOW]** Release to PyPI from CI.
@@ -20,8 +18,6 @@ Docs
   are in the C client). This is to ensure they don't "bit rot" as the code
   changes.
 
-* **[MEDIUM]** Document on a per-version basis.
-
 Development
 ===========
 

ci/cibuildwheel.yaml

@@ -68,7 +68,7 @@ stages:
           - bash: |
               set -o errexit
               python3 -m pip install --upgrade pip
-              pip3 install cibuildwheel==2.11.1
+              python3 -m pip install cibuildwheel==2.11.2
             displayName: Install dependencies
           - bash: cibuildwheel --output-dir wheelhouse .
             displayName: Build wheels
@@ -83,7 +83,7 @@ stages:
           - bash: |
               set -o errexit
               python3 -m pip install --upgrade pip
-              pip3 install cibuildwheel==2.11.1
+              python3 -m pip install cibuildwheel==2.11.2
             displayName: Install dependencies
           - bash: cibuildwheel --output-dir wheelhouse .
             displayName: Build wheels
@@ -100,7 +100,7 @@ stages:
           - bash: |
               set -o errexit
               python3 -m pip install --upgrade pip
-              pip3 install cibuildwheel==2.11.1
+              python3 -m pip install cibuildwheel==2.11.2
             displayName: Install dependencies
           - bash: cibuildwheel --output-dir wheelhouse .
             displayName: Build wheels
@@ -117,7 +117,7 @@ stages:
           - bash: |
               set -o errexit
               python3 -m pip install --upgrade pip
-              pip3 install cibuildwheel==2.11.1
+              python3 -m pip install cibuildwheel==2.11.2
             displayName: Install dependencies
           - bash: cibuildwheel --output-dir wheelhouse .
             displayName: Build wheels
@@ -134,7 +134,7 @@ stages:
           - bash: |
               set -o errexit
               python3 -m pip install --upgrade pip
-              pip3 install cibuildwheel==2.11.1
+              python3 -m pip install cibuildwheel==2.11.2
             displayName: Install dependencies
           - bash: cibuildwheel --output-dir wheelhouse .
             displayName: Build wheels
@@ -151,7 +151,7 @@ stages:
           - bash: |
               set -o errexit
               python3 -m pip install --upgrade pip
-              python3 -m pip install cibuildwheel==2.11.1
+              python3 -m pip install cibuildwheel==2.11.2
             displayName: Install dependencies
           - bash: cibuildwheel --output-dir wheelhouse .
             displayName: Build wheels
@@ -165,8 +165,8 @@ stages:
           - task: UsePythonVersion@0
           - bash: |
               set -o errexit
-              python -m pip install --upgrade pip
-              pip install cibuildwheel==2.11.1
+              python3 -m pip install --upgrade pip
+              python3 -m pip install cibuildwheel==2.11.2
             displayName: Install dependencies
           - bash: cibuildwheel --output-dir wheelhouse .
             displayName: Build wheels

ci/pip_install_deps.py

@@ -0,0 +1,71 @@
+import sys
+import subprocess
+import shlex
+import textwrap
+import platform
+
+
+class UnsupportedDependency(Exception):
+    pass
+
+
+def pip_install(package):
+    args = [
+        sys.executable,
+        '-m', 'pip', 'install',
+        '--upgrade',
+        '--only-binary', ':all:',
+        package]
+    args_s = ' '.join(shlex.quote(arg) for arg in args)
+    sys.stderr.write(args_s + '\n')
+    res = subprocess.run(
+        args,
+        stderr=subprocess.STDOUT,
+        stdout=subprocess.PIPE)
+    if res.returncode == 0:
+        return
+    output = res.stdout.decode('utf-8')
+    if 'Could not find a version that satisfies the requirement' in output:
+        raise UnsupportedDependency(output)
+    else:
+        sys.stderr.write(output + '\n')
+        sys.exit(res.returncode)
+
+
+def try_pip_install(package):
+    try:
+        pip_install(package)
+    except UnsupportedDependency as e:
+        msg = textwrap.indent(str(e), ' ' * 8)
+        sys.stderr.write(f'    Ignored unsatisfiable dependency:\n{msg}\n')
+
+
+def ensure_timezone():
+    try:
+        import zoneinfo
+        if platform.system() == 'Windows':
+            pip_install('tzdata')  # for zoneinfo
+    except ImportError:
+        pip_install('pytz')
+
+
+def main():
+    ensure_timezone()
+    try_pip_install('pandas')
+    try_pip_install('numpy')
+    try_pip_install('pyarrow')
+
+    on_linux_is_glibc = (
+        (not platform.system() == 'Linux') or
+        (platform.libc_ver()[0] == 'glibc'))
+    is_64bits = sys.maxsize > 2**32
+    is_cpython = platform.python_implementation() == 'CPython'
+    if on_linux_is_glibc and is_64bits and is_cpython:
+        # Ensure that we've managed to install the expected dependencies.
+        import pandas
+        import numpy
+        import pyarrow
+
+
+if __name__ == "__main__":
+    main()

ci/run_tests_pipeline.yaml

@@ -28,11 +28,17 @@ stages:
             submodules: true
           - task: UsePythonVersion@0
           - script: python3 --version
-          - script: python3 -m pip install cython
+          - script: |
+              python3 -m pip install cython
+              python3 ci/pip_install_deps.py
             displayName: Installing Python dependencies
           - script: python3 proj.py build
             displayName: "Build"
           - script: python3 proj.py test 1
             displayName: "Test"
             env:
               JAVA_HOME: $(JAVA_HOME_11_X64)
+
+# TODO: Add tests with and tests without installing `pyarrow` as it's
+# an optional dependency (that can't always be installed).
+# The tests without to test the fallback logic.

dev_requirements.txt

@@ -1,8 +1,10 @@
 setuptools>=45.2.0
 Cython>=0.29.32
 wheel>=0.34.2
-cibuildwheel>=2.11.1
+cibuildwheel>=2.11.2
 Sphinx>=5.0.2
 sphinx-rtd-theme>=1.0.0
 twine>=4.0.1
 bump2version>=1.0.1
+pandas>=1.3.5
+numpy>=1.21.6

perf/README.md

@@ -0,0 +1,28 @@
+# Profiling with Linux Perf
+
+https://juanjose.garciaripoll.com/blog/profiling-code-with-linux-perf/index.html
+
+```bash
+$ TEST_QUESTDB_PATCH_PATH=1 perf record -g --call-graph dwarf python3 test/benchmark.py -v TestBencharkPandas.test_string_encoding_1m
+test_string_encoding_1m (__main__.TestBencharkPandas.test_string_encoding_1m) ... Time: 4.682273147998785, size: 4593750000
+ok
+
+----------------------------------------------------------------------
+Ran 1 test in 10.166s
+
+OK
+[ perf record: Woken up 1341 times to write data ]
+Warning:
+Processed 54445 events and lost 91 chunks!
+
+Check IO/CPU overload!
+
+[ perf record: Captured and wrote 405.575 MB perf.data (50622 samples) ]
+```
+
+# Rendering results
+
+```bash
+$ perf script | python3 perf/gprof2dot.py --format=perf | dot -Tsvg > perf/profile_graph.svg
+$ (cd perf && python3 -m http.server)
+```