[CLIENT-3366] Add support for dynamic config

.github/workflows/build-wheels.yml

@@ -245,11 +245,11 @@ jobs:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
-    # It is not documented in Github that openssl 3 is installed in their macos images
+    # It is not documented in Github that openssl 3 / libyaml is installed in their macos images
     # so we install it here to be safe
     - if: ${{ startsWith(inputs.platform-tag, 'macosx') }}
-      name: Ensure openssl is installed on Github macOS runners.
-      run: brew install openssl@${{ env.MACOS_OPENSSL_VERSION }}
+      name: Ensure openssl and libyaml are installed on Github macOS runners.
+      run: brew install openssl@${{ env.MACOS_OPENSSL_VERSION }} libyaml
 
     - if: ${{ startsWith(inputs.platform-tag, 'macosx') }}
       name: Set minimum macos version required to install wheel
@@ -266,7 +266,16 @@ jobs:
       name: Ensure that linker can find brew packages
       # On mac m1, packages installed via brew are not in the linker's default library path
       run: |
-        echo LIBRARY_PATH="${LIBRARY_PATH}:$(brew --prefix openssl)/lib" >> $GITHUB_ENV
+        libraries=('libyaml' 'openssl')
+        for library in "${libraries[@]}"; do
+          LIBRARY_PATH="${LIBRARY_PATH}:$(brew --prefix $library)/lib"
+        done
+        echo LIBRARY_PATH=$LIBRARY_PATH >> $GITHUB_ENV
+      shell: bash
+
+    - if: ${{ inputs.platform-tag == 'macosx_x86_64' && matrix.python-tag == 'cp313' }}
+      # This fixes an issue where there is not enough room in the wheel's shared library to replace the rpath
+      run: echo CIBW_ENVIRONMENT_MACOS="LDFLAGS='-headerpad_max_install_names'" >> $GITHUB_ENV
 
     - name: Build wheel
       uses: pypa/[email protected]
@@ -275,13 +284,14 @@ jobs:
         CIBW_BUILD: ${{ env.BUILD_IDENTIFIER }}
         CIBW_BUILD_FRONTEND: build
         CIBW_BEFORE_ALL_LINUX: >
-          yum install python-setuptools -y
+          yum install python-setuptools libyaml-devel -y
         # delvewheel is not enabled by default but we do need to repair the wheel
         CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel==1.*"
         # We want to check that our wheel links to the new openssl 3 install, not the system default
         # This assumes that ldd prints out the "soname" for the libraries
         # We can also manually verify the repair worked by checking the repaired wheel's compatibility tag
         CIBW_REPAIR_WHEEL_COMMAND_LINUX: >
+          auditwheel show {wheel} &&
           WHEEL_DIR=wheel-contents &&
           unzip {wheel} -d $WHEEL_DIR &&
           ldd $WHEEL_DIR/*.so | awk '{print $1}' | grep libssl.so.3 &&
@@ -297,8 +307,16 @@ jobs:
           delocate-listdeps {dest_dir}/*.whl | grep libssl.3.dylib
         CIBW_TEST_COMMAND: ${{ env.TEST_COMMAND }}
 
+    # - name: Setup tmate session
+    #   if: ${{ !cancelled() }}
+    #   uses: mxschmitt/action-tmate@v3
+
     # For debugging
-    - run: docker logs aerospike
+    - run: |
+        # Checks that server started up properly
+        docker logs aerospike
+        # Prints the actual logs from asd
+        docker exec aerospike cat /var/log/aerospike/aerospike.log
       if: ${{ always() && env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }}
       shell: bash
 

.github/workflows/tests.yml

@@ -142,6 +142,7 @@ jobs:
         mkdir -p query/src/main/query
         mkdir -p scan/src/main/scan
         mkdir -p transaction/src/main/transaction
+        mkdir -p config_provider/src/main/config_provider
 
         cd ../../../../
         cp src/main/*.c build/temp*/src/main/src/main
@@ -154,6 +155,7 @@ jobs:
         cp src/main/query/*.c build/temp*/src/main/query/src/main/query/
         cp src/main/scan/*.c build/temp*/src/main/scan/src/main/scan/
         cp src/main/transaction/*.c build/temp*/src/main/transaction/src/main/transaction/
+        cp src/main/config_provider/*.c build/temp*/src/main/config_provider/src/main/config_provider/
 
     - name: Generate coverage report for all object files
       if: ${{ !cancelled() }}

.gitmodules

@@ -2,4 +2,4 @@
 	path = aerospike-client-c
 	# url = [email protected]:aerospike/aerospike-client-c.git
 	url = https://github.com/aerospike/aerospike-client-c.git
-	branch = stage
+	branch = CLIENT-3346

aerospike-stubs/aerospike.pyi

@@ -324,6 +324,12 @@ class Transaction:
     state: int
     timeout: int
 
+@final
+class ConfigProvider:
+    def __new__(cls, path: str, interval: int = 60) -> ConfigProvider: ...
+    path: str
+    interval: int
+
 class Client:
     def __init__(self, *args, **kwargs) -> None: ...
     def admin_change_password(self, username: str, password: str, policy: dict = ...) -> None: ...

doc/aerospike.rst

@@ -362,6 +362,11 @@ Only the `hosts` key is required; the rest of the keys are optional.
             (Optional) A defined user with roles in the cluster. See :meth:`admin_create_user`.
         * **password** (:class:`str`)
             (Optional) The password will be hashed by the client using bcrypt.
+        * **config_provider** (:class:`aerospike.ConfigProvider`)
+            (Optional) Dynamic configuration provider.
+
+            An alternate way to enable dynamic config is to set environment variable ``AEROSPIKE_CLIENT_CONFIG_URL``
+            to the path of the config file before running the application.
         * **lua** (:class:`dict`)
             (Optional) Contains the paths to two types of Lua modules
 
@@ -446,6 +451,8 @@ Only the `hosts` key is required; the rest of the keys are optional.
             * **txn_roll** (:class:`dict`)
                 Default transaction policy when rolling the transaction records forward (commit) or back (abort) in a batch.
                 Contains :ref:`aerospike_batch_policies`.
+            * **metrics** (:class:`~aerospike_helpers.metrics.MetricsPolicy`)
+                Default metrics policy. Only :py:attr:`~aerospike_helpers.metrics.MetricsPolicy.latency_columns` and :py:attr:`~aerospike_helpers.metrics.MetricsPolicy.latency_shift` will override transaction-level metrics policies.
             * **total_timeout** (:class:`int`)
                 **Deprecated**: set this individually in the :ref:`aerospike_policies` dictionaries.
 

doc/config_provider.rst

@@ -0,0 +1,35 @@
+.. _aerospike.ConfigProvider:
+
+.. currentmodule:: aerospike
+
+===================================================================
+:class:`aerospike.ConfigProvider` --- Dynamic config provider class
+===================================================================
+
+Methods
+=======
+
+.. class:: ConfigProvider
+
+    Dynamic configuration provider. Determines how to retrieve cluster policies.
+
+    An instance of this class is immutable.
+
+    For the ``interval`` parameter, an unsigned 32-bit integer must be passed.
+
+    :param path: Dynamic configuration file path. Cluster policies will be read from the yaml file at cluster initialization and whenever the file changes. The policies fields in the file override all command policies.
+    :type path: str
+    :param interval: Check dynamic configuration file for changes after this number of cluster tend iterations. Defaults to ``60``.
+    :type interval: int, optional
+
+    .. py:attribute:: path
+
+        This attribute is read-only.
+
+        :type: str
+
+    .. py:attribute:: interval
+
+        This attribute is read-only.
+
+        :type: int

doc/index.rst

@@ -60,6 +60,7 @@ Class                                Description
 :ref:`aerospike.geojson`             Handles GeoJSON type data.
 :ref:`aerospike.KeyOrderedDict`      Key ordered dictionary
 :ref:`aerospike.Transaction`         Transaction
+:ref:`aerospike.ConfigProvider`      Dynamic config provider
 =================================    ===========
 
 In addition, the :ref:`Data_Mapping` page explains how **Python** types map to **Aerospike Server** types.
@@ -81,6 +82,7 @@ Content
     geojson
     key_ordered_dict
     transaction
+    config_provider
     predicates
     exception
     aerospike_helpers

doc/spelling_wordlist.txt

@@ -87,3 +87,4 @@ msg
 func
 bcrypt
 namespaces
+yaml

setup.py

@@ -104,7 +104,8 @@
     'crypto',
     'pthread',
     'm',
-    'z'
+    'z',
+    'yaml'
 ]
 
 ##########################

src/include/config_provider.h

@@ -0,0 +1,3 @@
+#include <Python.h>
+
+PyTypeObject *AerospikeConfigProvider_Ready();

src/include/conversions.h

@@ -232,6 +232,7 @@ PyObject *create_py_cluster_from_as_cluster(as_error *error_p,
 PyObject *create_py_node_from_as_node(as_error *error_p,
                                       struct as_node_s *node);
 
+// obj must be non-NULL
 // Checks if pyobject is an instance of a class type defined in aerospike_helpers or one of its submodules
 // If expected_submodule_name is NULL, the type is expected to be defined directly in the aerospike_helpers package
 // If is_subclass_instance is true, we expect the instance's type to directly inherit from aerospike_helpers.<expected_submodule_name>.<expected_type_name>
@@ -245,3 +246,8 @@ PyObject *create_class_instance_from_module(as_error *error_p,
                                             const char *module_name,
                                             const char *class_name,
                                             PyObject *py_arg);
+
+// Error indicator must always be checked after this call
+// Constructor parameter name needed for constructing error message
+uint32_t convert_pyobject_to_uint32_t(PyObject *pyobject,
+                                      const char *param_name_of_pyobj);

src/include/policy.h

@@ -287,9 +287,11 @@ as_status pyobject_to_batch_remove_policy(AerospikeClient *self, as_error *err,
                                           as_exp *exp_list,
                                           as_exp **exp_list_p);
 
-as_status init_and_set_as_metrics_policy_using_pyobject(
-    as_error *err, PyObject *py_metrics_policy,
-    as_metrics_policy *metrics_policy);
+// metrics_policy must be declared already
+// py_metrics_policy must be non-NULL
+as_status
+set_as_metrics_policy_using_pyobject(as_error *err, PyObject *py_metrics_policy,
+                                     as_metrics_policy *metrics_policy);
 
 typedef struct {
     // Use listener name for error messages

src/include/types.h

@@ -26,6 +26,8 @@
 #include <aerospike/as_bin.h>
 #include <aerospike/as_operations.h>
 #include <aerospike/as_txn.h>
+#include <aerospike/as_config.h>
+
 #include "pool.h"
 
 #define AEROSPIKE_MODULE_NAME "aerospike"
@@ -108,3 +110,10 @@ typedef struct {
 } AerospikeTransaction;
 
 extern PyTypeObject AerospikeTransaction_Type;
+
+typedef struct {
+    PyObject_HEAD char *path;
+    uint32_t interval;
+} AerospikeConfigProvider;
+
+extern PyTypeObject AerospikeConfigProvider_Type;

src/main/aerospike.c

@@ -33,6 +33,7 @@
 #include "nullobject.h"
 #include "cdt_types.h"
 #include "transaction.h"
+#include "config_provider.h"
 
 #include <aerospike/as_operations.h>
 #include <aerospike/as_log_macros.h>
@@ -556,6 +557,7 @@ static struct type_name_to_creation_method py_module_types[] = {
     {"CDTWildcard", AerospikeWildcardObject_Ready},
     {"CDTInfinite", AerospikeInfiniteObject_Ready},
     {"Transaction", AerospikeTransaction_Ready},
+    {"ConfigProvider", AerospikeConfigProvider_Ready},
 };
 
 PyMODINIT_FUNC PyInit_aerospike(void)

src/main/client/metrics.c

@@ -38,20 +38,29 @@ PyObject *AerospikeClient_EnableMetrics(AerospikeClient *self, PyObject *args,
         goto RAISE_EXCEPTION_WITHOUT_AS_ERROR;
     }
 
-    as_status status = init_and_set_as_metrics_policy_using_pyobject(
-        &err, py_metrics_policy, &metrics_policy);
-    if (status != AEROSPIKE_OK) {
-        goto RAISE_EXCEPTION_USING_AS_ERROR;
+    as_metrics_policy *metrics_policy_ref;
+    if (py_metrics_policy == NULL || py_metrics_policy == Py_None) {
+        // Use C client's config metrics policy
+        metrics_policy_ref = NULL;
+    }
+    else {
+        as_metrics_policy_init(&metrics_policy);
+        as_status status = set_as_metrics_policy_using_pyobject(
+            &err, py_metrics_policy, &metrics_policy);
+        if (status != AEROSPIKE_OK) {
+            goto RAISE_EXCEPTION_USING_AS_ERROR;
+        }
+        metrics_policy_ref = &metrics_policy;
     }
 
     // 2 scenarios:
     // 1. If the user does not pass their own MetricsListeners object to client.enable_metrics(), udata is NULL
     // 2. Otherwise, udata is non-NULL and set to heap-allocated PyListenerData
     bool free_udata_as_py_listener_data =
-        metrics_policy.metrics_listeners.udata != NULL;
+        metrics_policy_ref && metrics_policy.metrics_listeners.udata != NULL;
 
     Py_BEGIN_ALLOW_THREADS
-    aerospike_enable_metrics(self->as, &err, &metrics_policy);
+    aerospike_enable_metrics(self->as, &err, metrics_policy_ref);
     Py_END_ALLOW_THREADS
 
     if (err.code != AEROSPIKE_OK) {