Use a local librdkafka installation different for every started test

Use test code from the checked out branch instead of the selected CKPy version
Bootstrap the same way as when changing version: with `build.sh`

Author: emasab

tests/soak/bootstrap.sh

@@ -20,7 +20,7 @@ otel_collector_version=0.130.0
 otel_collector_package_url="https://github.com/open-telemetry/"\
 "opentelemetry-collector-releases/releases/download/"\
 "v${otel_collector_version}/otelcol-contrib_${otel_collector_version}_linux_amd64.deb"
-venv=$PWD/venv
+
 
 sudo apt update
 sudo apt install -y git curl wget make gcc g++ zlib1g-dev libssl-dev \
@@ -31,36 +31,12 @@ rm otel_collector_package.deb
 sudo cp otel-config.yaml /etc/otelcol-contrib/config.yaml
 sudo systemctl restart otelcol-contrib
 
-if [[ ! -d confluent-kafka-python ]]; then
-    git clone https://github.com/confluentinc/confluent-kafka-python
-fi
-
-pushd confluent-kafka-python
-
-git checkout $python_branch
-
-echo "Installing librdkafka $librdkafka_branch"
-tools/bootstrap-librdkafka.sh --require-ssl $librdkafka_branch /usr
-rm -rf tmp-build
-
-# echo "Installing interceptors"
-# tools/install-interceptors.sh
+./build.sh $librdkafka_branch $python_branch
 
-echo "Setting up virtualenv in $venv"
-if [[ ! -d $venv ]]; then
-    python3 -m venv $venv
-fi
+venv=$PWD/venv
 source $venv/bin/activate
-
-pip install -U pip
-
-pip install -v .[soaktest]
-
-popd # ..python
-
 echo "Verifying python client installation"
 python -c "import confluent_kafka; print(confluent_kafka.version(), confluent_kafka.libversion())"
-
 deactivate
 
 echo "All done, activate the virtualenv in $venv before running the client:"

tests/soak/build.sh

@@ -11,28 +11,50 @@ fi
 
 set -eu
 
+testdir=$PWD
+mkdir -p $testdir/librdkafka-installation
 
+if [[ ! -d confluent-kafka-python ]]; then
+    git clone https://github.com/confluentinc/confluent-kafka-python
+fi
+
+venv=$PWD/venv
+if [[ ! -d $venv ]]; then
+    echo "Setting up virtualenv in $venv"
+    python3 -m venv $venv
+    source $venv/bin/activate
+    pip install -U pip
+    pip install -r $testdir/../../requirements/requirements-soaktest.txt
+    deactivate
+fi
 
 echo "Building and installing librdkafka $librdkafka_version"
 if [[ ! -d librdkafka ]]; then
     git clone https://github.com/confluentinc/librdkafka.git
 fi
 pushd librdkafka
-sudo make uninstall
 git fetch --tags
 git checkout $librdkafka_version
-./configure --reconfigure
+echo "Configuring librdkafka $librdkafka_version with prefix $testdir/librdkafka-installation"
+./configure --prefix=$testdir/librdkafka-installation
+sudo make uninstall
 make clean
 make -j
-sudo make install
+make install
 popd
 
+export LIBRARY_PATH=$testdir/librdkafka-installation/lib
+export LD_LIBRARY_PATH=$testdir/librdkafka-installation/lib
+export CPLUS_INCLUDE_PATH=$testdir/librdkafka-installation/include
+export C_INCLUDE_PATH=$testdir/librdkafka-installation/include
 
 echo "Building confluent-kafka-python $cflpy_version"
 set +u
 source venv/bin/activate
+python3 -m pip uninstall -y confluent-kafka
 set -u
 pushd confluent-kafka-python
+rm -rf ./build
 git fetch --tags
 git checkout $cflpy_version
 python3 -m pip install .
@@ -41,4 +63,3 @@ popd
 echo ""
 echo "=============================================================================="
 (cd / ; python3 -c 'import confluent_kafka as c; print("python", c.version(), "librdkafka", c.libversion())')
-

tests/soak/run.sh

@@ -2,6 +2,8 @@
 
 source venv/bin/activate
 
+testdir=$PWD
+export LD_LIBRARY_PATH=$testdir/librdkafka-installation/lib
 librdkafka_version=$(python3 -c 'from confluent_kafka import libversion; print(libversion()[0])')
 
 if [[ -z $librdkafka_version ]]; then
@@ -17,7 +19,7 @@ echo "Starting soak client using topic $topic. Logging to $logfile."
 set +x
 # Ignore SIGINT in children (inherited)
 trap "" SIGINT
-time opentelemetry-instrument confluent-kafka-python/tests/soak/soakclient.py -i $TESTID -t $topic -r 80 -f $1 |& tee /dev/tty | bzip2 > $logfile &
+time opentelemetry-instrument $testdir/soakclient.py -i $TESTID -t $topic -r 80 -f $1 |& tee /dev/tty | bzip2 > $logfile &
 PID=$!
 # On SIGINT kill only the first process of the pipe
 onsigint() {