Thread unsafe code in XACC

[miniskar]

XACC is being used in QIREE and we are trying to launch multiple instances of QIREE with XACC using multiple threads.

XACC initialization in multilple threads is causing trouble due to simultaneous access of global variables used in XACC for service registry, etc.

To reproduce this issue, you need to build XACC, QIREE, and IRIS, which is demonstrated using IRIS quantum application where it creates 64 tasks, which can be executed on multiple QIREE devices simultaneously.

apptainer and Makefile used to reproduce this issue is given here.

apptainer.def

# Author: Narasinga Rao Miniskar
# Contact: [email protected]

Bootstrap: docker
From: ubuntu:22.04
Stage: build

%labels
    Author Narasinga Rao Miniskar
    Version v1.0

%environment
    export DEBIAN_FRONTEND=noninteractive
    export QIREE=/software/qiree
    export LANG=C.UTF-8
    export LC_ALL=C.UTF-8
    export IRIS=/software/iris/install
    export PATH=/software/iris/install/bin:/software/qiree/build/bin:/root/.xacc/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH
    export LD_LIBRARY_PATH=/software/iris/install/lib:/software/iris/install/lib64:/software/qiree/build/lib:/root/.xacc/lib:$LD_LIBRARY_PATH
    export PYTHONPATH=/software/iris/install
    export CPATH=/software/iris/install/include

%post
    echo "Installing prerequisites..."
    apt -y update && \
    apt-get install -y apt-transport-https ca-certificates gnupg software-properties-common wget lsb-release

    wget https://github.com/Kitware/CMake/releases/download/v3.31.5/cmake-3.31.5-linux-x86_64.sh
    echo y | bash cmake-3.31.5-linux-x86_64.sh --skip-license --prefix=/usr/local --exclude-subdir
    apt -y update && \
    apt install -y --no-install-recommends ninja-build \
                   libssl-dev \
                   libcurl4-openssl-dev \
                   autoconf automake autotools-dev \
                   build-essential \
                   python3 \
                   libpython3-dev \
                   python3-venv \
                   python3-pip \
                   libblas-dev \
                   liblapack-dev \
                   libncurses-dev \
                   gcc-11 \
                   g++-11 \
                   gdb \
                   lld \
                   ccache make \
                   git \
                   doxygen pandoc lcov \
                   vim \
                   pybind11-dev \
                   unzip \
                   curl \
                   wget \
                   lsb-release \
                   software-properties-common \
                   gpg \
                   libzstd-dev && \
    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \
    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 && \
    update-alternatives --set gcc /usr/bin/gcc-11 && \
    update-alternatives --set g++ /usr/bin/g++-11

    # Build XACC
    mkdir -p /software && cd /software && \
    git clone https://github.com/ORNL-QCI/xacc.git && \
    mkdir -p /software/xacc/build && cd /software/xacc/build && \
    cmake -GNinja /software/xacc -DXACC_BUILD_TESTS=TRUE -DCMAKE_BUILD_TYPE=Debug -DXACC_REMOTE_ACCELERATORS=ON -DQIREE_MINIMAL_BUILD=ON && \
    cmake --build . --target install && \
    ctest --output-on-failure

    # Install LLVM
    mkdir -p /software/llvm && cd /software/llvm && \
    wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && ./llvm.sh 19
    apt install -y clang-format-19 libomp-19-dev
    ln -s /usr/bin/clang-19 /usr/bin/clang
    ln -s /usr/bin/clang++-19 /usr/bin/clang++

    #git clone https://github.com/ORNL-QCI/qiree.git
    # Build QIREE
    mkdir -p /software && cd /software && \
    git clone https://github.com/miniskar/qiree.git && \
    mkdir -p /software/qiree/build && cd /software/qiree/build && \
    cmake -GNinja /software/qiree -DCMAKE_PREFIX_PATH=$HOME/.xacc 
    cmake --build .
    cd ../../

    git clone https://code.ornl.gov/brisbane/iris.git -b hunter.julia.v1  
    cd iris
    IRIS_INSTALL_ROOT=/software/iris/install  bash build.sh
    cd ..

%runscript
    echo "Container initialized. Customize the runscript section to add your commands."

Build apptainer using below command.

$ apptainer build --fakeroot qiree.sif apptainer.def

Enter into apptainer shell

$ apptainer shell --shell /bin/bash qiree.sif

Run the application

$ cd /software/iris/apps/quantum
$ IRIS_ARCHS=qiree IRIS_QIREE_DEVICE_FACTOR=2 gdb --args python3 test_qiree_iris.py

It throws segmentation fault.

[Thread 0x7fff50db3640 (LWP 3871475) exited]
4argv0x555556141508
terminate called after throwing an instance of 'std::runtime_error'
  what():  The bundle context is no longer valid

Thread 66 "python3" received signal SIGABRT, Aborted.
[Switching to Thread 0x7fff51db5640 (LWP 3871473)]
0x00007ffff7cee9fc in pthread_kill () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) up
#1  0x00007ffff7c9a476 in raise () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) up
#2  0x00007ffff7c807f3 in abort () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) up
#3  0x00007fff55e60b9e in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6
(gdb) up
#4  0x00007fff55e6c20c in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6
(gdb) up
#5  0x00007fff55e6c277 in std::terminate() () from /lib/x86_64-linux-gnu/libstdc++.so.6
(gdb) up
#6  0x00007fff55e6c4d8 in __cxa_throw () from /lib/x86_64-linux-gnu/libstdc++.so.6
(gdb) up
#7  0x00007fff531146d6 in cppmicroservices::BundleContextPrivate::CheckValid() const [clone .cold] () from /root/.xacc/lib/libCppMicroServices.so.4.0.0
(gdb) up
#8  0x00007fff5316eb66 in cppmicroservices::BundleContext::GetServiceReferences(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /root/.xacc/lib/libCppMicroServices.so.4.0.0
(gdb) up
#9  0x00007fff53702020 in cppmicroservices::BundleContext::GetServiceReferences<xacc::OptionsProvider> (this=0x55555612a878, filter="") at /software/xacc/tpls/cppmicroservices/framework/include/cppmicroservices/BundleContext.h:445
445         BaseVectorT serviceRefs = GetServiceReferences(clazz, filter);
(gdb) up
#10 0x00007fff536ff68e in xacc::ServiceRegistry::getServices<xacc::OptionsProvider> (this=0x55555612a850) at /software/xacc/xacc/service/ServiceRegistry.hpp:161
161         auto allServiceRefs = context.GetServiceReferences<ServiceInterface>();
(gdb) up
#11 0x00007fff536fa5cc in xacc::ServiceRegistry::getRegisteredOptions[abi:cxx11]() (this=0x55555612a850) at /software/xacc/xacc/service/ServiceRegistry.hpp:187
187         auto optionProviders = getServices<xacc::OptionsProvider>();
(gdb) up
#12 0x00007fff536f91e4 in xacc::getRegisteredOptions[abi:cxx11]() () at /software/xacc/xacc/service/xacc_service.cpp:74
74        return serviceRegistry->getRegisteredOptions();

bell.ll file is added. please place this file /software/iris/apps/quantum folder.

; ModuleID = 'Bell'
source_filename = "Bell"

%Qubit = type opaque
%Result = type opaque

define void @main() #0 {
entry:
  call void @__quantum__qis__h__body(%Qubit* null)
  call void @__quantum__qis__cnot__body(%Qubit* null, %Qubit* inttoptr (i64 1 to %Qubit*))
  call void @__quantum__qis__mz__body(%Qubit* null, %Result* null)
  call void @__quantum__qis__mz__body(%Qubit* inttoptr (i64 1 to %Qubit*), %Result* inttoptr (i64 1 to %Result*))
  call void @__quantum__rt__array_record_output(i64 2, i8* null)
  call void @__quantum__rt__result_record_output(%Result* null, i8* null)
  call void @__quantum__rt__result_record_output(%Result* inttoptr (i64 1 to %Result*), i8* null)
  ret void
}

declare void @__quantum__qis__h__body(%Qubit*)

declare void @__quantum__qis__cnot__body(%Qubit*, %Qubit*)

declare void @__quantum__qis__mz__body(%Qubit*, %Result* writeonly) #1

declare void @__quantum__rt__array_record_output(i64, i8*)

declare void @__quantum__rt__result_record_output(%Result*, i8*)

attributes #0 = { "entry_point" "num_required_qubits"="2" "num_required_results"="2" "output_labeling_schema" "qir_profiles"="custom" }
attributes #1 = { "irreversible" }

!llvm.module.flags = !{!0, !1, !2, !3}

!0 = !{i32 1, !"qir_major_version", i32 1}
!1 = !{i32 7, !"qir_minor_version", i32 0}
!2 = !{i32 1, !"dynamic_qubit_management", i1 false}
!3 = !{i32 1, !"dynamic_result_management", i1 false}

[ausbin]

Akihiro Hayashi (@ahayashi) made a PR that might help with this, but it never got merged: eclipse-xacc#455

This is the paper he wrote about it: https://ieeexplore.ieee.org/abstract/document/10196570