Merge pull request #3086 from QMCPACK/rc_3110

Rc 3110

Author: prckent

.github/pull_request_template.md

@@ -14,7 +14,8 @@ _Delete the items that do not apply_
 - Code style update (formatting, renaming)
 - Refactoring (no functional changes, no api changes)
 - Build related changes
-- Documentation content changes
+- Testing changes (e.g. new unit/integration/performance tests)
+- Documentation changes
 - Other (please describe):
 
 ### Does this introduce a breaking change?

.github/workflows/ci-github-actions.yaml

@@ -0,0 +1,74 @@
+
+name: GitHub Actions CI
+
+on: 
+  push:
+    branches: 
+    - develop
+  pull_request:
+    branches: 
+    - develop
+
+jobs:
+
+  linux:
+    runs-on: ubuntu-latest
+    container: ${{ matrix.container }}
+    env:
+      GH_JOBNAME: ${{ matrix.jobname }}
+      GH_OS: Linux
+    strategy:
+      fail-fast: false
+      matrix:
+        jobname: [
+          gcc-openmpi-real-coverage,
+          gcc-openmpi-complex-coverage,
+          clang-openmpi-real-asan,
+          clang-openmpi-real-ubsan
+        ]
+        include:
+        - jobname: gcc-openmpi-real-coverage
+          container: 
+            image: williamfgc/qmcpack-ci:ubuntu20-openmpi
+            options: -u 1001
+        
+        - jobname: gcc-openmpi-complex-coverage
+          container: 
+            image: williamfgc/qmcpack-ci:ubuntu20-openmpi
+            options: -u 1001
+            
+        - jobname: clang-openmpi-real-asan
+          container: 
+            image: williamfgc/qmcpack-ci:ubuntu20-openmpi
+            options: -u 1001
+        
+        - jobname: clang-openmpi-real-ubsan
+          container: 
+            image: williamfgc/qmcpack-ci:ubuntu20-openmpi
+            options: -u 1001
+
+    steps:
+    - name: Checkout Action
+      uses: actions/checkout@v1
+
+    - name: Configure
+      run: tests/test_automation/github-actions/ci/run_step.sh configure
+
+    - name: Build
+      run: tests/test_automation/github-actions/ci/run_step.sh build
+
+    - name: Test
+      run: tests/test_automation/github-actions/ci/run_step.sh test
+    
+    - name: Coverage
+      if: contains(matrix.jobname, 'coverage')
+      run: tests/test_automation/github-actions/ci/run_step.sh coverage
+    
+    - name: Upload Coverage
+      if: contains(matrix.jobname, 'coverage')
+      uses: codecov/codecov-action@v1
+      with:
+        file:  ../qmcpack-build/coverage.xml
+        flags: tests-deterministic # optional
+        name: codecov-QMCPACK # optional
+        fail_ci_if_error: true # optional (default = false)

.gitignore

@@ -9,3 +9,11 @@ qmcpack_manual.pdf
 .DS_Store
 /build_*/
 docs/_build
+nexus/tests/unit/*output
+tests/solids/NiO_a4_e48_pp/NiO-fcc-supertwist111-supershift000-S1.h5
+
+# Eclipse IDE
+.cproject
+.project
+.settings/
+.pydevproject

CHANGELOG.md

@@ -2,6 +2,51 @@
 
 Notable changes to QMCPACK are documented in this file.
 
+## [3.11.0] - 2021-04-09
+
+### Notes
+
+This release includes a large number of refinements to QMCPACK and the supporting ecosystem. These include support for the latest version of
+Quantum ESPRESSO, new capabilities in AFQMC, space-warp transformation for forces, numerous bug fixes, user-requested feature improvements,
+and further upgrades to the test system.
+
+* Quantum ESPRESSO (QE) v6.7 support. [\#2927](https://github.com/QMCPACK/qmcpack/pull/2927).
+* Detect and automatically use patched version of QE found on the PATH. [\#2974](https://github.com/QMCPACK/qmcpack/pull/2974).
+* Support for global max\_seconds and STOP file to cleanly halt QMCPACK during a run. [\#3028](https://github.com/QMCPACK/qmcpack/pull/3028).
+* Freezing of two-body Jastrow parameters in optimization works. [\#2814](https://github.com/QMCPACK/qmcpack/issues/2814).
+* Multideterminant code now works with only alpha determinants \(no down electrons\). [\#2698](https://github.com/QMCPACK/qmcpack/issues/2698).
+* High l-momentum channels as local channels in ECPs work. [\#2920](https://github.com/QMCPACK/qmcpack/pull/2920).
+* Space Warp Transformation for ZVZB Forces. [\#2828](https://github.com/QMCPACK/qmcpack/pull/2828).
+* Important bug fixes in legacy CUDA implementation causing incorrect energies. [\#2883](https://github.com/QMCPACK/qmcpack/pull/2883).
+* Implemented DLA in legacy CUDA. [\#2887](https://github.com/QMCPACK/qmcpack/pull/2887).
+* Updates to support CUDA 11.2.1 e.g. [\#2950](https://github.com/QMCPACK/qmcpack/pull/2950).
+* AFQMC supports energy estimator with different Hamiltonian \(from propagation\). [\#2795](https://github.com/QMCPACK/qmcpack/pull/2795).
+* Trial wavefunction optimization with spin-orbit supported. [\#3034](https://github.com/QMCPACK/qmcpack/pull/3034).
+* ppconvert executable automatically built when configured. [\#2904](https://github.com/QMCPACK/qmcpack/pull/2904).
+* Tests added for ppconvert. [\#2929](https://github.com/QMCPACK/qmcpack/issues/2929).
+* Fixed SIMD alignment for AVX512 on some systems. [\#2981](https://github.com/QMCPACK/qmcpack/pull/2981).
+* Improved wavefunction restart logic in AFQMC. [\#2942](https://github.com/QMCPACK/qmcpack/pull/2942).
+* Spin-density supported in batched code. [\#2840](https://github.com/QMCPACK/qmcpack/pull/2840).
+* Reduced I/O operations during cmake. [\#2808](https://github.com/QMCPACK/qmcpack/pull/2808).
+* Improved detection of unsupported-by-Intel combinations of Intel compilers and libstdc++. [\#2794](https://github.com/QMCPACK/qmcpack/pull/2794).
+* Initial support for Andes at OLCF. [\#3073](https://github.com/QMCPACK/qmcpack/pull/3073).
+* Deterministic tests expanded in scope and made reliable for more build types and compilers.
+* Various minor bug fixes and feature improvements based on user requests for both real-space and AFQMC.
+* Improved error handling throughout.
+* Numerous performance improvements, expansion of tests, and bug fixes to the batched VMC and DMC codes. Reasonable but not optimal GPU acceleration can now be achieved for spline-based wavefunctions.
+
+### NEXUS
+
+* Support AMD nodes on Cori. [\#2809](https://github.com/QMCPACK/qmcpack/pull/2809).
+* Interface for RMG code. [\#2932](https://github.com/QMCPACK/qmcpack/pull/2932).
+* Added h-channel to list of possible local channels in pseudopotential. [\#2915](https://github.com/QMCPACK/qmcpack/pull/2915).
+* Allow non spin-specific occupations in case of noncollinear. [\#2957](https://github.com/QMCPACK/qmcpack/pull/2957).
+* More robust handling of QE output when printed eigenvalues touch. [\#3042](https://github.com/QMCPACK/qmcpack/pull/3042).
+* Fixed type check for reblock\_factors in qmc-fit. [\#2830](https://github.com/QMCPACK/qmcpack/pull/2830).
+* Fixed a Jastrow read error/warning, add several QE inputs. [\#2819](https://github.com/QMCPACK/qmcpack/pull/2819).
+* Fixed tests on Summit. [\#2983](https://github.com/QMCPACK/qmcpack/pull/2983).
+* Fixed module overwrite bug in qmca. [\#2802](https://github.com/QMCPACK/qmcpack/pull/2802).
+
 ## [3.10.0] - 2020-11-10
 
 ### Notes

CMake/CheckSIMDAlignment.cmake

@@ -0,0 +1,12 @@
+# Check if AVX512 is activated in the compilation
+# Since cross-compiling is not unusual on HPC systems (Cray),
+# try_compile is robust against 
+try_compile(CXX_COMPILER_HAVE_AVX512_MACRO ${CMAKE_BINARY_DIR}
+      ${PROJECT_CMAKE}/try_compile_sources/check_AVX512.cpp
+      CMAKE_FLAGS "${CMAKE_CXX_FLAGS}")
+
+if (CXX_COMPILER_HAVE_AVX512_MACRO)
+  set(default_alignment 64)
+else()
+  set(default_alignment 32)
+endif()

CMake/ClangCompilers.cmake

@@ -4,6 +4,10 @@ IF ( CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0 )
   MESSAGE(FATAL_ERROR "Requires clang 7.0 or higher ")
 ENDIF()
 
+IF ( CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 11.0.0 AND QMC_CXX_STANDARD EQUAL 17 AND BUILD_AFQMC )
+  MESSAGE(FATAL_ERROR "Avoid Clang 11.0.0 which cannot compile AFQMC properly with C++17!")
+ENDIF()
+
 # Set the std
 SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -std=c99")
 
@@ -12,10 +16,14 @@ IF(QMC_OMP)
   SET(ENABLE_OPENMP 1)
   IF(ENABLE_OFFLOAD AND NOT CMAKE_SYSTEM_NAME STREQUAL "CrayLinuxEnvironment")
     SET(OFFLOAD_TARGET "nvptx64-nvidia-cuda" CACHE STRING "Offload target architecture")
+    SET(OPENMP_OFFLOAD_COMPILE_OPTIONS "-fopenmp-targets=${OFFLOAD_TARGET}")
+
     IF(DEFINED OFFLOAD_ARCH)
-      SET(CLANG_OPENMP_OFFLOAD_FLAGS "-fopenmp-targets=${OFFLOAD_TARGET} -Xopenmp-target=${OFFLOAD_TARGET} -march=${OFFLOAD_ARCH}")
-    ELSE()
-      SET(CLANG_OPENMP_OFFLOAD_FLAGS "-fopenmp-targets=${OFFLOAD_TARGET}")
+      SET(OPENMP_OFFLOAD_COMPILE_OPTIONS "${OPENMP_OFFLOAD_COMPILE_OPTIONS} -Xopenmp-target=${OFFLOAD_TARGET} -march=${OFFLOAD_ARCH}")
+    ENDIF()
+
+    IF(OFFLOAD_TARGET MATCHES "nvptx64")
+      SET(OPENMP_OFFLOAD_COMPILE_OPTIONS "${OPENMP_OFFLOAD_COMPILE_OPTIONS} -Wno-unknown-cuda-version")
     ENDIF()
 
     # Intel clang compiler needs a different flag for the host side OpenMP library when offload is used.
@@ -110,12 +118,6 @@ ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64" OR CMAKE_SYSTEM_PROCESSOR MATCHES
   ENDIF()
 ENDIF()
 
-# Add OpenMP offload flags
-# This step is intentionally put after the -march parsing for CPUs.
-IF(DEFINED CLANG_OPENMP_OFFLOAD_FLAGS)
-  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CLANG_OPENMP_OFFLOAD_FLAGS}")
-ENDIF()
-
 # Add static flags if necessary
 IF(QMC_BUILD_STATIC)
     SET(CMAKE_CXX_LINK_FLAGS " -static")
@@ -139,23 +141,3 @@ IF(XRAY_PROFILE)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${XRAY_FLAGS}")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${XRAY_FLAGS}")
 ENDIF(XRAY_PROFILE)
-
-SET(LLVM_SANITIZE_ADDRESS FALSE CACHE BOOL "Use llvm address sanitizer library")
-MARK_AS_ADVANCED(LLVM_SANITIZE_ADDRESS)
-IF(LLVM_SANITIZE_ADDRESS)
-  SET(CMAKE_C_FLAGS "-fno-omit-frame-pointer -fsanitize=address -fsanitize-address-use-after-scope ${CMAKE_C_FLAGS}")
-  SET(CMAKE_CXX_FLAGS "-fno-omit-frame-pointer -fsanitize=address -fsanitize-address-use-after-scope ${CMAKE_CXX_FLAGS}")
-  SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fno-omit-frame-pointer -fsanitize=address -fsanitize-address-use-after-scope")
-  SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fno-omit-frame-pointer -fsanitize=address -fsanitize-address-use-after-scope")
-ENDIF(LLVM_SANITIZE_ADDRESS)
-
-# Don't expect this to be useful unless you have msan instrumented all libraries
-SET(LLVM_SANITIZE_MEMORY FALSE CACHE BOOL "Use llvm memory sanitizer library")
-MARK_AS_ADVANCED(LLVM_SANITIZE_MEMORY)
-IF(LLVM_SANITIZE_MEMORY)
-  SET(LLVM_BLACKLIST_SANITIZE_MEMORY "-fsanitize-blacklist=${PROJECT_SOURCE_DIR}/llvm_misc/memory_sanitizer_blacklist.txt")
-  SET(CMAKE_C_FLAGS_DEBUG "-fsanitize=memory ${LLVM_BLACKLIST_SANITIZE_MEMORY} ${CMAKE_C_FLAGS_DEBUG}")
-  SET(CMAKE_CXX_FLAGS_DEBUG "-fsanitize=memory ${LLVM_BLACKLIST_SANITIZE_MEMORY} ${CMAKE_CXX_FLAGS_DEBUG}")
-  SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=memory ${LLVM_BLACKLIST_SANITIZE_MEMORY}")
-  SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=memory ${LLVM_BLACKLIST_SANITIZE_MEMORY}")
-ENDIF(LLVM_SANITIZE_MEMORY)

CMake/FindQE.cmake

@@ -0,0 +1,22 @@
+# Locate QE via the patched pw2qmcpack.x
+# Require both to be present to consider QE_FOUND
+# Take QE_BIN as hint for location
+
+FIND_PATH(QE_PW_DIR pw.x HINTS ${QE_BIN})
+FIND_PATH(QE_PW2Q_DIR pw2qmcpack.x HINTS ${QE_BIN})
+
+SET(QE_FOUND FALSE)
+IF(QE_PW2Q_DIR AND QE_PW_DIR)
+  IF ( NOT (QE_PW2Q_DIR STREQUAL QE_PW_DIR) )
+    MESSAGE(WARNING "Found pw.x and pw2qmcpack.x in different locations, ${QE_PW_DIR} and ${QE_PW2Q_DIR}, verify this is intentional.")
+  ENDIF()
+  #MESSAGE(STATUS "QE_PW2Q_DIR=${QE_PW2Q_DIR}")
+  #MESSAGE(STATUS "QE_PW_DIR=${QE_PW_DIR}")
+  SET(QE_FOUND TRUE)
+ENDIF()
+
+MARK_AS_ADVANCED(
+   QE_PW2Q_DIR
+   QE_PW_DIR
+   QE_FOUND
+)

CMake/GNUCompilers.cmake

@@ -1,6 +1,6 @@
 # Check compiler version
-IF ( CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0 )
-MESSAGE(FATAL_ERROR "Requires gcc 5.0 or higher ")
+IF ( CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0 )
+MESSAGE(FATAL_ERROR "Requires gcc 7.0 or higher ")
 ENDIF()
 
 # Set the std
@@ -11,6 +11,10 @@ IF(QMC_OMP)
   SET(ENABLE_OPENMP 1)
   SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -fopenmp")
   SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
+  IF(ENABLE_OFFLOAD AND NOT CMAKE_SYSTEM_NAME STREQUAL "CrayLinuxEnvironment")
+    SET(OFFLOAD_TARGET "nvptx-none" CACHE STRING "Offload target architecture")
+    SET(OPENMP_OFFLOAD_COMPILE_OPTIONS "-foffload=${OFFLOAD_TARGET} -foffload=\"-lm -latomic\"")
+  ENDIF()
 ENDIF(QMC_OMP)
 
 # Set gnu specific flags (which we always want)

CMake/IBMCompilers.cmake

@@ -26,10 +26,10 @@ SET( CMAKE_CXX_FLAGS_RELWITHDEBINFO "-g -O3" )
 
 IF(QMC_OMP)
   SET(ENABLE_OPENMP 1)
+  SET(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} -qsmp=omp")
+  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qsmp=omp")
   IF(ENABLE_OFFLOAD)
-    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qsmp=omp -qoffload")
-  ELSE(ENABLE_OFFLOAD)
-    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qsmp=omp")
+    set(OPENMP_OFFLOAD_COMPILE_OPTIONS "-qoffload")
   ENDIF(ENABLE_OFFLOAD)
 ELSE(QMC_OMP)
   SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qnothreaded")

CMake/PGICompilers.cmake

@@ -9,7 +9,16 @@ SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -c99")
 IF(QMC_OMP)
   SET(ENABLE_OPENMP 1)
   SET(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} -mp=allcores")
-  SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mp=allcores")
+  IF(ENABLE_OFFLOAD AND NOT CMAKE_SYSTEM_NAME STREQUAL "CrayLinuxEnvironment")
+    MESSAGE(WARNING "QMCPACK OpenMP offload is not ready for NVIDIA HPC compiler.")
+    IF(NOT DEFINED OFFLOAD_ARCH)
+      MESSAGE(FATAL_ERROR "NVIDIA HPC compiler requires -gpu=ccXX option set based on the target GPU architecture! "
+                          "Please add -DOFFLOAD_ARCH=ccXX to cmake. For example, cc70 is for Volta.")
+    ENDIF()
+    SET(OPENMP_OFFLOAD_COMPILE_OPTIONS "-mp=gpu -gpu=${OFFLOAD_ARCH}")
+  ELSE()
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mp=allcores")
+  ENDIF()
 ENDIF(QMC_OMP)
 
 ADD_DEFINITIONS( -Drestrict=__restrict__ )

CMake/TestCxx14Library.cmake

@@ -27,7 +27,7 @@ if (NOT CXX14_LIBRARY_OKAY)
   set(COMPILE_FAIL_OUTPUT cpp14_compile_fail.txt)
   file(WRITE "${CMAKE_BINARY_DIR}/${COMPILE_FAIL_OUTPUT}" "${COMPILE_OUTPUT}")
 
-  message(STATUS "C++ 14 standard library support not found")
+  message(STATUS "C++14 standard library support not found")
   message("compiler is ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
   if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") 
     message("Compiler detected is g++.\n  Use version 5.0 or newer for a C++14 standard library")
@@ -39,6 +39,5 @@ if (NOT CXX14_LIBRARY_OKAY)
   message("  Output of test compile is in ${COMPILE_FAIL_OUTPUT}")
   message(FATAL_ERROR "stopping")
 else()
-  message(STATUS "C++ 14 standard library supported")
+  message(STATUS "C++14 standard library supported")
 endif()
-

CMake/TestCxx17Library.cmake

@@ -0,0 +1,42 @@
+
+# Test that the compiler is configured with a C++17 standard library
+
+set(TEST_CXX17_SOURCE ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/try_cxx17_library.cpp)
+file(WRITE ${TEST_CXX17_SOURCE}
+"// Test for C++17 standard library support
+#include <variant>
+#include <string>
+
+int main(int argc, char **argv)
+{
+    std::variant<int, float, std::string> intFloatString;
+    return 0;
+}
+")
+
+
+try_compile(CXX17_LIBRARY_OKAY ${CMAKE_BINARY_DIR}
+            ${TEST_CXX17_SOURCE}
+            CXX_STANDARD 17
+            CXX_STANDARD_REQUIRED ON
+            OUTPUT_VARIABLE COMPILE_OUTPUT)
+
+
+if (NOT CXX17_LIBRARY_OKAY)
+  set(COMPILE_FAIL_OUTPUT cpp17_compile_fail.txt)
+  file(WRITE "${CMAKE_BINARY_DIR}/${COMPILE_FAIL_OUTPUT}" "${COMPILE_OUTPUT}")
+
+  message(STATUS "C++17 standard library support not found")
+  message("compiler is ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
+  if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") 
+    message("Compiler detected is g++.\n  Use version 7.0 or newer for C++17 standard library support.")
+  elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+    message("Compiler detected is clang++.\n  If not using libcxx, ensure a g++ version greater than 7.0 is also on the path so that its C++17 library can be used.")
+  elseif(CMAKE_CXX_COMPILER_ID MATCHES "Intel")
+    message("Compiler detected is icpc.\n  Ensure a gcc version greater than 7.0 is also on the path so that its C++17 library can be used.  Or use the -cxxlib switch to point to a newer gcc install.")
+  endif()
+  message("  Output of test compile is in ${COMPILE_FAIL_OUTPUT}")
+  message(FATAL_ERROR "stopping")
+else()
+  message(STATUS "C++17 standard library supported")
+endif()

CMake/Testlibstdc++.cmake

@@ -1,23 +1,8 @@
 
 # Test that if a C++ compiler is compatiable with the libstdc++ in use
 
-set(TEST_LIBSTDCXX_SOURCE ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/try_libstdcxx.cpp)
-file(WRITE ${TEST_LIBSTDCXX_SOURCE}
-"// Test the compatibility between the compiler and the libstdc++ from GNU
-#include <cstdio>
-
-int main(int argc, char **argv)
-{
-#if ( ( __INTEL_COMPILER == 1800 ) && (  _GLIBCXX_RELEASE > 7 ) )
-#error too new libstdc++ from GNU for Intel 18, use GNU version <= 7
-#endif
-    return 0;
-}
-")
-
-
 try_compile(LIBSTDCXX_OKAY ${CMAKE_BINARY_DIR}
-            ${TEST_LIBSTDCXX_SOURCE}
+            ${PROJECT_CMAKE}/try_compile_sources/check_libstdcxx.cpp
             CXX_STANDARD 14
             CXX_STANDARD_REQUIRED ON
             OUTPUT_VARIABLE COMPILE_OUTPUT)

CMake/ctest_script.cmake

@@ -199,10 +199,6 @@ IF ( DEFINED QMC_MIXED_PRECISION )
   SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DQMC_MIXED_PRECISION=${QMC_MIXED_PRECISION}" )
 ENDIF()
 
-IF ( DEFINED ENABLE_SOA )
-  SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DENABLE_SOA=${ENABLE_SOA}" )
-ENDIF()
-
 IF ( DEFINED CUDA_ARCH )
   SET( CTEST_OPTIONS "${CTEST_OPTIONS};-DCUDA_ARCH='${CUDA_ARCH}'" )
 ENDIF()