From a8204a50d4d9828a460cd3bb4c06c27842fe8cde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samuel=20K=2E=20Guti=C3=A9rrez?= Date: Tue, 25 Jun 2024 10:49:34 -0600 Subject: [PATCH] Add first cut of qv_thread_scope_split_*. (#179) Add Guillaume's first cut of Edgar's new scope split interfaces with cleanups. Signed-off-by: Samuel K. Gutierrez Co-authored-by: Guillaume Mercier Co-authored-by: Edgar A. Leon <800736+eleon@users.noreply.github.com> --- include/quo-vadis-thread.h | 23 ++++ include/quo-vadis.h | 2 +- src/quo-vadis-thread.cc | 64 +++++++++-- src/qvi-bbuff-rmi.h | 9 ++ src/qvi-context.h | 1 + src/qvi-hwloc.cc | 56 +++++++++ src/qvi-hwloc.h | 11 ++ src/qvi-rmi.cc | 51 +++++++++ src/qvi-rmi.h | 11 ++ src/qvi-scope.cc | 25 ++++ src/qvi-scope.h | 7 ++ tests/CMakeLists.txt | 12 ++ tests/test-pthread-split.c | 229 +++++++++++++++++++++++++++++++++++++ 13 files changed, 492 insertions(+), 9 deletions(-) create mode 100644 tests/test-pthread-split.c diff --git a/include/quo-vadis-thread.h b/include/quo-vadis-thread.h index f5f3d8c6..08f9e36e 100644 --- a/include/quo-vadis-thread.h +++ b/include/quo-vadis-thread.h @@ -97,6 +97,29 @@ qv_pthread_create( qv_context_t *ctx, qv_scope_t *scope ); + + +int +qv_thread_scope_split_at( + qv_context_t *ctxt, + qv_scope_t *scope, + qv_hw_obj_type_t type, + int *color_array, + int nthreads, + qv_scope_t ***subscope +); + + +int +qv_thread_scope_split( + qv_context_t *ctxt, + qv_scope_t *scope, + int npieces, + int *color_array, + int nthreads, + qv_scope_t ***subscope +); + #else /** * Layout for fine-grain binding diff --git a/include/quo-vadis.h b/include/quo-vadis.h index b0a3ab41..f1acec90 100644 --- a/include/quo-vadis.h +++ b/include/quo-vadis.h @@ -106,7 +106,7 @@ typedef enum { } qv_hw_obj_type_t; /** - * Binding string representaiton formats. + * Binding string representation formats. */ typedef enum { QV_BIND_STRING_AS_BITMAP = 0, diff --git a/src/quo-vadis-thread.cc b/src/quo-vadis-thread.cc index a8ba4698..cb0db738 100644 --- a/src/quo-vadis-thread.cc +++ b/src/quo-vadis-thread.cc @@ -1,4 +1,4 @@ - /* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */ +/* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2020-2024 Triad National Security, LLC * All rights reserved. @@ -23,6 +23,11 @@ #include "qvi-group-thread.h" // IWYU pragma: keep #include "qvi-utils.h" + +#include "qvi-scope.h" +//#include "qvi-hwpool.h" +//#include "qvi-map.h" + #ifdef OPENMP_FOUND #include #endif @@ -76,6 +81,51 @@ qv_thread_context_create( return rc; } +int +qv_thread_scope_split( + qv_context_t *ctxt, + qv_scope_t *scope, + int npieces, + int *color_array, + int nthreads, + qv_scope_t ***subscope +){ + int rc = QV_SUCCESS; + + QVI_UNUSED(ctxt); + + qv_hw_obj_type_t type; + + rc = qvi_scope_obj_type(scope, npieces, &type); + if (rc != QV_SUCCESS) { + rc = QV_ERR_INVLD_ARG; + goto out; + } + //fprintf(stdout,"=================== Type for split is %i\n",(int)type); + rc = qvi_scope_ksplit_at(scope, type, color_array, nthreads, subscope); + + out: + return rc; +} + +int +qv_thread_scope_split_at( + qv_context_t *ctxt, + qv_scope_t *scope, + qv_hw_obj_type_t type, + int *color_array, + int nthreads, + qv_scope_t ***subscope +){ + int rc = QV_SUCCESS; + + QVI_UNUSED(ctxt); + + rc = qvi_scope_ksplit_at(scope, type, color_array, nthreads, subscope); + + return rc; +} + #ifndef USE_LAYOUTS //New interface void * @@ -83,19 +133,18 @@ qv_thread_routine( void * arg ) { qv_thread_args_t *arg_ptr = (qv_thread_args_t *) arg; - // fprintf(stdout,"qv_thread_routine: ctx=%p scope=%p\n", qvp->ctx, qvp->scope); int rc = qv_bind_push(arg_ptr->ctx, arg_ptr->scope); + + if (rc != QV_SUCCESS) { - // char const *ers = "qv_bind_push() failed"; - // qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + fprintf(stdout,"==== Bind Push error \n"); pthread_exit(NULL); } void *ret = arg_ptr->thread_routine(arg_ptr->arg); // Free memory allocated in qv_pthread_create - //free(arg_ptr); delete arg_ptr; pthread_exit(ret); } @@ -111,13 +160,12 @@ qv_pthread_create( ) { // Memory will be freed in qv_thread_routine to avoid memory leaks qv_thread_args_t *arg_ptr = qvi_new qv_thread_args_t(); - //(qv_thread_args_t *)malloc(sizeof(qv_thread_args_t)); + arg_ptr->ctx = ctx; arg_ptr->scope = scope; arg_ptr->thread_routine = thread_routine; arg_ptr->arg = arg; - // fprintf(stdout,"qv_pthread_create: ctx=%p scope=%p\n", ctx, scope); return pthread_create(thread, attr, qv_thread_routine, arg_ptr); } #else // USE_LAYOUTS @@ -267,7 +315,7 @@ qv_thread_layout_apply( //use map interface if necessary int rc = QV_SUCCESS; qv_context_t *parent_ctx = th_args.ctx; qv_scope_t *parent_scope = th_args.scope; - qv_layout_t *thread_layout = th_args.thread_layout; int thr_idx = th_args.th_id; + qv_layout_t *thread_layout = th_args.thread_layout; int num_thr = th_args.num_th; /* Brand new case: compute and cache as much as possible*/ diff --git a/src/qvi-bbuff-rmi.h b/src/qvi-bbuff-rmi.h index 2a593752..305dd309 100644 --- a/src/qvi-bbuff-rmi.h +++ b/src/qvi-bbuff-rmi.h @@ -174,6 +174,15 @@ qvi_bbuff_rmi_pack_type_picture( picture += "i"; } +template<> +inline void +qvi_bbuff_rmi_pack_type_picture( + std::string &picture, + qv_hw_obj_type_t * +) { + picture += "i"; +} + template<> inline void qvi_bbuff_rmi_pack_type_picture( diff --git a/src/qvi-context.h b/src/qvi-context.h index 41555113..ce008a23 100644 --- a/src/qvi-context.h +++ b/src/qvi-context.h @@ -57,6 +57,7 @@ struct qv_context_s { qvi_delete(&zgroup); qvi_rmi_client_free(&rmi); } + }; /** diff --git a/src/qvi-hwloc.cc b/src/qvi-hwloc.cc index 1f5043b5..776624d1 100644 --- a/src/qvi-hwloc.cc +++ b/src/qvi-hwloc.cc @@ -111,6 +111,40 @@ qvi_hwloc_get_obj_type( } } +// TODO(skg) Merge with qvi_hwloc_get_obj_type()? +static qv_hw_obj_type_t +qvi_hwloc_convert_obj_type( + hwloc_obj_type_t external +) { + switch(external) { + case(HWLOC_OBJ_MACHINE): + return QV_HW_OBJ_MACHINE; + case(HWLOC_OBJ_PACKAGE): + return QV_HW_OBJ_PACKAGE; + case(HWLOC_OBJ_CORE): + return QV_HW_OBJ_CORE; + case(HWLOC_OBJ_PU): + return QV_HW_OBJ_PU; + case(HWLOC_OBJ_L1CACHE): + return QV_HW_OBJ_L1CACHE; + case(HWLOC_OBJ_L2CACHE): + return QV_HW_OBJ_L2CACHE; + case(HWLOC_OBJ_L3CACHE): + return QV_HW_OBJ_L3CACHE; + case(HWLOC_OBJ_L4CACHE): + return QV_HW_OBJ_L4CACHE; + case(HWLOC_OBJ_L5CACHE): + return QV_HW_OBJ_L5CACHE; + case(HWLOC_OBJ_NUMANODE): + return QV_HW_OBJ_NUMANODE; + case(HWLOC_OBJ_OS_DEVICE): + return QV_HW_OBJ_GPU; + default: + // This is an internal development error. + qvi_abort(); + } +} + bool qvi_hwloc_obj_type_is_host_resource( qv_hw_obj_type_t type @@ -1082,6 +1116,28 @@ qvi_hwloc_get_nobjs_in_cpuset( return QV_ERR_INTERNAL; } +int +qvi_hwloc_get_obj_type_in_cpuset( + qvi_hwloc_t *hwl, + int npieces, + hwloc_const_cpuset_t cpuset, + qv_hw_obj_type_t *target_obj +) { + hwloc_topology_t topo = hwl->topo; + hwloc_obj_t obj = hwloc_get_first_largest_obj_inside_cpuset(topo, cpuset); + int num = hwloc_get_nbobjs_inside_cpuset_by_type(topo, cpuset, obj->type); + int depth = obj->depth; + + while(!((num > npieces) && (!hwloc_obj_type_is_cache(obj->type)))){ + depth++; + obj = hwloc_get_obj_inside_cpuset_by_depth(topo, cpuset, depth, 0); + num = hwloc_get_nbobjs_inside_cpuset_by_type(topo, cpuset, obj->type); + } + + *target_obj = qvi_hwloc_convert_obj_type(obj->type); + return QV_SUCCESS; +} + int qvi_hwloc_get_obj_in_cpuset_by_depth( qvi_hwloc_t *hwl, diff --git a/src/qvi-hwloc.h b/src/qvi-hwloc.h index 89c24cc9..a7069f9f 100644 --- a/src/qvi-hwloc.h +++ b/src/qvi-hwloc.h @@ -286,6 +286,17 @@ qvi_hwloc_get_nobjs_in_cpuset( int *nobjs ); +/** + * + */ +int +qvi_hwloc_get_obj_type_in_cpuset( + qvi_hwloc_t *hwl, + int npieces, + hwloc_const_cpuset_t cpuset, + qv_hw_obj_type_t *target_obj +); + /** * */ diff --git a/src/qvi-rmi.cc b/src/qvi-rmi.cc index 8d3d5960..b17c096e 100644 --- a/src/qvi-rmi.cc +++ b/src/qvi-rmi.cc @@ -67,6 +67,7 @@ typedef enum qvi_rpc_funid_e { FID_TASK_SET_CPUBIND_FROM_CPUSET, FID_OBJ_TYPE_DEPTH, FID_GET_NOBJS_IN_CPUSET, + FID_GET_OBJ_TYPE_IN_CPUSET, FID_GET_DEVICE_IN_CPUSET, FID_SCOPE_GET_INTRINSIC_HWPOOL } qvi_rpc_funid_t; @@ -537,6 +538,31 @@ rpc_ssi_get_nobjs_in_cpuset( return qvrc; } +static int +rpc_ssi_get_obj_type_in_cpuset( + qvi_rmi_server_t *server, + qvi_msg_header_t *hdr, + void *input, + qvi_bbuff_t **output +) { + int npieces = 0; + hwloc_cpuset_t cpuset = nullptr; + int qvrc = qvi_bbuff_rmi_unpack( + input, &npieces, &cpuset + ); + if (qvrc != QV_SUCCESS) return qvrc; + + qv_hw_obj_type_t target_obj; + const int rpcrc = qvi_hwloc_get_obj_type_in_cpuset( + server->config.hwloc, npieces, cpuset, &target_obj + ); + + qvrc = rpc_pack(output, hdr->fid, rpcrc, target_obj); + + hwloc_bitmap_free(cpuset); + return qvrc; +} + static int rpc_ssi_get_device_in_cpuset( qvi_rmi_server_t *server, @@ -656,6 +682,7 @@ static const qvi_rpc_fun_ptr_t rpc_dispatch_table[] = { rpc_ssi_task_set_cpubind_from_cpuset, rpc_ssi_obj_type_depth, rpc_ssi_get_nobjs_in_cpuset, + rpc_ssi_get_obj_type_in_cpuset, rpc_ssi_get_device_in_cpuset, rpc_ssi_scope_get_intrinsic_hwpool }; @@ -1102,6 +1129,30 @@ qvi_rmi_get_nobjs_in_cpuset( return rpcrc; } +int +qvi_rmi_get_obj_type_in_cpuset( + qvi_rmi_client_t *client, + int npieces, + hwloc_const_cpuset_t cpuset, + qv_hw_obj_type_t *target_obj +) { + int qvrc = rpc_req( + client->zsock, + FID_GET_OBJ_TYPE_IN_CPUSET, + npieces, + cpuset + ); + if (qvrc != QV_SUCCESS) return qvrc; + + // Should be set by rpc_rep, so assume an error. + int rpcrc = QV_ERR_MSG; + qvrc = rpc_rep(client->zsock, &rpcrc, target_obj); + if (qvrc != QV_SUCCESS) return qvrc; + + return rpcrc; +} + + int qvi_rmi_get_device_in_cpuset( qvi_rmi_client_t *client, diff --git a/src/qvi-rmi.h b/src/qvi-rmi.h index 301c21fa..ba183ca4 100644 --- a/src/qvi-rmi.h +++ b/src/qvi-rmi.h @@ -154,6 +154,17 @@ qvi_rmi_get_nobjs_in_cpuset( int *nobjs ); +/** + * + */ +int +qvi_rmi_get_obj_type_in_cpuset( + qvi_rmi_client_t *client, + int npieces, + hwloc_const_cpuset_t cpuset, + qv_hw_obj_type_t *target_obj +); + /** * */ diff --git a/src/qvi-scope.cc b/src/qvi-scope.cc index 2d184e09..90bce4b6 100644 --- a/src/qvi-scope.cc +++ b/src/qvi-scope.cc @@ -207,6 +207,19 @@ get_nobjs_in_hwpool( return QV_SUCCESS; } +static int +get_obj_type_in_hwpool( + qvi_rmi_client_t *rmi, + qvi_hwpool_s *hwpool, + int npieces, + qv_hw_obj_type_t *obj +) { + return qvi_rmi_get_obj_type_in_cpuset( + rmi, npieces, hwpool->get_cpuset().cdata(), obj + ); +} + + template static int gather_values( @@ -1116,6 +1129,7 @@ qvi_scope_split( return rc; } + int qvi_scope_ksplit( qv_scope_t *parent, @@ -1295,6 +1309,17 @@ qvi_scope_nobjs( ); } +int +qvi_scope_obj_type( + qv_scope_t *scope, + int npieces, + qv_hw_obj_type_t *obj +) { + return get_obj_type_in_hwpool( + scope->rmi, scope->hwpool, npieces, obj + ); +} + int qvi_scope_get_device_id( qv_scope_t *scope, diff --git a/src/qvi-scope.h b/src/qvi-scope.h index f7077bd5..0da13f5d 100644 --- a/src/qvi-scope.h +++ b/src/qvi-scope.h @@ -169,6 +169,13 @@ qvi_scope_nobjs( int *n ); +int +qvi_scope_obj_type( + qv_scope_t *scope, + int npieces, + qv_hw_obj_type_t *obj +); + int qvi_scope_get_device_id( qv_scope_t *scope, diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 541fb098..f030e7b6 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -154,6 +154,18 @@ if(MPI_FOUND) test-progress-thread quo-vadis quo-vadis-mpi + ) + + add_executable( + test-pthread-split + qvi-test-common.h + test-pthread-split.c + ) + + target_link_libraries( + test-pthread-split + quo-vadis + quo-vadis-mpi ) add_executable( diff --git a/tests/test-pthread-split.c b/tests/test-pthread-split.c new file mode 100644 index 00000000..8a9324a6 --- /dev/null +++ b/tests/test-pthread-split.c @@ -0,0 +1,229 @@ +/* -*- Mode: C; c-basic-offset:4; indent-tabs-mode:nil -*- */ +#include +#include +#include "quo-vadis-mpi.h" +#include "quo-vadis-thread.h" +#include "qvi-test-common.h" +#include + +void *thread_work(void *arg) +{ + qv_context_t *ctx = (qv_context_t *) arg; + char *binds; + char const *ers = NULL; + + int rc = qv_bind_string(ctx, QV_BIND_STRING_AS_LIST, &binds); + + if (rc != QV_SUCCESS) { + ers = "qv_bind_string() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + fprintf(stdout,"Thread running on %s\n", binds); + free(binds); + + //int toto = 0; + //while(++toto); + + return NULL; +} + + +//int main(int argc, char *argv[]) +int +main(void) +{ + char const *ers = NULL; + MPI_Comm comm = MPI_COMM_WORLD; + int wrank, wsize; + int n_numas, n_cores, n_pus, my_numa_id; + int rc = QV_SUCCESS; + + fprintf(stdout,"# Starting Hybrid MPI + Pthreads test\n"); + + rc = MPI_Init(NULL, NULL); + if (rc != MPI_SUCCESS) { + ers = "MPI_Init() failed"; + qvi_test_panic("%s (rc=%d)", ers, rc); + } + + rc = MPI_Comm_size(comm, &wsize); + if (rc != MPI_SUCCESS) { + ers = "MPI_Comm_size() failed"; + qvi_test_panic("%s (rc=%d)", ers, rc); + } + + rc = MPI_Comm_rank(comm, &wrank); + if (rc != MPI_SUCCESS) { + ers = "MPI_Comm_rank() failed"; + qvi_test_panic("%s (rc=%d)", ers, rc); + } + + qv_context_t *mpi_ctx; + rc = qv_mpi_context_create(comm, &mpi_ctx); + if (rc != QV_SUCCESS) { + ers = "qv_mpi_context_create() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + qv_scope_t *mpi_scope; + rc = qv_scope_get(mpi_ctx, QV_SCOPE_JOB, &mpi_scope); + if (rc != QV_SUCCESS) { + ers = "qv_scope_get() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + qvi_test_scope_report(mpi_ctx, mpi_scope, "mpi_job_scope"); + + rc = qv_scope_nobjs(mpi_ctx, mpi_scope, QV_HW_OBJ_NUMANODE, &n_numas); + if (rc != QV_SUCCESS) { + ers = "qv_scope_nobjs() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + fprintf(stdout,"[%d] Number of NUMANodes in mpi_scope is %d\n", wrank, n_numas); + + qv_scope_t *mpi_numa_scope; + rc = qv_scope_split_at(mpi_ctx, mpi_scope, QV_HW_OBJ_NUMANODE, wrank % n_numas, &mpi_numa_scope); + if (rc != QV_SUCCESS) { + ers = "qv_scope_split_at() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + qvi_test_scope_report(mpi_ctx, mpi_numa_scope, "mpi_numa_scope"); + + qvi_test_bind_push(mpi_ctx, mpi_numa_scope); + + rc = qv_scope_taskid(mpi_ctx, mpi_numa_scope, &my_numa_id); + if (rc != QV_SUCCESS) { + ers = "qv_scope_taskid() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + fprintf(stdout,"[%d] NUMA id is %d\n", wrank, my_numa_id); + + rc = qv_scope_nobjs(mpi_ctx, mpi_numa_scope, QV_HW_OBJ_CORE, &n_cores); + if (rc != QV_SUCCESS) { + ers = "qv_scope_nobjs() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + fprintf(stdout,"[%d] Number of Cores in mpi_numa_scope is %d\n", wrank, n_cores); + + rc = qv_scope_nobjs(mpi_ctx, mpi_numa_scope, QV_HW_OBJ_PU, &n_pus); + if (rc != QV_SUCCESS) { + ers = "qv_scope_nobjs() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + fprintf(stdout,"[%d] Number of PUs in mpi_numa_scope is %d\n", wrank, n_pus); + + + qv_scope_t **th_scopes = NULL; + + //test qv_thread_scope_split + int npieces = n_cores/2; + int nthreads = n_cores; + + assert(nthreads <= n_pus); + + fprintf(stdout,"[%d] ====== Testing thread_scope_split (number of threads : %i)\n", wrank, nthreads); + + int colors[nthreads]; + for(int i = 0 ; i < nthreads ; i++) + colors[i] = i%2;//n_cores; + + rc = qv_thread_scope_split(mpi_ctx, mpi_numa_scope, npieces , colors , nthreads, &th_scopes); + if (rc != QV_SUCCESS) { + ers = "qv_thread_scope_split() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + assert(th_scopes); + + pthread_t thid[nthreads]; + pthread_attr_t *attr = NULL; + + for(int i = 0 ; i < nthreads; i ++){ + //sleep(1); + if (qv_pthread_create(&thid[i], attr, thread_work, (void *)(mpi_ctx), mpi_ctx, th_scopes[i]) != 0) { + perror("pthread_create() error"); + exit(1); + } + } + + void *ret; + for(int i = 0 ; i < nthreads; i ++){ + if (pthread_join(thid[i], &ret) != 0) { + perror("pthread_create() error"); + exit(3); + } + fprintf(stdout,"Thread finished with '%s'\n", (char *)ret); + } + + /* Clean up */ + for(int i = 0 ; i < nthreads; i ++){ + rc = qv_scope_free(mpi_ctx, th_scopes[i]); + if (rc != QV_SUCCESS) { + ers = "qv_scope_free() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + } + free(th_scopes); + + //Test qv_thread_scope_split_at + nthreads = 2*n_cores; + assert(nthreads <= n_pus); + + fprintf(stdout,"[%d] ====== Testing thread_scope_split_at (number of threads : %i)\n", wrank, nthreads); + + int colors2[nthreads]; + for(int i = 0 ; i < nthreads ; i++) + colors2[i] = i%n_cores; + + rc = qv_thread_scope_split_at(mpi_ctx, mpi_numa_scope, QV_HW_OBJ_CORE, colors2, nthreads, &th_scopes); + if (rc != QV_SUCCESS) { + ers = "qv_thread_scope_split_at() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + assert(th_scopes); + + pthread_t thid2[nthreads]; + for(int i = 0 ; i < nthreads; i ++){ + if (qv_pthread_create(&thid2[i], attr, thread_work, (void *)(mpi_ctx), mpi_ctx, th_scopes[i]) != 0) { + perror("pthread_create() error"); + exit(1); + } + } + + for(int i = 0 ; i < nthreads; i ++){ + if (pthread_join(thid2[i], &ret) != 0) { + perror("pthread_create() error"); + exit(3); + } + fprintf(stdout,"Thread finished with '%s'\n", (char *)ret); + } + + /* Clean up */ + for(int i = 0 ; i < nthreads; i ++){ + rc = qv_scope_free(mpi_ctx, th_scopes[i]); + if (rc != QV_SUCCESS) { + ers = "qv_scope_free() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + } + free(th_scopes); + + rc = qv_scope_free(mpi_ctx, mpi_numa_scope); + if (rc != QV_SUCCESS) { + ers = "qv_scope_free() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + rc = qv_scope_free(mpi_ctx, mpi_scope); + if (rc != QV_SUCCESS) { + ers = "qv_scope_free() failed"; + qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc)); + } + + MPI_Finalize(); + + return EXIT_SUCCESS; +} + +/* + * vim: ft=cpp ts=4 sts=4 sw=4 expandtab + */