diff --git a/include/quo-vadis-thread.h b/include/quo-vadis-thread.h index 4b80535..92a3200 100644 --- a/include/quo-vadis-thread.h +++ b/include/quo-vadis-thread.h @@ -32,19 +32,6 @@ extern "C" { /** * Mapping policies types. */ -/* -typedef enum qv_policy_s { - QV_POLICY_PACKED = 1, - QV_POLICY_COMPACT = 1, - QV_POLICY_CLOSE = 1, - QV_POLICY_SPREAD = 2, - QV_POLICY_DISTRIBUTE = 3, - QV_POLICY_ALTERNATE = 3, - QV_POLICY_CORESFIRST = 3, - QV_POLICY_SCATTER = 4, - QV_POLICY_CHOOSE = 5, -} qv_policy_t; -*/ typedef enum qv_policy_s { QV_POLICY_PACKED = 1, @@ -75,7 +62,6 @@ qv_thread_context_free( qv_context_t *ctx ); -#ifndef USE_LAYOUTS typedef struct { qv_context_t *ctx; qv_scope_t *scope; @@ -118,95 +104,6 @@ qv_thread_scope_split( qv_scope_t ***subscope ); -#else // TODO(skg) Can we get rid of the old interface? -/** - * Layout for fine-grain binding - * with default behaviour - */ -typedef struct qv_layout_params_s { - qv_policy_t policy; - qv_hw_obj_type_t obj_type; - int stride; -} qv_layout_params_t; - -typedef struct qv_layout_cached_info_s { - qv_layout_params_t params; - hwloc_const_cpuset_t cpuset; - hwloc_obj_type_t hwloc_obj_type; - int nb_objs; - int nb_threads; - int *rsrc_idx; -} qv_layout_cached_info_t; - -typedef struct qv_layout_s { - qvi_hwloc_t *hwl; // in cached infos instead? - hwloc_topology_t hw_topo; // in cached infos instead? - qv_layout_params_t params; - qv_context_t *ctx; - qv_layout_cached_info_t cached_info; - int is_cached; -} qv_layout_t; - -/** - * Used to pass args to pthreads. - */ -typedef struct { - qv_context_t *ctx; - qv_scope_t *scope; - qv_layout_t *thread_layout; - int th_id; - int num_th; -} qv_thread_args_t; - -int -qv_thread_layout_create( - qv_context_t *ctx, - qv_layout_params_t params, - qv_layout_t **layout -); - -int -qv_thread_layout_free( - qv_layout_t *layout -); - -int -qv_thread_layout_apply( - qv_thread_args_t th_args -); - -/** - * Used for args to pthreads. - */ -int -qv_thread_args_set( - qv_context_t *ctx, - qv_scope_t *scope, - qv_layout_t *thread_layout, - int th_id, - int num_th, - qv_thread_args_t *th_args -); - -int -qv_thread_layout_set_policy( - qv_layout_t *layout, - qv_policy_t policy -); - -int -qv_thread_layout_set_obj_type( - qv_layout_t *layout, - qv_hw_obj_type_t obj_type -); - -int -qv_thread_layout_set_stride( - qv_layout_t *layout, - int stride -); -#endif // USE_LAYOUTS - #ifdef __cplusplus } #endif diff --git a/src/quo-vadis-thread.cc b/src/quo-vadis-thread.cc index e93ed3b..de728c5 100644 --- a/src/quo-vadis-thread.cc +++ b/src/quo-vadis-thread.cc @@ -115,8 +115,6 @@ qv_thread_scope_split_at( return qvi_scope_ksplit_at(scope, type, color_array, nthreads, subscopes); } -#ifndef USE_LAYOUTS -//New interface void * qv_thread_routine( void * arg @@ -155,290 +153,6 @@ qv_pthread_create( return pthread_create(thread, attr, qv_thread_routine, arg_ptr); } -#else // USE_LAYOUTS -// Layout interface -int -qv_thread_layout_create( // use hwpool if necessary - qv_context_t *ctx, - qv_layout_params_t params, - qv_layout_t **layout -) { - int rc = QV_SUCCESS; - qv_layout_t *ilay = qvi_new qv_layout_t(); - - ilay->hwl = qvi_rmi_client_hwloc_get(ctx->rmi); - ilay->hw_topo = qvi_hwloc_get_topo_obj(ilay->hwl); - ilay->params = params; - ilay->ctx = ctx; - memset(&ilay->cached_info,0,sizeof(qv_layout_cached_info_t)); - ilay->is_cached = 0; - - *layout = ilay; - return rc; -} - -int -qv_thread_layout_free( - qv_layout_t *layout -){ - if (!layout) return QV_ERR_INVLD_ARG; - //fixme check that is was actually allocated. - delete[] layout->cached_info.rsrc_idx; - delete layout; - return QV_SUCCESS; -} - -int -qv_thread_layout_set_policy( - qv_layout_t *layout, - qv_policy_t policy -) { - int rc = QV_SUCCESS; - - if (!layout) { - rc = QV_ERR_INVLD_ARG; - goto out; - } - - layout->params.policy = policy; - - out: - return rc; -} - -int -qv_thread_layout_set_obj_type( - qv_layout_t *layout, - qv_hw_obj_type_t obj_type -) { - int rc = QV_SUCCESS; - - if (!layout) { - rc = QV_ERR_INVLD_ARG; - goto out; - } - - layout->params.obj_type = obj_type; - - out: - return rc; -} - -int -qv_thread_layout_set_stride( - qv_layout_t *layout, - int stride -) { - int rc = QV_SUCCESS; - - if (!layout) { - rc = QV_ERR_INVLD_ARG; - goto out; - } - - layout->params.stride = stride; - - out: - return rc; -} - -// Intel policies (KMP_AFFINITY) are : -// - disabled: prevents the runtime library from making any affinity-related -// system calls (to avoid interference with other platform affinity mechanisms). -// - compact: threads are placed as close together as possible. -// - scatter: threads are distributed as evenly as possible across the entire system. -// (opposite of compact). -// - explicit: threads are placed according to a list of OS proc IDs (required) -static int -compute( - int *array, - int nb_threads, - int nb_objs, - int stride, - qv_policy_t policy -) { - int rc = QV_SUCCESS; - switch(policy) - { - case QV_POLICY_SPREAD: // Same as SCATTER? - { - break; - } - case QV_POLICY_DISTRIBUTE: - case QV_POLICY_ALTERNATE: - case QV_POLICY_CORESFIRST: - { - break; - } - case QV_POLICY_SCATTER: - { - break; - } - case QV_POLICY_CHOOSE: // rename to EXPLICIT? - { - break; - } - case QV_POLICY_PACKED: - case QV_POLICY_COMPACT: - case QV_POLICY_CLOSE: - default: - { - for(int idx = 0 ; idx < nb_threads ; idx++) { - array[idx] = (idx+idx*(stride-1))%(nb_objs); - } - break; - } - } - return rc; -} - -int -qv_thread_layout_apply( //use map interface if necessary - qv_thread_args_t th_args -) { - int rc = QV_SUCCESS; - qv_context_t *parent_ctx = th_args.ctx; - qv_scope_t *parent_scope = th_args.scope; - qv_layout_t *thread_layout = th_args.thread_layout; - int num_thr = th_args.num_th; - - /* Brand new case: compute and cache as much as possible*/ - if(thread_layout->is_cached == 0){ - thread_layout->ctx = parent_ctx; - thread_layout->cached_info.params = thread_layout->params; - thread_layout->cached_info.cpuset = qvi_scope_cpuset_get(parent_scope); - thread_layout->cached_info.hwloc_obj_type = qvi_hwloc_get_obj_type(thread_layout->cached_info.params.obj_type); - rc = qvi_hwloc_get_nobjs_in_cpuset(thread_layout->hwl, - thread_layout->cached_info.params.obj_type, - thread_layout->cached_info.cpuset, - &(thread_layout->cached_info.nb_objs)); - thread_layout->cached_info.nb_threads = num_thr; - thread_layout->cached_info.rsrc_idx = new int[thread_layout->cached_info.nb_threads]{-1}; - - rc = compute(thread_layout->cached_info.rsrc_idx, - thread_layout->cached_info.nb_threads, - thread_layout->cached_info.nb_objs, - thread_layout->cached_info.params.stride, - thread_layout->cached_info.params.policy); - - thread_layout->is_cached = 1; - - } else if (thread_layout->is_cached) { - // Are we using the right context? - if (thread_layout->ctx != parent_ctx) { - // pointer comparison might not be enough. - // do we need a unique ctxt id? - rc = QV_ERR_INVLD_ARG; - return rc; - } else { - // check changes and recompute if necessary - int recompute = 0; - - hwloc_obj_type_t hwloc_obj_type = qvi_hwloc_get_obj_type(thread_layout->params.obj_type); - if ( hwloc_obj_type != thread_layout->cached_info.hwloc_obj_type ){ - int nb_objs; - - //no need to systematically recompute in this case: - // recompute iff number of objects is different. - // otherwise: keep same parameters and apply to new objects - thread_layout->cached_info.hwloc_obj_type = hwloc_obj_type; - - rc = qvi_hwloc_get_nobjs_in_cpuset(thread_layout->hwl, - thread_layout->params.obj_type, - thread_layout->cached_info.cpuset, - &nb_objs); - - if(nb_objs != thread_layout->cached_info.nb_objs){ - thread_layout->cached_info.nb_objs = nb_objs; - recompute++; - } - } - - /* cpuset is different, recompute info */ - //if (hwloc_bitmap_compare(thread_layout->cached_info.cpuset,qvi_scope_cpuset_get(parent_scope)) != 0 ) { - // thread_layout->cached_info.cpuset = qvi_scope_cpuset_get(parent_scope); - // recompute++; - //} - - - if (num_thr != thread_layout->cached_info.nb_threads) { - thread_layout->cached_info.nb_threads = num_thr; - recompute++; - } - - if(thread_layout->cached_info.params.stride != thread_layout->params.stride){ - thread_layout->cached_info.params.stride = thread_layout->params.stride; - recompute++; - } - - if (thread_layout->cached_info.params.policy != thread_layout->params.policy){ - thread_layout->cached_info.params.policy = thread_layout->params.policy; - recompute++; - } - - if(recompute){ - rc = compute(thread_layout->cached_info.rsrc_idx, - thread_layout->cached_info.nb_threads, - thread_layout->cached_info.nb_objs, - thread_layout->cached_info.params.stride, - thread_layout->cached_info.params.policy); - //fprintf(stdout,"***********************>>>>>>>>> recompute done\n"); - } - } - } - - // FIXME : what should we do in oversubscribing case? - // GM EDIT: Mutiple threads on objects featuring multiple PUs is not an issue - // Oversub test : check that an index does not appear twice in the resrc indices array - if(thread_layout->cached_info.nb_objs < thread_layout->cached_info.nb_threads) - { - if (!thr_idx) - fprintf(stdout,"====> Oversubscribing \n"); - } - else { - if (!thr_idx) - fprintf(stdout,"====> Resource number is ok\n"); - } - - /* enforce binding */ - hwloc_obj_t obj = hwloc_get_obj_inside_cpuset_by_type(thread_layout->hw_topo, - thread_layout->cached_info.cpuset, - thread_layout->cached_info.hwloc_obj_type, - thread_layout->cached_info.rsrc_idx[thr_idx]); - - hwloc_set_cpubind(thread_layout->hw_topo,obj->cpuset,HWLOC_CPUBIND_THREAD); - - fprintf(stdout,"[THREAD #%i] === bound on resrc #%i\n",thr_idx,obj->logical_index); - - /* Sanity check */ - //assert(obj->logical_index == thread_layout->cached_info.rsrc_idx[thr_idx]); - - return rc; -} - -int -qv_thread_args_set( - qv_context_t *ctx, - qv_scope_t *scope, - qv_layout_t *thread_layout, - int th_id, - int num_th, - qv_thread_args_t *th_args -) { - if (!ctx || !scope || !thread_layout || !th_args) { - return QV_ERR_INVLD_ARG; - } - - th_args->ctx = ctx; - th_args->scope = scope; - th_args->thread_layout = thread_layout; - th_args->th_id = th_id; - th_args->num_th = num_th; - - return QV_SUCCESS; -} - -#endif // USE_LAYOUTS /* * vim: ft=cpp ts=4 sts=4 sw=4 expandtab diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f030e7b..19e9c4d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -198,7 +198,6 @@ if(MPI_FOUND) test-progress-thread test-mpi-scope-create test-mpi-getdev - #test-mpi-pthreads-layout PROPERTIES LINKER_LANGUAGE C ) @@ -238,18 +237,8 @@ if(MPI_FOUND) ############################# HYBRID TESTS (PThreads) ########################## ################################################################################ - #add_executable( - #test-mpi-pthreads-layout - #test-mpi-pthreads-layout.c - #) - #target_link_libraries( - #test-mpi-pthreads-layout - #quo-vadis - #quo-vadis-mpi - #) - ############################# HYBRID TESTS (OpenMP) ############################ ################################################################################ if(OPENMP_FOUND) @@ -265,22 +254,8 @@ if(MPI_FOUND) OpenMP::OpenMP_C ) - #add_executable( - # test-mpi-threads-layout - # test-mpi-threads-layout.c - #) - - - #target_link_libraries( - # test-mpi-threads-layout - # quo-vadis - # quo-vadis-mpi - # OpenMP::OpenMP_C - #) - set_target_properties( test-mpi-threads - #test-mpi-threads-layout PROPERTIES LINKER_LANGUAGE C ) endif()