diff --git a/src/quo-vadis-mpi.cc b/src/quo-vadis-mpi.cc index 112e517..27e7eb5 100644 --- a/src/quo-vadis-mpi.cc +++ b/src/quo-vadis-mpi.cc @@ -88,7 +88,7 @@ qvi_mpi_scope_comm_dup( MPI_Comm *comm ) { qvi_group_mpi_t *const mpi_group = dynamic_cast( - qvi_scope_group_get(scope) + qvi_scope_group(scope) ); return mpi_group->comm_dup(comm); } diff --git a/src/quo-vadis-pthread.cc b/src/quo-vadis-pthread.cc index 44ef5ad..932d813 100644 --- a/src/quo-vadis-pthread.cc +++ b/src/quo-vadis-pthread.cc @@ -108,7 +108,7 @@ qv_pthread_create( // pthread_create(), return a reasonable errno. if (qvi_unlikely(rc != QV_SUCCESS)) return ENOMEM; - auto group = dynamic_cast(qvi_scope_group_get(scope)); + auto group = dynamic_cast(qvi_scope_group(scope)); qvi_pthread_group_pthread_create_args_s *cargs = nullptr; rc = qvi_new(&cargs, group->thgroup, qvi_pthread_routine, arg_ptr); if (qvi_unlikely(rc != QV_SUCCESS)) { diff --git a/src/quo-vadis.cc b/src/quo-vadis.cc index bfd26df..4baf81d 100644 --- a/src/quo-vadis.cc +++ b/src/quo-vadis.cc @@ -217,7 +217,7 @@ qv_scope_get_device_id( return QV_ERR_INVLD_ARG; } try { - return qvi_scope_get_device_id( + return qvi_scope_device_id( scope, dev_obj, dev_index, id_type, dev_id ); } diff --git a/src/qvi-bbuff.cc b/src/qvi-bbuff.cc index d4480a6..05c426c 100644 --- a/src/qvi-bbuff.cc +++ b/src/qvi-bbuff.cc @@ -87,7 +87,7 @@ qvi_bbuff_size( int qvi_bbuff_append( qvi_bbuff_t *buff, - const void *data, + const void *const data, size_t size ) { const size_t req_capacity = size + buff->size; diff --git a/src/qvi-bbuff.h b/src/qvi-bbuff.h index 484031d..f4866f6 100644 --- a/src/qvi-bbuff.h +++ b/src/qvi-bbuff.h @@ -21,10 +21,6 @@ #include "qvi-common.h" -#ifdef __cplusplus -extern "C" { -#endif - /** * */ @@ -69,14 +65,10 @@ qvi_bbuff_size( int qvi_bbuff_append( qvi_bbuff_t *buff, - const void *data, + const void *const data, size_t size ); -#ifdef __cplusplus -} -#endif - #endif /* diff --git a/src/qvi-common.h b/src/qvi-common.h index baf1425..f2b506b 100644 --- a/src/qvi-common.h +++ b/src/qvi-common.h @@ -65,6 +65,10 @@ typedef uint8_t byte_t; typedef char const* cstr_t; typedef unsigned int uint_t; +/** Opaque hwloc instance. */ +struct qvi_hwloc_s; +typedef struct qvi_hwloc_s qvi_hwloc_t; + /** Opaque RMI client. */ struct qvi_rmi_client_s; typedef struct qvi_rmi_client_s qvi_rmi_client_t; diff --git a/src/qvi-group.cc b/src/qvi-group.cc index 4acaa84..f974403 100644 --- a/src/qvi-group.cc +++ b/src/qvi-group.cc @@ -13,8 +13,15 @@ #include "qvi-group.h" #include "qvi-group-pthread.h" +#include "qvi-task.h" // IWYU pragma: keep #include "qvi-utils.h" +qvi_hwloc_t * +qvi_group_s::hwloc(void) +{ + return task()->hwloc(); +} + int qvi_group_s::thsplit( int nthreads, diff --git a/src/qvi-group.h b/src/qvi-group.h index a86a0ab..a05d68f 100644 --- a/src/qvi-group.h +++ b/src/qvi-group.h @@ -34,6 +34,9 @@ struct qvi_group_s : qvi_refc_s { /** Returns pointer to the caller's task information. */ virtual qvi_task_t * task(void) = 0; + /** Returns pointer to the task's hwloc information. */ + qvi_hwloc_t * + hwloc(void); /** Returns the number of members in this group. */ virtual int size(void) = 0; diff --git a/src/qvi-hwloc.cc b/src/qvi-hwloc.cc index f29ad8b..d3d3b87 100644 --- a/src/qvi-hwloc.cc +++ b/src/qvi-hwloc.cc @@ -170,6 +170,23 @@ qvi_hwloc_obj_type_is_host_resource( } } +int +qvi_hwloc_bitmap_string( + hwloc_const_bitmap_t bitmap, + qv_bind_string_format_t format, + char **result +) { + switch (format) { + case QV_BIND_STRING_AS_BITMAP: + return qvi_hwloc_bitmap_asprintf(result, bitmap); + case QV_BIND_STRING_AS_LIST: + return qvi_hwloc_bitmap_list_asprintf(result, bitmap); + default: + *result = nullptr; + return QV_ERR_INVLD_ARG; + } +} + int qvi_hwloc_obj_type_depth( qvi_hwloc_t *hwloc, diff --git a/src/qvi-hwloc.h b/src/qvi-hwloc.h index 0a96e9b..315c24d 100644 --- a/src/qvi-hwloc.h +++ b/src/qvi-hwloc.h @@ -28,9 +28,6 @@ const int QVI_HWLOC_DEVICE_INVISIBLE_ID = -1; /** ID used to indicate an invalid or unset ID. */ const int QVI_HWLOC_DEVICE_INVALID_ID = -1; -struct qvi_hwloc_s; -typedef struct qvi_hwloc_s qvi_hwloc_t; - struct qvi_hwloc_device_s; typedef struct qvi_hwloc_device_s qvi_hwloc_device_t; @@ -531,6 +528,13 @@ qvi_hwloc_get_devices_in_bitmap( qvi_hwloc_dev_list_t &devs ); +int +qvi_hwloc_bitmap_string( + hwloc_const_bitmap_t bitmap, + qv_bind_string_format_t format, + char **result +); + #endif #endif diff --git a/src/qvi-hwpool.cc b/src/qvi-hwpool.cc index 7ca97b8..f7e2f4e 100644 --- a/src/qvi-hwpool.cc +++ b/src/qvi-hwpool.cc @@ -134,7 +134,6 @@ pool_release_cpus_by_cpuset( } #endif - // TODO(skg) Acquire devices. int qvi_hwpool_s::add_devices_with_affinity( qvi_hwloc_t *hwloc @@ -288,8 +287,8 @@ qvi_hwpool_s::unpack( } out: if (qvi_unlikely(rc != QV_SUCCESS)) { - total_bw = 0; qvi_delete(&ihwp); + total_bw = 0; } *bytes_written = total_bw; *hwp = ihwp; diff --git a/src/qvi-hwpool.h b/src/qvi-hwpool.h index 9765992..463803d 100644 --- a/src/qvi-hwpool.h +++ b/src/qvi-hwpool.h @@ -16,7 +16,7 @@ #ifndef QVI_HWPOOL_H #define QVI_HWPOOL_H -#include "qvi-common.h" // IWYU pragma: keep +#include "qvi-common.h" #include "qvi-hwloc.h" /** diff --git a/src/qvi-rmi.h b/src/qvi-rmi.h index 9b6bd33..59ce56a 100644 --- a/src/qvi-rmi.h +++ b/src/qvi-rmi.h @@ -20,7 +20,6 @@ #define QVI_RMI_H #include "qvi-common.h" -#include "qvi-hwloc.h" #include "qvi-hwpool.h" #ifdef __cplusplus diff --git a/src/qvi-scope.cc b/src/qvi-scope.cc index 0da3954..50be931 100644 --- a/src/qvi-scope.cc +++ b/src/qvi-scope.cc @@ -18,8 +18,8 @@ // TODO(skg) Add RMI to acquire/release resources. #include "qvi-scope.h" -#include "qvi-bbuff.h" #include "qvi-task.h" +#include "qvi-bbuff.h" #include "qvi-rmi.h" #include "qvi-bbuff-rmi.h" #include "qvi-hwpool.h" @@ -29,7 +29,6 @@ /** Maintains a mapping between IDs to device information. */ using id_devinfo_multimap_t = std::multimap; -/** Scope type definition. */ struct qv_scope_s { /** Task group associated with this scope instance. */ qvi_group_t *group = nullptr; @@ -60,6 +59,33 @@ qvi_scope_new( return qvi_new(scope, group, hwpool); } +int +qvi_scope_get( + qvi_group_t *group, + qv_scope_intrinsic_t iscope, + qv_scope_t **scope +) { + qvi_hwpool_s *hwpool = nullptr; + // Get the requested intrinsic group. + int rc = group->make_intrinsic(iscope); + if (qvi_unlikely(rc != QV_SUCCESS)) goto out; + // Get the requested intrinsic hardware pool. + rc = qvi_rmi_scope_get_intrinsic_hwpool( + group->task()->rmi(), + qvi_task_s::mytid(), + iscope, &hwpool + ); + if (qvi_unlikely(rc != QV_SUCCESS)) goto out; + // Create and initialize the scope. + rc = qvi_scope_new(group, hwpool, scope); +out: + if (qvi_unlikely(rc != QV_SUCCESS)) { + qvi_delete(&hwpool); + qvi_scope_free(scope); + } + return rc; +} + void qvi_scope_free( qv_scope_t **scope @@ -82,7 +108,7 @@ qvi_scope_thfree( } qvi_group_t * -qvi_scope_group_get( +qvi_scope_group( qv_scope_t *scope ) { assert(scope); @@ -90,7 +116,7 @@ qvi_scope_group_get( } const qvi_hwpool_s * -qvi_scope_hwpool_get( +qvi_scope_hwpool( qv_scope_t *scope ) { assert(scope); @@ -98,7 +124,7 @@ qvi_scope_hwpool_get( } const qvi_hwloc_bitmap_s & -qvi_scope_cpuset_get( +qvi_scope_cpuset( qv_scope_t *scope ) { assert(scope); @@ -140,33 +166,33 @@ qvi_scope_nobjs( int *n ) { return scope->hwpool->nobjects( - scope->group->task()->hwloc(), obj, n + scope->group->hwloc(), obj, n ); } int -qvi_scope_get_device_id( +qvi_scope_device_id( qv_scope_t *scope, - qv_hw_obj_type_t dev_obj, - int i, + qv_hw_obj_type_t dev_type, + int dev_index, qv_device_id_type_t id_type, char **dev_id ) { int rc = QV_SUCCESS, id = 0, nw = 0; - + // Look for the requested device. qvi_hwpool_dev_s *finfo = nullptr; for (const auto &dinfo : scope->hwpool->devices()) { - if (dev_obj != dinfo.first) continue; - if (id++ == i) { + if (dev_type != dinfo.first) continue; + if (id++ == dev_index) { finfo = dinfo.second.get(); break; } } - if (!finfo) { + if (qvi_unlikely(!finfo)) { rc = QV_ERR_NOT_FOUND; goto out; } - + // Format the device ID based on the caller's request. switch (id_type) { case (QV_DEVICE_ID_UUID): nw = asprintf(dev_id, "%s", finfo->uuid.c_str()); @@ -181,9 +207,9 @@ qvi_scope_get_device_id( rc = QV_ERR_INVLD_ARG; goto out; } - if (nw == -1) rc = QV_ERR_OOR; + if (qvi_unlikely(nw == -1)) rc = QV_ERR_OOR; out: - if (rc != QV_SUCCESS) { + if (qvi_unlikely(rc != QV_SUCCESS)) { *dev_id = nullptr; } return rc; @@ -209,54 +235,14 @@ int qvi_scope_bind_string( qv_scope_t *scope, qv_bind_string_format_t format, - char **str + char **result ) { - char *istr = nullptr; - - hwloc_cpuset_t cpuset = nullptr; - int rc = scope->group->task()->bind_top(&cpuset); - if (rc != QV_SUCCESS) return rc; - - switch (format) { - case QV_BIND_STRING_AS_BITMAP: - rc = qvi_hwloc_bitmap_asprintf(&istr, cpuset); - break; - case QV_BIND_STRING_AS_LIST: - rc = qvi_hwloc_bitmap_list_asprintf(&istr, cpuset); - break; - default: - rc = QV_ERR_INVLD_ARG; - break; - } - qvi_hwloc_bitmap_free(&cpuset); - *str = istr; - return rc; -} + hwloc_cpuset_t bitmap = nullptr; + int rc = scope->group->task()->bind_top(&bitmap); + if (qvi_unlikely(rc != QV_SUCCESS)) return rc; -int -qvi_scope_get( - qvi_group_t *group, - qv_scope_intrinsic_t iscope, - qv_scope_t **scope -) { - qvi_hwpool_s *hwpool = nullptr; - // Get the requested intrinsic group. - int rc = group->make_intrinsic(iscope); - if (qvi_unlikely(rc != QV_SUCCESS)) goto out; - // Get the requested intrinsic hardware pool. - rc = qvi_rmi_scope_get_intrinsic_hwpool( - group->task()->rmi(), - qvi_task_s::mytid(), - iscope, &hwpool - ); - if (qvi_unlikely(rc != QV_SUCCESS)) goto out; - // Create and initialize the scope. - rc = qvi_scope_new(group, hwpool, scope); -out: - if (qvi_unlikely(rc != QV_SUCCESS)) { - qvi_delete(&hwpool); - qvi_scope_free(scope); - } + rc = qvi_hwloc_bitmap_string(bitmap, format, result); + qvi_hwloc_bitmap_free(&bitmap); return rc; } @@ -1337,7 +1323,7 @@ qvi_scope_create( rc = qvi_new(&hwpool); if (rc != QV_SUCCESS) goto out; - rc = hwpool->initialize(parent->group->task()->hwloc(), cpuset); + rc = hwpool->initialize(parent->group->hwloc(), cpuset); if (rc != QV_SUCCESS) goto out; // Create underlying group. Notice the use of self here. rc = parent->group->self(&group); diff --git a/src/qvi-scope.h b/src/qvi-scope.h index 6cada58..3e5b02f 100644 --- a/src/qvi-scope.h +++ b/src/qvi-scope.h @@ -21,6 +21,28 @@ #include "qvi-group.h" #include "qvi-hwloc.h" +/** + * Returns a new intrinsic scope. + */ +int +qvi_scope_get( + qvi_group_t *group, + qv_scope_intrinsic_t iscope, + qv_scope_t **scope +); + +/** + * + */ +int +qvi_scope_create( + qv_scope_t *parent, + qv_hw_obj_type_t type, + int nobjs, + qv_scope_create_hints_t hints, + qv_scope_t **child +); + /** * Frees a scope. */ @@ -38,6 +60,22 @@ qvi_scope_thfree( uint_t k ); +/** + * Returns a pointer to the scope's underlying group. + */ +qvi_group_t * +qvi_scope_group( + qv_scope_t *scope +); + +/** + * Returns a const reference to the provided scope's cpuset. + */ +const qvi_hwloc_bitmap_s & +qvi_scope_cpuset( + qv_scope_t *scope +); + /** * Returns the caller's group rank in the provided scope. */ @@ -48,7 +86,7 @@ qvi_scope_group_rank( ); /** - * + * Returns the scope's group size. */ int qvi_scope_group_size( @@ -57,7 +95,30 @@ qvi_scope_group_size( ); /** - * + * Returns the number of hardware objects in the provided scope. + */ +int +qvi_scope_nobjs( + qv_scope_t *scope, + qv_hw_obj_type_t obj, + int *n +); + +/** + * Returns the device ID string according to the ID type for the requested + * device type and index. + */ +int +qvi_scope_device_id( + qv_scope_t *scope, + qv_hw_obj_type_t dev_type, + int dev_index, + qv_device_id_type_t id_type, + char **dev_id +); + +/** + * Performs a scope-level barrier. */ int qvi_scope_barrier( @@ -78,33 +139,7 @@ int qvi_scope_bind_string( qv_scope_t *scope, qv_bind_string_format_t format, - char **str -); - -/** - * Returns a new intrinsic scope. - */ -int -qvi_scope_get( - qvi_group_t *group, - qv_scope_intrinsic_t iscope, - qv_scope_t **scope -); - -/** - * Returns a pointer to the scope's underlying group. - */ -qvi_group_t * -qvi_scope_group_get( - qv_scope_t *scope -); - -/** - * Returns a const reference to the provided scope's cpuset. - */ -const qvi_hwloc_bitmap_s & -qvi_scope_cpuset_get( - qv_scope_t *scope + char **result ); int @@ -149,37 +184,6 @@ qvi_scope_split_at( qv_scope_t **child ); -/** - * - */ -int -qvi_scope_create( - qv_scope_t *parent, - qv_hw_obj_type_t type, - int nobjs, - qv_scope_create_hints_t hints, - qv_scope_t **child -); - -/** - * - */ -int -qvi_scope_nobjs( - qv_scope_t *scope, - qv_hw_obj_type_t obj, - int *n -); - -int -qvi_scope_get_device_id( - qv_scope_t *scope, - qv_hw_obj_type_t dev_obj, - int i, - qv_device_id_type_t id_type, - char **dev_id -); - #endif /*