Skip to content

Commit

Permalink
Checkpoint hwpool work. (hpc#172)
Browse files Browse the repository at this point in the history 
Signed-off-by: Samuel K. Gutierrez <[email protected]>
  • Loading branch information
@samuelkgutierrez
samuelkgutierrez authored Jun 13, 2024
1 parent aafc3cc commit eb404da
Show file tree
Hide file tree
Showing 5 changed files with 112 additions and 165 deletions.
75 changes: 9 additions & 66 deletions src/qvi-hwpool.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,39 +46,6 @@
#include "qvi-bbuff-rmi.h"
#include "qvi-utils.h"

int
qvi_hwpool_dup(
const qvi_hwpool_s *const pool,
qvi_hwpool_s **dup
) {
*dup = new qvi_hwpool_s(*pool);
return QV_SUCCESS;
}

int
qvi_hwpool_release_devices(
qvi_hwpool_s *pool
) {
pool->devs.clear();
return QV_SUCCESS;
}

hwloc_const_cpuset_t
qvi_hwpool_cpuset_get(
qvi_hwpool_s *pool
) {
if (!pool) qvi_abort();
return pool->cpu.cpuset.data;
}

const qvi_hwpool_devs_t *
qvi_hwpool_devinfos_get(
qvi_hwpool_s *pool
) {
if (!pool) qvi_abort();
return &pool->devs;
}

#if 0
/**
* Returns whether two cpusets are equal.
Expand Down Expand Up @@ -126,6 +93,7 @@ cpus_available(
* request 1000 1010
* obcpuset' 1110 1111
*/
#if 0
static int
pool_obtain_cpus_by_cpuset(
qvi_hwpool_s *pool,
Expand All @@ -143,6 +111,7 @@ pool_obtain_cpus_by_cpuset(
QVI_UNUSED(request);
return QV_SUCCESS;
}
#endif

/**
* Example:
Expand All @@ -166,46 +135,21 @@ pool_release_cpus_by_cpuset(
#endif

int
qvi_hwpool_add_devices_with_affinity(
qvi_hwpool_s *pool,
qvi_hwloc_t *hwloc
) {
int rc = QV_SUCCESS;
// Iterate over the supported device types.
for (const auto devt : qvi_hwloc_supported_devices()) {
qvi_hwloc_dev_list_t devs;
rc = qvi_hwloc_get_devices_in_bitmap(
hwloc, devt, pool->cpu.cpuset, devs
);
if (rc != QV_SUCCESS) return rc;
for (const auto &dev : devs) {
rc = pool->add_device(qvi_hwpool_dev_s(dev));
if (rc != QV_SUCCESS) return rc;
}
}
return rc;
}

int
qvi_hwpool_obtain_by_cpuset(
qvi_hwpool_s *pool,
qvi_hwpool_s::obtain_new_hwpool_by_cpuset(
qvi_hwloc_t *hwloc,
hwloc_const_cpuset_t cpuset,
qvi_hwpool_s **opool
) {
qvi_hwpool_s *ipool = nullptr;

int rc = pool_obtain_cpus_by_cpuset(pool, cpuset);
if (rc != QV_SUCCESS) goto out;
// We obtained the CPUs, so create the new pool.
rc = qvi_new_rc(&ipool);
qvi_hwpool_s *ipool = nullptr;
int rc = qvi_new_rc(&ipool);
if (rc != QV_SUCCESS) goto out;
// Initialize the hardware pool.
rc = ipool->initialize(cpuset);
if (rc != QV_SUCCESS) goto out;
// Add devices with affinity to the new hardware pool.
// TODO(skg) Acquire devices.
rc = qvi_hwpool_add_devices_with_affinity(ipool, hwloc);
rc = ipool->add_devices_with_affinity(hwloc);
out:
if (rc != QV_SUCCESS) {
qvi_delete(&ipool);
Expand All @@ -215,15 +159,14 @@ qvi_hwpool_obtain_by_cpuset(
}

int
qvi_hwpool_pack(
const qvi_hwpool_s *hwp,
qvi_hwpool_s::pack(
qvi_bbuff_t *buff
) {
return qvi_bbuff_rmi_pack(buff, hwp);
return qvi_bbuff_rmi_pack(buff, this);
}

int
qvi_hwpool_unpack(
qvi_hwpool_s::unpack(
void *buff,
qvi_hwpool_s **hwp
) {
Expand Down
135 changes: 63 additions & 72 deletions src/qvi-hwpool.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,22 @@ struct qvi_hwpool_s {
) {
return cpu.cpuset.set(cpuset);
}
/**
* Obtains a new hardware pool based on the
* affinity encoded in the provided cpuset.
*/
int
obtain_new_hwpool_by_cpuset(
qvi_hwloc_t *hwloc,
hwloc_const_cpuset_t cpuset,
qvi_hwpool_s **opool
);
/** Returns a pointer to the hwpool's cpuset. */
hwloc_const_cpuset_t
get_cpuset(void)
{
return cpu.cpuset.data;
}
/** Adds a qvi_hwpool_dev_s device. */
int
add_device(
Expand All @@ -109,80 +125,55 @@ struct qvi_hwpool_s {
devs.insert({dev.type, shdev});
return QV_SUCCESS;
}
/**
* Adds all devices with affinity to the
* provided, initialized hardware resource pool.
*/
int
add_devices_with_affinity(
qvi_hwloc_t *hwloc
) {
int rc = QV_SUCCESS;
// Iterate over the supported device types.
for (const auto devt : qvi_hwloc_supported_devices()) {
qvi_hwloc_dev_list_t devs;
rc = qvi_hwloc_get_devices_in_bitmap(
hwloc, devt, cpu.cpuset, devs
);
if (rc != QV_SUCCESS) return rc;
for (const auto &dev : devs) {
rc = add_device(qvi_hwpool_dev_s(dev));
if (rc != QV_SUCCESS) return rc;
}
}
return rc;
}
/** Releases all devices in the hwpool. */
int
release_devices(void)
{
devs.clear();
return QV_SUCCESS;
}
/** Returns a const reference to the hardware pool's devices. */
const qvi_hwpool_devs_t &
get_devices(void)
{
return devs;
}
/** Packs the instance into a bbuff. */
int
pack(
qvi_bbuff_t *buff
);
/** Unpacks the buffer and creates a new hardware pool instance. */
static int
unpack(
void *buff,
qvi_hwpool_s **hwp
);
};

/**
*
*/
int
qvi_hwpool_dup(
const qvi_hwpool_s *const pool,
qvi_hwpool_s **dup
);

/**
* Adds all devices with affinity to the provided,
* initialized hardware resource pool.
*/
int
qvi_hwpool_add_devices_with_affinity(
qvi_hwpool_s *pool,
qvi_hwloc_t *hwloc
);

/**
*
*/
int
qvi_hwpool_release_devices(
qvi_hwpool_s *pool
);

/**
*
*/
hwloc_const_cpuset_t
qvi_hwpool_cpuset_get(
qvi_hwpool_s *pool
);

/**
*
*/
const qvi_hwpool_devs_t *
qvi_hwpool_devinfos_get(
qvi_hwpool_s *pool
);

/**
*
*/
int
qvi_hwpool_obtain_by_cpuset(
qvi_hwpool_s *pool,
qvi_hwloc_t *hwloc,
hwloc_const_cpuset_t cpuset,
qvi_hwpool_s **opool
);

/**
*
*/
int
qvi_hwpool_pack(
const qvi_hwpool_s *hwp,
qvi_bbuff_t *buff
);

/**
*
*/
int
qvi_hwpool_unpack(
void *buff,
qvi_hwpool_s **hwp
);

#endif

/*
Expand Down
11 changes: 4 additions & 7 deletions src/qvi-rmi.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -572,8 +572,7 @@ get_intrinsic_scope_user(
qvi_hwpool_s **hwpool
) {
// TODO(skg) Is the cpuset the best way to do this?
return qvi_hwpool_obtain_by_cpuset(
server->hwpool,
return server->hwpool->obtain_new_hwpool_by_cpuset(
server->config.hwloc,
qvi_hwloc_topo_get_cpuset(server->config.hwloc),
hwpool
Expand All @@ -592,8 +591,8 @@ get_intrinsic_scope_proc(
);
if (rc != QV_SUCCESS) goto out;

rc = qvi_hwpool_obtain_by_cpuset(
server->hwpool, server->config.hwloc, cpuset, hwpool
rc = server->hwpool->obtain_new_hwpool_by_cpuset(
server->config.hwloc, cpuset, hwpool
);
out:
if (cpuset) hwloc_bitmap_free(cpuset);
Expand Down Expand Up @@ -816,9 +815,7 @@ server_populate_base_hwpool(
const int rc = server->hwpool->initialize(cpuset);
if (rc != QV_SUCCESS) return rc;
// Add all the discovered devices since the cpuset is the root.
return qvi_hwpool_add_devices_with_affinity(
server->hwpool, hwloc
);
return server->hwpool->add_devices_with_affinity(hwloc);
}

static void *
Expand Down
Loading

0 comments on commit eb404da

Please sign in to comment.