Skip to content

Commit

Permalink
Checkpoint hwpool work. (#99)
Browse files Browse the repository at this point in the history 
Signed-off-by: Samuel K. Gutierrez <[email protected]>
  • Loading branch information
@samuelkgutierrez
samuelkgutierrez authored Mar 20, 2024
1 parent 8240172 commit c51073d
Show file tree
Hide file tree
Showing 5 changed files with 106 additions and 135 deletions.
98 changes: 30 additions & 68 deletions src/qvi-hwpool.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,84 +43,41 @@
// approach using the device IDs instead of the bit positions.

#include "qvi-hwpool.h"
#include "qvi-hwloc.h"
#include "qvi-utils.h"

/**
* Maintains a mapping between a resource ID and associated hint flags.
*/
using qvi_hwpool_resource_id_hint_map_t = std::unordered_map<
uint32_t, qv_scope_create_hint_t
>;

/**
* Maintains information about a pool of hardware resources by resource ID.
*/
struct qvi_hwpool_resource_info_s {
/**
* Vector maintaining the reference count for a given resource ID. There is
* a one-to-one correspondence between resource IDs and addressable vector
* slots since every resource is reference counted. For example, each bit in
* a cpuset will have a corresponding reference count indexed by the bit's
* location in the cpuset.
*/
std::vector<uint32_t> resid_ref_count;
/**
* Maps resource IDs to hint flags that they might have associated with
* them.
*/
qvi_hwpool_resource_id_hint_map_t resid_hint_map;
};

/**
* Base hardware pool resource class.
*/
struct qvi_hwpool_resource_s {
/** Resource info. */
qvi_hwpool_resource_info_s resinfo;
/** Base constructor that does minimal work. */
qvi_hwpool_resource_s(void) = default;
/** Virtual destructor. */
virtual ~qvi_hwpool_resource_s(void) = default;
};
qvi_hwpool_devinfo_s::qvi_hwpool_devinfo_s(
qv_hw_obj_type_t a_type,
int a_id,
cstr_t a_pci_bus_id,
cstr_t a_uuid,
hwloc_const_cpuset_t a_affinity
) : type(a_type)
, id(a_id)
, pci_bus_id(a_pci_bus_id)
, uuid(a_uuid)
{
qvim_rc = qvi_hwloc_bitmap_copy(a_affinity, affinity.data);
}

/**
* Defines a cpuset pool.
*/
struct qvi_hwpool_cpus_s : qvi_hwpool_resource_s {
int qvim_rc = QV_ERR_INTERNAL;
/** The cpuset of the maintained CPUs. */
qvi_hwloc_bitmap_t cpuset;
/** Constructor */
qvi_hwpool_cpus_s(void)
{
qvim_rc = qvi_construct_rc(cpuset);
}
/** Destructor */
virtual ~qvi_hwpool_cpus_s(void) = default;
};
bool
qvi_hwpool_devinfo_s::operator==(
const qvi_hwpool_devinfo_s &x
) const {
return id == x.id && type == x.type;
}

struct qvi_hwpool_s {
int qvim_rc = QV_ERR_INTERNAL;
/** The CPUs in the resource pool. */
qvi_hwpool_cpus_s cpus;
/** Device information. */
qvi_hwpool_devinfos_t devinfos;
/** The obtained cpuset of this resource pool. */
hwloc_bitmap_t obcpuset = nullptr;

/** Constructor */
qvi_hwpool_s(void)
{
qvim_rc = qvi_construct_rc(cpus);
if (qvim_rc != QV_SUCCESS) return;

qvim_rc = qvi_hwloc_bitmap_calloc(&obcpuset);
}

~qvi_hwpool_s(void)
{
qvi_hwloc_bitmap_free(&obcpuset);
}
/** Destructor */
~qvi_hwpool_s(void) = default;
};

int
Expand Down Expand Up @@ -189,21 +146,21 @@ qvi_hwpool_new_line_from_hwpool(
iline->devinfos[idx].id = dinfo.second->id;
// Duplicate the cpuset
rc = qvi_hwloc_bitmap_dup(
dinfo.second->affinity,
dinfo.second->affinity.data,
&iline->devinfos[idx].affinity
);
if (rc != QV_SUCCESS) break;
nw = asprintf(
&iline->devinfos[idx].pci_bus_id,
"%s", dinfo.second->pci_bus_id
"%s", dinfo.second->pci_bus_id.c_str()
);
if (nw == -1) {
rc = QV_ERR_OOR;
break;
}
nw = asprintf(
&iline->devinfos[idx].uuid,
"%s", dinfo.second->uuid
"%s", dinfo.second->uuid.c_str()
);
if (nw == -1) {
rc = QV_ERR_OOR;
Expand Down Expand Up @@ -329,12 +286,17 @@ pool_obtain_cpus_by_cpuset(
qvi_hwpool_t *pool,
hwloc_const_cpuset_t request
) {
#if 0
int hwrc = hwloc_bitmap_or(
pool->obcpuset,
pool->obcpuset,
request
);
return (hwrc == 0 ? QV_SUCCESS : QV_ERR_HWLOC);
#endif
QVI_UNUSED(pool);
QVI_UNUSED(request);
return QV_SUCCESS;
}

/**
Expand Down
110 changes: 65 additions & 45 deletions src/qvi-hwpool.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,63 @@
#define QVI_HWPOOL_H

#include "qvi-common.h"
#include "qvi-hwloc.h"
#include "qvi-line.h"
#include "qvi-utils.h"

/**
* Maintains a mapping between a resource ID and associated hint flags.
*/
using qvi_hwpool_resource_id_hint_map_t = std::unordered_map<
uint32_t, qv_scope_create_hint_t
>;

/**
* Maintains information about a pool of hardware resources by resource ID.
*/
struct qvi_hwpool_resource_info_s {
/**
* Vector maintaining the reference count for a given resource ID. There is
* a one-to-one correspondence between resource IDs and addressable vector
* slots since every resource is reference counted. For example, each bit in
* a cpuset will have a corresponding reference count indexed by the bit's
* location in the cpuset.
*/
std::vector<uint32_t> resid_ref_count;
/**
* Maps resource IDs to hint flags that
* they might have associated with them.
*/
qvi_hwpool_resource_id_hint_map_t resid_hint_map;
};

/**
* Base hardware pool resource class.
*/
struct qvi_hwpool_resource_s {
/** Resource info. */
qvi_hwpool_resource_info_s resinfo;
/** Base constructor that does minimal work. */
qvi_hwpool_resource_s(void) = default;
/** Virtual destructor. */
virtual ~qvi_hwpool_resource_s(void) = default;
};

/**
* Defines a cpuset pool.
*/
struct qvi_hwpool_cpus_s : qvi_hwpool_resource_s {
int qvim_rc = QV_ERR_INTERNAL;
/** The cpuset of the maintained CPUs. */
qvi_hwloc_bitmap_t cpuset;
/** Constructor */
qvi_hwpool_cpus_s(void)
{
qvim_rc = qvi_construct_rc(cpuset);
}
/** Destructor */
virtual ~qvi_hwpool_cpus_s(void) = default;
};

/** Device information. */
struct qvi_hwpool_devinfo_s {
Expand All @@ -27,48 +83,24 @@ struct qvi_hwpool_devinfo_s {
/** Device ID. */
int id = 0;
/** The PCI bus ID. */
char *pci_bus_id = nullptr;
std::string pci_bus_id;
/** UUID */
char *uuid = nullptr;
std::string uuid;
/** The bitmap encoding CPU affinity. */
hwloc_bitmap_t affinity = nullptr;
qvi_hwloc_bitmap_t affinity;
/** Constructor */
qvi_hwpool_devinfo_s(
qv_hw_obj_type_t t,
int i,
qv_hw_obj_type_t type,
int id,
cstr_t pci_bus_id,
cstr_t uuid,
hwloc_const_cpuset_t c
) : type(t)
, id(i)
{
int nw = asprintf(&this->pci_bus_id, "%s", pci_bus_id);
if (nw == -1) {
qvim_rc = QV_ERR_OOR;
return;
}

nw = asprintf(&this->uuid, "%s", uuid);
if (nw == -1) {
qvim_rc = QV_ERR_OOR;
return;
}

qvim_rc = qvi_hwloc_bitmap_dup(c, &affinity);
}
hwloc_const_cpuset_t affinity
);
/** Destructor */
~qvi_hwpool_devinfo_s(void)
{
qvi_hwloc_bitmap_free(&affinity);
free(pci_bus_id);
free(uuid);
}
~qvi_hwpool_devinfo_s(void) = default;
/** Equality operator. */
bool
operator==(const qvi_hwpool_devinfo_s &x) const
{
return id == x.id && type == x.type;
}
operator==(const qvi_hwpool_devinfo_s &x) const;
};

using qvi_hwpool_devinfos_t = std::multimap<
Expand Down Expand Up @@ -179,18 +211,6 @@ qvi_hwpool_obtain_by_cpuset(
qvi_hwpool_t **opool
);

/**
*
*/
int
qvi_hwpool_split_devices(
qvi_hwpool_t **pools,
int npools,
qvi_hwloc_t *hwloc,
int ncolors,
int color
);

/**
*
*/
Expand Down
1 change: 1 addition & 0 deletions src/qvi-map.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ qvi_map_packed(
/**
* Maps round-robin over the given resources.
*/
// TODO(skg) Verify name and functionality, rename?
int
qvi_map_spread(
qvi_map_t &map,
Expand Down
12 changes: 0 additions & 12 deletions src/qvi-map.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,6 @@
#include "qvi-common.h"
#include "qvi-hwloc.h" // IWYU pragma: keep

#ifdef __cplusplus
extern "C" {
#endif

/**
* Modes used to influence how affinity preserving mapping is done.
*/
Expand All @@ -47,12 +43,6 @@ qvi_map_maxiperk(
uint_t k
);

#ifdef __cplusplus
}
#endif

#ifdef __cplusplus

/** Maintains a mapping between 'From IDs' to 'To IDs'. */
using qvi_map_t = std::map<int, int>;

Expand Down Expand Up @@ -155,8 +145,6 @@ qvi_map_flatten(

#endif

#endif

/*
* vim: ft=cpp ts=4 sts=4 sw=4 expandtab
*/
20 changes: 10 additions & 10 deletions src/qvi-scope.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -675,9 +675,9 @@ global_split_devices_user_defined(
gsplit.hwpools[i],
c2d.second->type,
c2d.second->id,
c2d.second->pci_bus_id,
c2d.second->uuid,
c2d.second->affinity
c2d.second->pci_bus_id.c_str(),
c2d.second->uuid.c_str(),
c2d.second->affinity.data
);
if (rc != QV_SUCCESS) break;
}
Expand Down Expand Up @@ -716,7 +716,7 @@ qvi_global_split_devices_affinity_preserving(
// Store device affinities.
qvi_map_cpusets_t devaffs;
for (auto &dev : devs) {
devaffs.push_back(dev->affinity);
devaffs.push_back(dev->affinity.data);
}

qvi_map_t map;
Expand All @@ -737,9 +737,9 @@ qvi_global_split_devices_affinity_preserving(
gsplit.hwpools[pooli],
devs[devid]->type,
devs[devid]->id,
devs[devid]->pci_bus_id,
devs[devid]->uuid,
devs[devid]->affinity
devs[devid]->pci_bus_id.c_str(),
devs[devid]->uuid.c_str(),
devs[devid]->affinity.data
);
if (rc != QV_SUCCESS) break;
}
Expand Down Expand Up @@ -851,7 +851,7 @@ global_split_get_new_osdev_cpusets(
rc = qvi_hwloc_bitmap_calloc(&result[affi]);
if (rc != QV_SUCCESS) goto out;
// Copy the device's affinity to our list of device affinities.
rc = qvi_hwloc_bitmap_copy(dinfo.second->affinity, result[affi++]);
rc = qvi_hwloc_bitmap_copy(dinfo.second->affinity.data, result[affi++]);
if (rc != QV_SUCCESS) goto out;
}
out:
Expand Down Expand Up @@ -1181,10 +1181,10 @@ qvi_scope_get_device_id(

switch (id_type) {
case (QV_DEVICE_ID_UUID):
nw = asprintf(dev_id, "%s", finfo->uuid);
nw = asprintf(dev_id, "%s", finfo->uuid.c_str());
break;
case (QV_DEVICE_ID_PCI_BUS_ID):
nw = asprintf(dev_id, "%s", finfo->pci_bus_id);
nw = asprintf(dev_id, "%s", finfo->pci_bus_id.c_str());
break;
case (QV_DEVICE_ID_ORDINAL):
nw = asprintf(dev_id, "%d", finfo->id);
Expand Down

0 comments on commit c51073d

Please sign in to comment.