Add first cut of qv_thread_scope_split_*. (#179)

Add Guillaume's first cut of Edgar's new scope split interfaces with
cleanups.

Signed-off-by: Samuel K. Gutierrez <[email protected]>
Co-authored-by: Guillaume Mercier <[email protected]>
Co-authored-by: Edgar A. Leon <[email protected]>

include/quo-vadis-thread.h

@@ -97,6 +97,29 @@ qv_pthread_create(
     qv_context_t *ctx,
     qv_scope_t *scope
 );
+
+
+int
+qv_thread_scope_split_at(
+    qv_context_t *ctxt,
+    qv_scope_t *scope,
+    qv_hw_obj_type_t type,
+    int *color_array,
+    int nthreads,
+    qv_scope_t ***subscope
+);
+
+
+int
+qv_thread_scope_split(
+    qv_context_t *ctxt,
+    qv_scope_t *scope,
+    int npieces,
+    int *color_array,
+    int nthreads,
+    qv_scope_t ***subscope
+);
+
 #else
 /**                                                                                                     
  * Layout for fine-grain binding                                                                        

include/quo-vadis.h

@@ -106,7 +106,7 @@ typedef enum {
 } qv_hw_obj_type_t;
 
 /**
- * Binding string representaiton formats.
+ * Binding string representation formats.
  */
 typedef enum {
     QV_BIND_STRING_AS_BITMAP = 0,

src/quo-vadis-thread.cc

@@ -1,4 +1,4 @@
- /* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */
+/* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2020-2024 Triad National Security, LLC
  *                         All rights reserved.
@@ -23,6 +23,11 @@
 #include "qvi-group-thread.h" // IWYU pragma: keep
 #include "qvi-utils.h"
 
+
+#include "qvi-scope.h"
+//#include "qvi-hwpool.h"
+//#include "qvi-map.h"
+
 #ifdef OPENMP_FOUND
 #include <omp.h>
 #endif
@@ -76,26 +81,70 @@ qv_thread_context_create(
     return rc;
 }
 
+int
+qv_thread_scope_split(
+    qv_context_t *ctxt,
+    qv_scope_t *scope,
+    int npieces,
+    int *color_array,
+    int nthreads,
+    qv_scope_t ***subscope
+){
+    int rc = QV_SUCCESS;
+
+    QVI_UNUSED(ctxt);
+
+    qv_hw_obj_type_t type;
+
+    rc = qvi_scope_obj_type(scope, npieces, &type);
+    if (rc != QV_SUCCESS) {
+        rc = QV_ERR_INVLD_ARG;
+        goto out;
+    }
+    //fprintf(stdout,"=================== Type for split is %i\n",(int)type);
+    rc = qvi_scope_ksplit_at(scope, type, color_array, nthreads, subscope);
+
+ out:
+    return rc;
+}
+
+int
+qv_thread_scope_split_at(
+    qv_context_t *ctxt,
+    qv_scope_t *scope,
+    qv_hw_obj_type_t type,
+    int *color_array,
+    int nthreads,
+    qv_scope_t ***subscope
+){
+    int rc = QV_SUCCESS;
+
+    QVI_UNUSED(ctxt);
+
+    rc = qvi_scope_ksplit_at(scope, type, color_array, nthreads, subscope);
+
+    return rc;
+}
+
 #ifndef USE_LAYOUTS
 //New interface
 void *
 qv_thread_routine(
     void * arg
 ) {
     qv_thread_args_t *arg_ptr = (qv_thread_args_t *) arg;
-    //  fprintf(stdout,"qv_thread_routine: ctx=%p scope=%p\n", qvp->ctx, qvp->scope);
 
     int rc = qv_bind_push(arg_ptr->ctx, arg_ptr->scope);
+
+
     if (rc != QV_SUCCESS) {
-        // char const *ers = "qv_bind_push() failed";
-        // qvi_test_panic("%s (rc=%s)", ers, qv_strerr(rc));
+        fprintf(stdout,"==== Bind Push error \n");
         pthread_exit(NULL);
     }
 
     void *ret = arg_ptr->thread_routine(arg_ptr->arg);
 
     // Free memory allocated in qv_pthread_create
-    //free(arg_ptr);
     delete arg_ptr;
     pthread_exit(ret);
 }
@@ -111,13 +160,12 @@ qv_pthread_create(
 ) {
      // Memory will be freed in qv_thread_routine to avoid  memory leaks
      qv_thread_args_t *arg_ptr = qvi_new qv_thread_args_t();
-     //(qv_thread_args_t *)malloc(sizeof(qv_thread_args_t));
+
      arg_ptr->ctx = ctx;
      arg_ptr->scope = scope;
      arg_ptr->thread_routine = thread_routine;
      arg_ptr->arg = arg;
 
-    // fprintf(stdout,"qv_pthread_create: ctx=%p scope=%p\n", ctx, scope);
     return pthread_create(thread, attr, qv_thread_routine, arg_ptr);
 }
 #else // USE_LAYOUTS
@@ -267,7 +315,7 @@ qv_thread_layout_apply( //use map interface if necessary
   int rc = QV_SUCCESS;
   qv_context_t *parent_ctx = th_args.ctx;
   qv_scope_t *parent_scope = th_args.scope;
-  qv_layout_t *thread_layout = th_args.thread_layout;                                                                                                                                                              int thr_idx = th_args.th_id;
+  qv_layout_t *thread_layout = th_args.thread_layout;
   int num_thr = th_args.num_th;
 
   /* Brand new case: compute and cache as much as possible*/

src/qvi-bbuff-rmi.h

@@ -174,6 +174,15 @@ qvi_bbuff_rmi_pack_type_picture(
     picture += "i";
 }
 
+template<>
+inline void
+qvi_bbuff_rmi_pack_type_picture(
+    std::string &picture,
+    qv_hw_obj_type_t *
+) {
+    picture += "i";
+}
+
 template<>
 inline void
 qvi_bbuff_rmi_pack_type_picture(

src/qvi-context.h

@@ -57,6 +57,7 @@ struct qv_context_s {
         qvi_delete(&zgroup);
         qvi_rmi_client_free(&rmi);
     }
+
 };
 
 /**

src/qvi-hwloc.cc

@@ -111,6 +111,40 @@ qvi_hwloc_get_obj_type(
     }
 }
 
+// TODO(skg) Merge with qvi_hwloc_get_obj_type()?
+static qv_hw_obj_type_t
+qvi_hwloc_convert_obj_type(
+   hwloc_obj_type_t external
+) {
+    switch(external) {
+        case(HWLOC_OBJ_MACHINE):
+             return QV_HW_OBJ_MACHINE;
+        case(HWLOC_OBJ_PACKAGE):
+             return QV_HW_OBJ_PACKAGE;
+        case(HWLOC_OBJ_CORE):
+             return QV_HW_OBJ_CORE;
+        case(HWLOC_OBJ_PU):
+             return QV_HW_OBJ_PU;
+        case(HWLOC_OBJ_L1CACHE):
+             return QV_HW_OBJ_L1CACHE;
+        case(HWLOC_OBJ_L2CACHE):
+             return QV_HW_OBJ_L2CACHE;
+        case(HWLOC_OBJ_L3CACHE):
+             return QV_HW_OBJ_L3CACHE;
+        case(HWLOC_OBJ_L4CACHE):
+             return QV_HW_OBJ_L4CACHE;
+        case(HWLOC_OBJ_L5CACHE):
+             return QV_HW_OBJ_L5CACHE;
+        case(HWLOC_OBJ_NUMANODE):
+            return QV_HW_OBJ_NUMANODE;
+        case(HWLOC_OBJ_OS_DEVICE):
+            return QV_HW_OBJ_GPU;
+        default:
+            // This is an internal development error.
+            qvi_abort();
+    }
+}
+
 bool
 qvi_hwloc_obj_type_is_host_resource(
     qv_hw_obj_type_t type
@@ -1082,6 +1116,28 @@ qvi_hwloc_get_nobjs_in_cpuset(
     return QV_ERR_INTERNAL;
 }
 
+int
+qvi_hwloc_get_obj_type_in_cpuset(
+    qvi_hwloc_t *hwl,
+    int npieces,
+    hwloc_const_cpuset_t cpuset,
+    qv_hw_obj_type_t *target_obj
+) {
+    hwloc_topology_t topo = hwl->topo;
+    hwloc_obj_t obj =  hwloc_get_first_largest_obj_inside_cpuset(topo, cpuset);
+    int num = hwloc_get_nbobjs_inside_cpuset_by_type(topo, cpuset, obj->type);
+    int depth = obj->depth;
+
+    while(!((num > npieces) && (!hwloc_obj_type_is_cache(obj->type)))){
+        depth++;
+        obj = hwloc_get_obj_inside_cpuset_by_depth(topo, cpuset, depth, 0);
+        num = hwloc_get_nbobjs_inside_cpuset_by_type(topo, cpuset, obj->type);
+    }
+
+    *target_obj = qvi_hwloc_convert_obj_type(obj->type);
+    return QV_SUCCESS;
+}
+
 int
 qvi_hwloc_get_obj_in_cpuset_by_depth(
     qvi_hwloc_t *hwl,

src/qvi-hwloc.h

@@ -286,6 +286,17 @@ qvi_hwloc_get_nobjs_in_cpuset(
     int *nobjs
 );
 
+/**
+ *
+ */
+int
+qvi_hwloc_get_obj_type_in_cpuset(
+    qvi_hwloc_t *hwl,
+    int npieces,
+    hwloc_const_cpuset_t cpuset,
+    qv_hw_obj_type_t *target_obj
+);
+
 /**
  *
  */

src/qvi-rmi.cc

@@ -67,6 +67,7 @@ typedef enum qvi_rpc_funid_e {
     FID_TASK_SET_CPUBIND_FROM_CPUSET,
     FID_OBJ_TYPE_DEPTH,
     FID_GET_NOBJS_IN_CPUSET,
+    FID_GET_OBJ_TYPE_IN_CPUSET,
     FID_GET_DEVICE_IN_CPUSET,
     FID_SCOPE_GET_INTRINSIC_HWPOOL
 } qvi_rpc_funid_t;
@@ -537,6 +538,31 @@ rpc_ssi_get_nobjs_in_cpuset(
     return qvrc;
 }
 
+static int
+rpc_ssi_get_obj_type_in_cpuset(
+    qvi_rmi_server_t *server,
+    qvi_msg_header_t *hdr,
+    void *input,
+    qvi_bbuff_t **output
+) {
+    int npieces = 0;
+    hwloc_cpuset_t cpuset = nullptr;
+    int qvrc = qvi_bbuff_rmi_unpack(
+        input, &npieces, &cpuset
+    );
+    if (qvrc != QV_SUCCESS) return qvrc;
+
+    qv_hw_obj_type_t target_obj;
+    const int rpcrc = qvi_hwloc_get_obj_type_in_cpuset(
+        server->config.hwloc, npieces, cpuset, &target_obj
+    );
+
+    qvrc = rpc_pack(output, hdr->fid, rpcrc, target_obj);
+
+    hwloc_bitmap_free(cpuset);
+    return qvrc;
+}
+
 static int
 rpc_ssi_get_device_in_cpuset(
     qvi_rmi_server_t *server,
@@ -656,6 +682,7 @@ static const qvi_rpc_fun_ptr_t rpc_dispatch_table[] = {
     rpc_ssi_task_set_cpubind_from_cpuset,
     rpc_ssi_obj_type_depth,
     rpc_ssi_get_nobjs_in_cpuset,
+    rpc_ssi_get_obj_type_in_cpuset,
     rpc_ssi_get_device_in_cpuset,
     rpc_ssi_scope_get_intrinsic_hwpool
 };
@@ -1102,6 +1129,30 @@ qvi_rmi_get_nobjs_in_cpuset(
     return rpcrc;
 }
 
+int
+qvi_rmi_get_obj_type_in_cpuset(
+    qvi_rmi_client_t *client,
+    int npieces,
+    hwloc_const_cpuset_t cpuset,
+    qv_hw_obj_type_t *target_obj
+) {
+    int qvrc = rpc_req(
+        client->zsock,
+        FID_GET_OBJ_TYPE_IN_CPUSET,
+        npieces,
+        cpuset
+    );
+    if (qvrc != QV_SUCCESS) return qvrc;
+
+    // Should be set by rpc_rep, so assume an error.
+    int rpcrc = QV_ERR_MSG;
+    qvrc = rpc_rep(client->zsock, &rpcrc, target_obj);
+    if (qvrc != QV_SUCCESS) return qvrc;
+
+    return rpcrc;
+}
+
+
 int
 qvi_rmi_get_device_in_cpuset(
     qvi_rmi_client_t *client,

src/qvi-rmi.h

@@ -154,6 +154,17 @@ qvi_rmi_get_nobjs_in_cpuset(
     int *nobjs
 );
 
+/**
+ *
+ */
+int
+qvi_rmi_get_obj_type_in_cpuset(
+    qvi_rmi_client_t *client,
+    int npieces,
+    hwloc_const_cpuset_t cpuset,
+    qv_hw_obj_type_t *target_obj
+);
+
 /**
  *
  */