Simplify zgroup handling. (hpc#168)

Signed-off-by: Samuel K. Gutierrez <[email protected]>

src/CMakeLists.txt

@@ -32,7 +32,6 @@ add_library(
       qvi-group.h
       qvi-map.h
       qvi-scope.h
-      qvi-zgroup.h
       qvi-bind.h
       qvi-log.cc
       qvi-utils.cc
@@ -90,10 +89,8 @@ set(
     ../include/quo-vadis-process.h
     qvi-process.h
     qvi-group-process.h
-    qvi-zgroup-process.h
     qvi-process.cc
     qvi-group-process.cc
-    qvi-zgroup-process.cc
     quo-vadis-process.cc
 )
 
@@ -103,11 +100,9 @@ set(
     QUO_VADIS_SOURCE
     ${QUO_VADIS_SOURCE}
     qvi-thread.h
-    qvi-zgroup-thread.h
     qvi-group-thread.h
     ../include/quo-vadis-thread.h
     qvi-thread.cc
-    qvi-zgroup-thread.cc
     qvi-group-thread.cc
     quo-vadis-thread.cc
 )
@@ -148,10 +143,8 @@ if(MPI_FOUND)
           ../include/quo-vadis-mpi.h
           qvi-mpi.h
           qvi-group-mpi.h
-          qvi-zgroup-mpi.h
           qvi-mpi.cc
           qvi-group-mpi.cc
-          qvi-zgroup-mpi.cc
           quo-vadis-mpi.cc
     )
 

src/quo-vadis-mpi.cc

@@ -17,8 +17,9 @@
 #include "quo-vadis-mpi.h"
 #include "qvi-context.h"
 #include "qvi-group-mpi.h"
-#include "qvi-zgroup-mpi.h"
+#include "qvi-group-mpi.h"
 #include "qvi-scope.h"
+#include "qvi-utils.h"
 
 /**
  * Simply a wrapper for our Fortran interface to C interface. No need to expose
@@ -58,41 +59,31 @@ qvi_mpi_context_create(
     MPI_Comm comm,
     qv_context_t **ctx
 ) {
-    int rc = QV_SUCCESS;
-    qv_context_t *ictx = nullptr;
-    qvi_zgroup_mpi_t *izgroup = nullptr;
     // Create base context.
-    rc = qvi_context_new(&ictx);
-    if (rc != QV_SUCCESS) {
-        goto out;
-    }
+    qv_context_t *ictx = nullptr;
+    int rc = qvi_context_new(&ictx);
+    if (rc != QV_SUCCESS) goto out;
     // Create and initialize the base group.
-    rc = qvi_zgroup_mpi_new(&izgroup);
-    if (rc != QV_SUCCESS) {
-        goto out;
-    }
-    // Save zgroup instance pointer to context.
-    ictx->zgroup = izgroup;
+    qvi_zgroup_mpi_s *izgroup;
+    rc = qvi_new_rc(&izgroup);
+    if (rc != QV_SUCCESS) goto out;
 
     rc = izgroup->initialize(comm);
-    if (rc != QV_SUCCESS) {
-        goto out;
-    }
+    if (rc != QV_SUCCESS) goto out;
+
+    ictx->zgroup = izgroup;
     // Connect to RMI server.
     rc = qvi_context_connect_to_server(ictx);
-    if (rc != QV_SUCCESS) {
-        goto out;
-    }
+    if (rc != QV_SUCCESS) goto out;
 
     rc = qvi_bind_stack_init(
         ictx->bind_stack,
-        qvi_mpi_task_get(izgroup->mpi),
+        ictx->zgroup->task(),
         ictx->rmi
     );
 out:
     if (rc != QV_SUCCESS) {
-        (void)qv_mpi_context_free(ictx);
-        ictx = nullptr;
+        qvi_delete(&ictx);
     }
     *ctx = ictx;
     return rc;
@@ -124,11 +115,8 @@ int
 qv_mpi_context_free(
     qv_context_t *ctx
 ) {
-    if (!ctx) {
-        return QV_ERR_INVLD_ARG;
-    }
+    if (!ctx) return QV_ERR_INVLD_ARG;
     try {
-        std::lock_guard<std::mutex> guard(ctx->mutex);
         return qvi_mpi_context_free(ctx);
     }
     qvi_catch_and_return();

src/quo-vadis-process.cc

@@ -13,26 +13,26 @@
 
 #include "quo-vadis-process.h"
 #include "qvi-context.h"
-#include "qvi-zgroup-process.h"
+#include "qvi-group-process.h"
+#include "qvi-utils.h"
 
 static int
 qvi_process_context_create(
     qv_context_t **ctx
 ) {
     int rc = QV_SUCCESS;
-    qv_context_t *ictx = nullptr;
-    qvi_zgroup_process_t *izgroup = nullptr;
     // Create base context.
+    qv_context_t *ictx = nullptr;
     rc = qvi_context_new(&ictx);
     if (rc != QV_SUCCESS) {
         goto out;
     }
     // Create and initialize the base group.
-    rc = qvi_zgroup_process_new(&izgroup);
+    qvi_zgroup_process_s *izgroup;
+    rc = qvi_new_rc(&izgroup);
     if (rc != QV_SUCCESS) {
         goto out;
     }
-    // Save zgroup instance pointer to context.
     ictx->zgroup = izgroup;
     // Connect to RMI server.
     rc = qvi_context_connect_to_server(ictx);
@@ -42,13 +42,12 @@ qvi_process_context_create(
 
     rc = qvi_bind_stack_init(
         ictx->bind_stack,
-        qvi_process_task_get(izgroup->zproc),
+        ictx->zgroup->task(),
         ictx->rmi
     );
 out:
     if (rc != QV_SUCCESS) {
-        (void)qv_process_context_free(ictx);
-        ictx = nullptr;
+        qvi_context_free(&ictx);
     }
     *ctx = ictx;
     return rc;
@@ -58,9 +57,7 @@ int
 qv_process_context_create(
     qv_context_t **ctx
 ) {
-    if (!ctx) {
-        return QV_ERR_INVLD_ARG;
-    }
+    if (!ctx) return QV_ERR_INVLD_ARG;
     try {
         return qvi_process_context_create(ctx);
     }
@@ -81,7 +78,6 @@ qv_process_context_free(
 ) {
     if (!ctx) return QV_ERR_INVLD_ARG;
     try {
-        std::lock_guard<std::mutex> guard(ctx->mutex);
         return qvi_process_context_free(ctx);
     }
     qvi_catch_and_return();

src/quo-vadis-thread.cc

@@ -19,8 +19,9 @@
 #include "quo-vadis-thread.h"
 
 #include "qvi-context.h"
-#include "qvi-zgroup-thread.h"
+#include "qvi-group-thread.h"
 #include "qvi-group-thread.h" // IWYU pragma: keep
+#include "qvi-utils.h"
 
 #ifdef OPENMP_FOUND
 #include <omp.h>
@@ -42,31 +43,19 @@ qv_thread_context_create(
     if (!ctx) {
         return QV_ERR_INVLD_ARG;
     }
-
-    int rc = QV_SUCCESS;
-    qv_context_t *ictx = nullptr;
-    qvi_zgroup_thread_t *izgroup = nullptr;
-
     // Create base context.
-    rc = qvi_context_new(&ictx);
+    qv_context_t *ictx = nullptr;
+    int rc = qvi_context_new(&ictx);
     if (rc != QV_SUCCESS) {
         goto out;
     }
-
     // Create and initialize the base group.
-    rc = qvi_zgroup_thread_new(&izgroup);
+    qvi_zgroup_thread_s *izgroup;
+    rc = qvi_new_rc(&izgroup);
     if (rc != QV_SUCCESS) {
         goto out;
     }
-
-    // Save zgroup instance pointer to context.
     ictx->zgroup = izgroup;
-
-    rc = izgroup->initialize();
-    if (rc != QV_SUCCESS) {
-        goto out;
-    }
-
     // Connect to RMI server.
     rc = qvi_context_connect_to_server(ictx);
     if (rc != QV_SUCCESS) {
@@ -75,7 +64,7 @@ qv_thread_context_create(
 
     rc = qvi_bind_stack_init(
         ictx->bind_stack,
-        qvi_thread_task_get(izgroup->zth),
+        ictx->zgroup->task(),
         ictx->rmi
     );
 out:

src/qvi-bbuff.cc

@@ -26,14 +26,14 @@ struct qvi_bbuff_s {
     size_t size = 0;
     /** Pointer to data backing store. */
     void *data = nullptr;
-    /** Constructor */
+    /** Constructor. */
     qvi_bbuff_s(void)
     {
         capacity = min_growth;
         data = calloc(capacity, sizeof(byte_t));
         if (!data) throw qvi_runtime_error();
     }
-    /** Destructor */
+    /** Destructor. */
     ~qvi_bbuff_s(void)
     {
         if (data) free(data);

src/qvi-context.h

@@ -19,7 +19,7 @@
 
 #include "qvi-common.h" // IWYU pragma: keep
 #include "qvi-rmi.h"
-#include "qvi-zgroup.h"
+#include "qvi-group.h"
 #include "qvi-bind.h"
 
 #ifdef __cplusplus
@@ -30,10 +30,17 @@ extern "C" {
  * The underlying data structure that defines an ultimately opaque QV context.
  */
 struct qv_context_s {
+    /** The context-level mutex. */
+    std::mutex mutex;
+    /** Client-side connection to the RMI. */
     qvi_rmi_client_t *rmi = nullptr;
-    qvi_zgroup_t *zgroup = nullptr;
+    /** The bind stack. */
     qvi_bind_stack_t *bind_stack = nullptr;
-    std::mutex mutex;
+    /**
+     * Zeroth group used for bootstrapping operations that may require
+     * group-level participation from the tasks composing the context.
+     */
+    qvi_group_t *zgroup = nullptr;
     /** Constructor. */
     qv_context_s(void)
     {
@@ -43,13 +50,14 @@ struct qv_context_s {
         rc = qvi_bind_stack_new(&bind_stack);
         if (rc != QV_SUCCESS) throw qvi_runtime_error();
 
-        // The zgroup is polymorphic and created elsewhere.
+        // The zgroup is polymorphic and created in infrastructure-specific
+        // context create functions like qvi_mpi_context_create().
     }
     /** Destructor. */
     ~qv_context_s(void)
     {
         qvi_bind_stack_free(&bind_stack);
-        qvi_zgroup_free(&zgroup);
+        qvi_group_free(&zgroup);
         qvi_rmi_client_free(&rmi);
     }
 };

src/qvi-group-mpi.cc

@@ -15,23 +15,65 @@
  */
 
 #include "qvi-group-mpi.h"
+#include "qvi-utils.h"
+
+int
+qvi_group_mpi_s::intrinsic(
+    qv_scope_intrinsic_t scope,
+    qvi_group_s **group
+) {
+    qvi_group_mpi_t *igroup = nullptr;
+    int rc = qvi_new_rc(&igroup);
+    if (rc != QV_SUCCESS) goto out;
+
+    rc = igroup->initialize(mpi);
+    if (rc != QV_SUCCESS) goto out;
+
+    qvi_mpi_group_id_t mpi_group;
+    // TODO(skg) Finish implementation.
+    switch (scope) {
+        case QV_SCOPE_SYSTEM:
+        case QV_SCOPE_USER:
+        case QV_SCOPE_JOB:
+            mpi_group = QVI_MPI_GROUP_NODE;
+            break;
+        case QV_SCOPE_PROCESS:
+            mpi_group = QVI_MPI_GROUP_SELF;
+            break;
+        default:
+            rc = QV_ERR_INVLD_ARG;
+            break;
+    }
+    if (rc != QV_SUCCESS) goto out;
+
+    rc = qvi_mpi_group_create_from_group_id(
+        mpi, mpi_group, &igroup->mpi_group
+    );
+out:
+    if (rc != QV_SUCCESS) {
+        qvi_delete(&igroup);
+    }
+    *group = igroup;
+    return rc;
+}
 
 int
 qvi_group_mpi_s::self(
     qvi_group_t **child
 ) {
-    qvi_group_mpi_t *ichild = new qvi_group_mpi_t();
+    qvi_group_mpi_t *ichild = nullptr;
+    int rc = qvi_new_rc(&ichild);
+    if (rc != QV_SUCCESS) goto out;
     // Initialize the child with the parent's MPI instance.
-    int rc = ichild->initialize(mpi);
+    rc = ichild->initialize(mpi);
     if (rc != QV_SUCCESS) goto out;
     // Create the underlying group using MPI_COMM_SELF.
     rc = qvi_mpi_group_create_from_mpi_comm(
         mpi, MPI_COMM_SELF, &ichild->mpi_group
     );
 out:
     if (rc != QV_SUCCESS) {
-        delete ichild;
-        ichild = nullptr;
+        qvi_delete(&ichild);
     }
     *child = ichild;
     return rc;
@@ -43,9 +85,11 @@ qvi_group_mpi_s::split(
     int key,
     qvi_group_t **child
 ) {
-    qvi_group_mpi_t *ichild = new qvi_group_mpi_t();
+    qvi_group_mpi_t *ichild = nullptr;
+    int rc = qvi_new_rc(&ichild);
+    if (rc != QV_SUCCESS) goto out;
     // Initialize the child with the parent's MPI instance.
-    int rc = ichild->initialize(mpi);
+    rc = ichild->initialize(mpi);
     if (rc != QV_SUCCESS) goto out;
     // Split this group using MPI.
     rc = qvi_mpi_group_create_from_split(
@@ -54,8 +98,7 @@ qvi_group_mpi_s::split(
     );
 out:
     if (rc != QV_SUCCESS) {
-        delete ichild;
-        ichild = nullptr;
+        qvi_delete(&ichild);
     }
     *child = ichild;
     return rc;