Add first cut of Pthread group split support.

src/quo-vadis-pthread.cc

@@ -24,14 +24,14 @@
 #include "qvi-scope.h"
 #include "qvi-utils.h"
 
-struct qvi_pthread_args_s {
+struct qvi_pthread_args {
     qv_scope_t *scope = nullptr;
     qvi_pthread_routine_fun_ptr_t th_routine = nullptr;
     void *th_routine_argp = nullptr;
+    /** Default constructor. */
+    qvi_pthread_args(void) = delete;
     /** Constructor. */
-    qvi_pthread_args_s(void) = delete;
-    /** Constructor. */
-    qvi_pthread_args_s(
+    qvi_pthread_args(
         qv_scope_t *scope_a,
         qvi_pthread_routine_fun_ptr_t th_routine_a,
         void *th_routine_argp_a
@@ -41,15 +41,15 @@ struct qvi_pthread_args_s {
 };
 
 static void *
-qvi_pthread_routine(
+qvi_pthread_start_routine(
     void *arg
 ) {
-    qvi_pthread_args_s *arg_ptr = (qvi_pthread_args_s *)arg;
+    qvi_pthread_args *args = (qvi_pthread_args *)arg;
     // TODO(skg) Check return code.
-    arg_ptr->scope->bind_push();
+    args->scope->bind_push();
 
-    void *const ret = arg_ptr->th_routine(arg_ptr->th_routine_argp);
-    qvi_delete(&arg_ptr);
+    void *const ret = args->th_routine(args->th_routine_argp);
+    qvi_delete(&args);
     pthread_exit(ret);
 }
 
@@ -102,17 +102,21 @@ qv_pthread_create(
     qv_scope_t *scope
 ) {
     // Memory will be freed in qv_pthread_routine to avoid memory leaks.
-    qvi_pthread_args_s *arg_ptr = nullptr;
-    int rc = qvi_new(&arg_ptr, scope, thread_routine, arg);
+    qvi_pthread_args *pthread_start_args = nullptr;
+    int rc = qvi_new(&pthread_start_args, scope, thread_routine, arg);
     // Since this is meant to behave similarly to
     // pthread_create(), return a reasonable errno.
     if (qvi_unlikely(rc != QV_SUCCESS)) return ENOMEM;
-
+    // Note: The provided scope should have been created by
+    // qv_pthread_scope_split*. That is why we can safely cast the scope's
+    // underlying group it to a qvi_group_pthread *.
     auto group = dynamic_cast<qvi_group_pthread *>(scope->group());
-    qvi_pthread_group_pthread_create_args_s *cargs = nullptr;
-    rc = qvi_new(&cargs, group->thgroup, qvi_pthread_routine, arg_ptr);
+    qvi_pthread_group_pthread_create_args *cargs = nullptr;
+    rc = qvi_new(
+        &cargs, group->thgroup, qvi_pthread_start_routine, pthread_start_args
+    );
     if (qvi_unlikely(rc != QV_SUCCESS)) {
-        qvi_delete(&arg_ptr);
+        qvi_delete(&pthread_start_args);
         return ENOMEM;
     }
     return pthread_create(

src/qvi-group-omp.h

@@ -72,7 +72,7 @@ struct qvi_group_omp : public qvi_group {
     );
 
     virtual int
-    thsplit(
+    thread_split(
         int,
         qvi_group **
     ) {

src/qvi-group-pthread.cc

@@ -15,12 +15,36 @@
 #include "qvi-utils.h"
 
 qvi_group_pthread::qvi_group_pthread(
+    qvi_pthread_group_context *ctx,
     int group_size
 ) {
-    const int rc = qvi_new(&thgroup, group_size, 0);
+    int rc = QV_SUCCESS;
+    // A context pointer was not provided, so create a new one.
+    if (!ctx) {
+        rc = qvi_new(&m_context);
+        if (qvi_unlikely(rc != QV_SUCCESS)) throw qvi_runtime_error();
+    }
+    else {
+        // Else a context pointer was provided, so use it.
+        m_context = ctx;
+        m_context->retain();
+    }
+    //
+    rc = qvi_new(&thgroup, m_context, group_size);
     if (qvi_unlikely(rc != QV_SUCCESS)) throw qvi_runtime_error();
 }
 
+qvi_group_pthread::qvi_group_pthread(
+    qvi_pthread_group_context *ctx,
+    qvi_pthread_group *thread_group
+) {
+    assert(ctx && thread_group);
+    m_context = ctx;
+    m_context->retain();
+    //
+    thgroup = thread_group;
+}
+
 qvi_group_pthread::~qvi_group_pthread(void)
 {
     qvi_delete(&thgroup);
@@ -40,13 +64,18 @@ qvi_group_pthread::split(
     int key,
     qvi_group **child
 ) {
+    // NOTE: This is a collective call across
+    // ALL threads in the parent thread group.
     qvi_group_pthread *ichild = nullptr;
-    int rc  = qvi_new(&ichild);
+
+    qvi_pthread_group *ithgroup = nullptr;
+    int rc = thgroup->split(color, key, &ithgroup);
     if (qvi_unlikely(rc != QV_SUCCESS)) goto out;
 
-    rc = thgroup->split(color, key, &ichild->thgroup);
+    rc = qvi_new(&ichild, m_context, ithgroup);
 out:
     if (qvi_unlikely(rc != QV_SUCCESS)) {
+        qvi_delete(&ithgroup);
         qvi_delete(&ichild);
     }
     *child = ichild;

src/qvi-group-pthread.h

@@ -20,14 +20,32 @@
 #include "qvi-bbuff.h"
 
 struct qvi_group_pthread : public qvi_group {
+protected:
+    /**
+     * Points to per-process, per-thread_split()
+     * information that maintains Pthread group context.
+     */
+    qvi_pthread_group_context *m_context = nullptr;
+public:
     /** Underlying group instance. */
     qvi_pthread_group *thgroup = nullptr;
-    /** Constructor. */
-    qvi_group_pthread(void) = default;
-    /** Constructor. */
+    /** Default constructor. */
+    qvi_group_pthread(void) = delete;
+    /**
+     * Constructor that is called by the parent process to setup
+     * base infrastructure required during a thread_split().
+     */
     qvi_group_pthread(
+        qvi_pthread_group_context *ctx,
         int group_size
     );
+    /**
+     * Constructor that is collective across ALL threads in the parent group.
+     */
+    qvi_group_pthread(
+        qvi_pthread_group_context *ctx,
+        qvi_pthread_group *thread_group
+    );
     /** Destructor. */
     virtual ~qvi_group_pthread(void);
 
@@ -59,7 +77,10 @@ struct qvi_group_pthread : public qvi_group {
     make_intrinsic(
         qv_scope_intrinsic_t
     ) {
-        // Nothing to do.
+        // Nothing to do here because a Pthread group cannot be created outside
+        // of another group. For example, a thread_split can be called from a
+        // process context, which can be an intrinsic group, but not from a
+        // threaded context alone.
         return QV_SUCCESS;
     }
 
@@ -69,14 +90,15 @@ struct qvi_group_pthread : public qvi_group {
     );
 
     virtual int
-    thsplit(
+    thread_split(
         int,
         qvi_group **
     ) {
-        // TODO(skg)
         return QV_ERR_NOT_SUPPORTED;
     }
-
+    /**
+     * This is a collective call across all threads in the parent thread group.
+     */
     virtual int
     split(
         int color,

src/qvi-group.cc

@@ -1,6 +1,6 @@
 /* -*- Mode: C++; c-basic-offset:4; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2021-2024 Triad National Security, LLC
+ * Copyright (c) 2021-2025 Triad National Security, LLC
  *                         All rights reserved.
  *
  * This file is part of the quo-vadis project. See the LICENSE file at the
@@ -23,12 +23,16 @@ qvi_group::hwloc(void)
 }
 
 int
-qvi_group::thsplit(
+qvi_group::thread_split(
     int nthreads,
     qvi_group **child
 ) {
     qvi_group_pthread *ichild = nullptr;
-    const int rc = qvi_new(&ichild, nthreads);
+    // This is the entry point for creating a new thread group, so nullptr
+    // passed to signal that a new context must be created by the new
+    // qvi_group_pthread. Also note this is called by a single thread of
+    // execution (i.e., the parent process).
+    const int rc = qvi_new(&ichild, nullptr, nthreads);
     if (qvi_unlikely(rc != QV_SUCCESS)) {
         qvi_delete(&ichild);
     }
@@ -44,14 +48,29 @@ qvi_group::next_id(
     // infrastructure (e.g., QVI_MPI_GROUP_WORLD) will never equal or exceed
     // this value.
     static std::atomic<qvi_group_id_t> group_id(64);
-    if (group_id == UINT64_MAX) {
-        qvi_log_error("Group ID space exhausted");
+    if (qvi_unlikely(group_id == UINT64_MAX)) {
+        qvi_log_error("Group ID space exhausted.");
         return QV_ERR_OOR;
     }
     *gid = group_id++;
     return QV_SUCCESS;
 }
 
+int
+qvi_group::next_ids(
+    size_t n,
+    std::vector<qvi_group_id_t> &gids
+) {
+    gids.resize(n);
+    for (size_t i = 0; i < n; ++i) {
+        qvi_group_id_t gid = 0;
+        const int rc = next_id(&gid);
+        if (qvi_unlikely(rc != QV_SUCCESS)) return rc;
+        gids[i] = gid;
+    }
+    return QV_SUCCESS;
+}
+
 /*
  * vim: ft=cpp ts=4 sts=4 sw=4 expandtab
  */

src/qvi-group.h

@@ -61,10 +61,11 @@ struct qvi_group : qvi_refc {
         qvi_group **child
     ) = 0;
     /**
-     * Creates a new thread group by splitting off of the caller's group.
+     * Creates a new thread group by splitting off of the calling process'
+     * group.
      */
     virtual int
-    thsplit(
+    thread_split(
         int nthreads,
         qvi_group **child
     );
@@ -98,6 +99,12 @@ struct qvi_group : qvi_refc {
     next_id(
         qvi_group_id_t *gid
     );
+    /** Populates gids with n unique group IDs after each call. */
+    static int
+    next_ids(
+        size_t n,
+        std::vector<qvi_group_id_t> &gids
+    );
 };
 
 #endif