Add first cut of new automatic thread splitting. (#313)

Signed-off-by: Samuel K. Gutierrez <[email protected]>

include/quo-vadis-pthread.h

@@ -50,6 +50,9 @@ typedef enum {
     QV_POLICY_CHOOSE     = 5
 } qv_pthread_placement_t;
 
+int *const QV_PTHREAD_SCOPE_SPLIT_PACKED = (int *)0x00000001;
+int *const QV_PTHREAD_SCOPE_SPLIT_SPREAD = (int *)0x00000002;
+
 /**
  * Similar to pthread_create(3).
  */

src/quo-vadis-pthread.cc

@@ -59,22 +59,51 @@ qvi_pthread_start_routine(
     pthread_exit(ret);
 }
 
+static int
+split_color_fixup(
+    int *kcolors,
+    int k,
+    std::vector<int> &kcolorsp
+) {
+    int real_color = QV_SCOPE_SPLIT_UNDEFINED;
+
+    if (kcolors == QV_PTHREAD_SCOPE_SPLIT_PACKED) {
+        real_color = QV_SCOPE_SPLIT_PACKED;
+    }
+    else if (kcolors == QV_PTHREAD_SCOPE_SPLIT_SPREAD) {
+        real_color = QV_SCOPE_SPLIT_SPREAD;
+    }
+    // Nothing to do. An automatic coloring was not requested.
+    if (real_color == QV_SCOPE_SPLIT_UNDEFINED) {
+        return QV_SUCCESS;
+    }
+    // An automatic coloring was requested.
+    kcolorsp.resize(k);
+    std::fill(kcolorsp.begin(), kcolorsp.end(), real_color);
+    return QV_SUCCESS;
+}
+
 int
 qv_pthread_scope_split(
     qv_scope_t *scope,
     int npieces,
-    int *color_array,
-    int nthreads,
+    int *kcolors,
+    int k,
     qv_scope_t ***subscopes
 ) {
-    const bool invalid_args = !scope || npieces < 0 || !color_array ||
-                              nthreads < 0 || !subscopes;
+    const bool invalid_args = !scope || npieces < 0 || !kcolors ||
+                              k < 0 || !subscopes;
     if (qvi_unlikely(invalid_args)) {
         return QV_ERR_INVLD_ARG;
     }
     try {
+        std::vector<int> color_fixup;
+        const int rc = split_color_fixup(kcolors, k, color_fixup);
+        if (qvi_unlikely(rc != QV_SUCCESS)) return rc;
+        // Set the colors array to the appropriate data.
+        int *kcolorsp = color_fixup.empty() ? kcolors : color_fixup.data();
         return scope->thread_split(
-            npieces, color_array, nthreads, QV_HW_OBJ_LAST, subscopes
+            npieces, kcolorsp, k, QV_HW_OBJ_LAST, subscopes
         );
     }
     qvi_catch_and_return();
@@ -92,7 +121,12 @@ qv_pthread_scope_split_at(
         return QV_ERR_INVLD_ARG;
     }
     try {
-        return scope->thread_split_at(type, kcolors, k, subscopes);
+        std::vector<int> color_fixup;
+        const int rc = split_color_fixup(kcolors, k, color_fixup);
+        if (qvi_unlikely(rc != QV_SUCCESS)) return rc;
+        // Set the colors array to the appropriate data.
+        int *kcolorsp = color_fixup.empty() ? kcolors : color_fixup.data();
+        return scope->thread_split_at(type, kcolorsp, k, subscopes);
     }
     qvi_catch_and_return();
 }

src/qvi-pthread.cc

@@ -268,7 +268,7 @@ qvi_pthread_group::split(
     // One thread creates the child group. The rest wait for the instance and
     // later grab a pointer to their group based on the sub-group index.
     if (sginfo.rank == qvi_subgroup_info::master_rank) {
-        // Recall this is the parent group.
+        // Recall that 'this' is the parent group.
         rc = qvi_new(&ichild, this, sginfo);
         barrier();
     }

src/qvi-scope.cc

@@ -315,11 +315,11 @@ qv_scope::thread_split(
 int
 qv_scope::thread_split_at(
     qv_hw_obj_type_t type,
-    int *kgroup_ids,
+    int *kcolors,
     uint_t k,
     qv_scope_t ***kchildren
 ) {
-    return thread_split(hwpool_nobjects(type), kgroup_ids, k, type, kchildren);
+    return thread_split(hwpool_nobjects(type), kcolors, k, type, kchildren);
 }
 
 /*

src/qvi-scope.h

@@ -122,7 +122,7 @@ struct qv_scope {
     int
     thread_split_at(
         qv_hw_obj_type_t type,
-        int *kgroup_ids,
+        int *kcolors,
         uint_t k,
         qv_scope_t ***kchildren
     );

tests/test-pthread-split.c

@@ -33,7 +33,6 @@ thread_work(
     ctu_scope_report(scope, "thread_scope_in_thread_routine");
     ctu_emit_task_bind(scope);
 #endif
-
     if (rank == 0) {
         printf("[%d] ============ Splitting thread scopes in two\n", tid);
     }
@@ -81,38 +80,21 @@ main(void)
 
     ctu_emit_task_bind(base_scope);
     //
-    // Test qv_pthread_scope_split
+    // Test qv_pthread_scope_split()
     //
     const int npieces = 2;
     const int nthreads = ncores;
-    int colors[nthreads];
 
     printf(
         "[%d] Testing thread_scope_split (nthreads=%d, npieces=%d)\n",
         tid, nthreads, npieces
     );
 
-    for (int i = 0 ; i < nthreads ; i++) {
-        colors[i] = i % npieces;
-    }
-#if 0
-    printf("Manual values: ");
-    for (int i = 0 ; i < nthreads ; i++) {
-        printf("val[%i]:%i | ",i,colors[i]);
-    }
-    printf("\n");
-#endif
-
-#if 0
-    fprintf(stdout,"Filled values: ");
-    for (int i = 0 ; i < nthreads ; i++) {
-        fprintf(stdout,"val[%i]:%i | ",i,colors[i]);
-    }
-    fprintf(stdout,"\n");
-#endif
     qv_scope_t **th_scopes = NULL;
     rc = qv_pthread_scope_split(
-        base_scope, npieces, colors, nthreads, &th_scopes
+        base_scope, npieces,
+        QV_PTHREAD_SCOPE_SPLIT_PACKED,
+        nthreads, &th_scopes
     );
     if (rc != QV_SUCCESS) {
         ers = "qv_pthread_scope_split() failed";
@@ -152,39 +134,18 @@ main(void)
         ers = "qv_pthread_scope_free() failed";
         ctu_panic("%s (rc=%s)", ers, qv_strerr(rc));
     }
-
-#if 0
+    //
     //Test qv_pthread_scope_split_at
-    nthreads = 2 * ncores;
-
-    printf("[%d] Testing thread_scope_split_at (nthreads=%i)\n", tid, nthreads);
-
-    int colors2[nthreads];
-    for (int i = 0 ; i < nthreads ; i++) {
-        colors2[i] = i % ncores;
-    }
-
-    fprintf(stdout,"Array values :");
-    for (int i = 0 ; i < nthreads ; i++) {
-        fprintf(stdout,"val[%i]: %i |",i,colors2[i]);
-    }
-    fprintf(stdout,"\n");
-
-
-    rc = qv_pthread_colors_fill(colors2, nthreads, QV_POLICY_PACKED, stride, ncores);
-    if (rc != QV_SUCCESS) {
-        ers = "qv_pthread_colors_fill() failed";
-        ctu_panic("%s (rc=%s)", ers, qv_strerr(rc));
-    }
-
-    fprintf(stdout,"Array values :");
-    for (int i = 0 ; i < nthreads ; i++) {
-        fprintf(stdout,"val[%i]: %i |",i,colors2[i]);
-    }
-    fprintf(stdout,"\n");
+    //
+    printf(
+        "[%d] Testing thread_scope_split_at (nthreads=%d, npieces=%d)\n",
+        tid, nthreads, npieces
+    );
 
     rc = qv_pthread_scope_split_at(
-        mpi_scope, QV_HW_OBJ_CORE, colors2, nthreads, &th_scopes
+        base_scope, QV_HW_OBJ_CORE,
+        QV_PTHREAD_SCOPE_SPLIT_PACKED,
+        nthreads, &th_scopes
     );
     if (rc != QV_SUCCESS) {
         ers = "qv_pthread_scope_split_at() failed";
@@ -215,14 +176,13 @@ main(void)
         }
         //fprintf(stdout,"Thread finished with '%s'\n", (char *)ret);
     }
-
     // Clean up.
     rc = qv_pthread_scopes_free(nthreads, th_scopes);
     if (rc != QV_SUCCESS) {
         ers = "qv_pthread_scope_free() failed";
         ctu_panic("%s (rc=%s)", ers, qv_strerr(rc));
     }
-#endif
+
     rc = qv_scope_free(base_scope);
     if (rc != QV_SUCCESS) {
         ers = "qv_scope_free() failed";