coll/han: parameterize the packing buf size and count

Allow the user to change the packing buffer size and how many maximum
buffers will be allocated.  Currently only han's alltoallv uses the
buffers.

Signed-off-by: Luke Robison <[email protected]>

Author: Luke Robison

ompi/mca/coll/han/coll_han.h

@@ -294,6 +294,8 @@ typedef struct mca_coll_han_component_t {
     int max_dynamic_errors;
 
     opal_free_list_t pack_buffers;
+    int64_t han_packbuf_max_count;
+    int64_t han_packbuf_bytes;
 } mca_coll_han_component_t;
 
 /*
@@ -576,7 +578,4 @@ static inline struct mca_smsc_endpoint_t *mca_coll_han_get_smsc_endpoint (struct
     return (struct mca_smsc_endpoint_t *) proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_SMSC];
 }
 
-#define COLL_HAN_PACKBUF_PAYLOAD_BYTES (128*1024)
-
-
 #endif                          /* MCA_COLL_HAN_EXPORT_H */

ompi/mca/coll/han/coll_han_alltoallv.c

@@ -313,20 +313,35 @@ static int alltoallv_sendrecv_w(
     ompi_request_t *requests[MAX_BUF_COUNT];
     opal_free_list_item_t *buf_items[MAX_BUF_COUNT];
 
-    size_t buf_len = COLL_HAN_PACKBUF_PAYLOAD_BYTES;
+    size_t buf_len = mca_coll_han_component.han_packbuf_bytes;
     int nbufs = MAX_BUF_COUNT;
     for (int jbuf=0; jbuf<nbufs; jbuf++) {
         buf_items[jbuf] = opal_free_list_get(&mca_coll_han_component.pack_buffers);
         if (buf_items[jbuf] == NULL) {
-            nbufs = jbuf - 1;
-            opal_output_verbose(20, mca_coll_han_component.han_output,
-                "Uh-oh, not enough buffers: %d\n",nbufs);
+            nbufs = jbuf;
+            opal_output_verbose(30, mca_coll_han_component.han_output,
+                "alltoallv_sendrecv_w: Number of buffers reduced to %d instead of %d.  "
+                "Check mca parameter coll_han_packbuf_max_count (currently %ld).\n",
+                nbufs, MAX_BUF_COUNT, mca_coll_han_component.han_packbuf_max_count);
             break;
         }
     }
+
+    /* although we could feasibly disqualify the higher-level alltoallv
+    algorithm for these conditions, they represent invalid user parameters, so
+    let's fail and tell the user why.  */
     if (nbufs < 2) {
         opal_output_verbose(1, mca_coll_han_component.han_output,
-                    "ERROR: Need at least 2 buffers from mca_coll_han_component.pack_buffers!");
+                    "ERROR: Need at least 2 buffers from HAN pack buffers!  "
+                    "Check mca parameter coll_han_packbuf_max_count (currently %ld)\n",
+                    mca_coll_han_component.han_packbuf_max_count);
+        return MPI_ERR_NO_MEM;
+    }
+    if (buf_len < 16) {
+        opal_output_verbose(1, mca_coll_han_component.han_output,
+                    "ERROR: Need a buffer that can hold at least 16 bytes!  "
+                    "Check mca parameter coll_han_packbuf_bytes (currently %ld)\n",
+                    mca_coll_han_component.han_packbuf_bytes);
         return MPI_ERR_NO_MEM;
     }
 

ompi/mca/coll/han/coll_han_component.c

@@ -126,9 +126,9 @@ static int han_open(void)
         /* opal_class_t *frag_class */
         OBJ_CLASS(opal_free_list_item_t),
         /* payload_buffer_size, payload_buffer_alignment */
-        COLL_HAN_PACKBUF_PAYLOAD_BYTES, 8,
+        mca_coll_han_component.han_packbuf_bytes, 8,
         /* num_elements_to_alloc, max_elements_to_alloc, num_elements_per_alloc */
-        0, 32, 8,
+        0, mca_coll_han_component.han_packbuf_max_count, 8,
         /* *mpool, rcache_reg_flags, *rcache, */
         NULL, 0, NULL,
         /* fn_t item_init, void *ctx */
@@ -456,6 +456,20 @@ static int han_register(void)
                                            MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
                                            OPAL_INFO_LVL_3,
                                            MCA_BASE_VAR_SCOPE_READONLY, &cs->han_reproducible);
+
+    cs->han_packbuf_bytes = 128*1024;
+    (void) mca_base_component_var_register(c, "packbuf_bytes",
+                                           "The number of bytes in each HAN packbuf.",
+                                           MCA_BASE_VAR_TYPE_INT64_T, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+                                           &cs->han_packbuf_bytes);
+    cs->han_packbuf_max_count = 32;
+    (void) mca_base_component_var_register(c, "packbuf_max_count",
+                                           "The maximum number of packbufs that are allowed to be allocated.",
+                                           MCA_BASE_VAR_TYPE_INT64_T, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
+                                           &cs->han_packbuf_max_count);
+
     /*
      * Han algorithms MCA parameters for each collective.
      * Shows algorithms thanks to enumerator