[PML/UBCL] add_procs fallbacks on higher transport when required

Signed-off-by: Brelle Emmanuel <[email protected]>

Author: DUPRAT, JULIEN

ompi/mca/pml/ubcl/pml_ubcl_endpoint.c

@@ -254,7 +254,7 @@ static int mca_pml_ubcl_create_recv_endpoint(uint64_t sender_rank, const int typ
 
     err = ubcl_export_local_endpoint_handle(type, endpoint_h, &remote_rank_u64);
     if (UBCL_SUCCESS != err) {
-        return OMPI_ERROR;
+        return ubcl_error_to_ompi(err);
     }
 
     return OMPI_SUCCESS;
@@ -269,11 +269,11 @@ static int mca_pml_ubcl_create_self_endpoints(uint64_t remote_rank)
 
     err = ubcl_export_local_endpoint_handle(type, endpoint_h, &my_rank);
     if (UBCL_SUCCESS != err) {
-        return OMPI_ERROR;
+        return ubcl_error_to_ompi(err);
     }
     err = ubcl_create_remote_endpoint(my_rank, my_rank, type, endpoint_h);
     if (UBCL_SUCCESS != err) {
-        return OMPI_ERROR;
+        return ubcl_error_to_ompi(err);
     }
 
     return OMPI_SUCCESS;
@@ -295,6 +295,25 @@ static int get_endpoint_type(ompi_proc_t *proc)
     }
 }
 
+static enum ubcl_endpoint_type_t mca_pml_ubcl_get_higher_transport(
+                                                 enum ubcl_endpoint_type_t type)
+{
+    switch ((int) type) {
+        case UBCL_ENDPOINT_TYPE_SELF:
+        case UBCL_ENDPOINT_TYPE_SHMEM:
+            type++;
+            break;
+        /* There are no valid higher transport */
+        case UBCL_ENDPOINT_TYPE_BXI:
+        default:
+            type = UBCL_ENDPOINT_TYPE_NONE;
+            /* Not a valid transport */
+            break;
+    }
+
+    return type;
+}
+
 void mca_pml_ubcl_endpoint_retain(ompi_proc_t *proc)
 {
     mca_common_ubcl_endpoint_t *endpoint = NULL;
@@ -311,6 +330,7 @@ void mca_pml_ubcl_endpoint_retain(ompi_proc_t *proc)
 static int mca_pml_ubcl_create_endpoints(ompi_proc_t *proc)
 {
     int err = OMPI_SUCCESS;
+    enum ubcl_endpoint_type_t type;
     mca_common_ubcl_endpoint_t *new_endpoint;
 
     new_endpoint = malloc(sizeof(mca_common_ubcl_endpoint_t));
@@ -321,32 +341,55 @@ static int mca_pml_ubcl_create_endpoints(ompi_proc_t *proc)
 
     new_endpoint->refcount = 0; //we increment it to 1 in endpoint_retain
     new_endpoint->rank = mca_pml_forge_rank(proc);
-    new_endpoint->type = get_endpoint_type(proc);
+    type               = get_endpoint_type(proc);
 
-    if (UBCL_ENDPOINT_TYPE_SELF == new_endpoint->type) {
+    if (UBCL_ENDPOINT_TYPE_SELF == type) {
         err = mca_pml_ubcl_create_self_endpoints((uint64_t) new_endpoint->rank);
-        goto end;
-    }
 
-    err = mca_pml_ubcl_create_recv_endpoint(new_endpoint->rank, new_endpoint->type);
-    if (OMPI_SUCCESS != err) {
-        mca_pml_ubcl_warn(err, "Failed to create recv endpoint for rank %zu\n",
-                          new_endpoint->rank);
-        return err;
+        /* If the transport is unvailable (either explicitely disabled,
+         * or just unavailable) we do not return any error
+         * If UBCL encountered another error we return it */
+        if (OMPI_SUCCESS == err) {
+            goto end;
+        } else if (OMPI_ERR_NOT_AVAILABLE != err) {
+            return err;
+        }
     }
 
-    err = mca_pml_ubcl_create_send_endpoint(proc, new_endpoint->rank, new_endpoint->type);
+    /* If a transport is unavailable only a higher transport can take its place,
+     * ie. if SHM is unavailable, SELF cannot replace it but BXI can */
+    do {
+        err = mca_pml_ubcl_create_recv_endpoint(new_endpoint->rank, type);
+
+        if (OMPI_ERR_NOT_AVAILABLE == err) {
+            type = mca_pml_ubcl_get_higher_transport(type);
+            if (UBCL_ENDPOINT_TYPE_NONE == type) {
+                mca_pml_ubcl_warn(err, "Failed to create recv endpoint for rank %zu\n",
+                                  new_endpoint->rank);
+                return err;
+            }
+        } else if (OMPI_SUCCESS != err) {
+            mca_pml_ubcl_warn(err, "Failed to create recv endpoint for rank %zu\n",
+                              new_endpoint->rank);
+            return err;
+        }
+    } while (OMPI_SUCCESS != err);
+
+    /* No need to loop again, if the transport became unavailable between
+     * the last operation and this one we can consider this a error */
+    err = mca_pml_ubcl_create_send_endpoint(proc, new_endpoint->rank, type);
     if (OMPI_SUCCESS != err) {
         mca_pml_ubcl_warn(err, "Failed to create send endpoint for rank %zu\n",
                          new_endpoint->rank);
         return err;
     }
 
 end:
+    new_endpoint->type = type;
     (proc)->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PML] = new_endpoint;
     mca_pml_ubcl_endpoint_retain(proc);
 
-    return err;
+    return UBCL_SUCCESS;
 }
 
 int mca_pml_ubcl_add_procs(ompi_proc_t **procs, size_t nprocs)