Merge pull request #346 from ludwig-cf/fix-issue-311

Replace repeated occurrences of `HAVE_OPENMP_`

src/field.c

@@ -14,7 +14,7 @@
  *  Edinburgh Soft Matter and Statistical Physics Group and
  *  Edinburgh Parallel Computing Centre
  *
- *  (c) 2012-2024 The University of Edinburgh
+ *  (c) 2012-2025 The University of Edinburgh
  *
  *  Contributing authors:
  *  Kevin Stratford ([email protected])
@@ -43,17 +43,6 @@ static int field_leesedwards_parallel(field_t * obj);
 
 __host__ int field_init(field_t * obj, int nhcomm, lees_edw_t * le);
 
-#ifdef HAVE_OPENMPI_
-/* This provides MPIX_CUDA_AWARE_SUPPORT .. */
-#include "mpi-ext.h"
-#endif
-
-#if defined (MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT
-static const int have_gpu_aware_mpi_ = 1;
-#else
-static const int have_gpu_aware_mpi_ = 0;
-#endif
-
 /*****************************************************************************
  *
  *  field_create
@@ -1443,7 +1432,7 @@ int field_halo_post(const field_t * field, field_halo_t * h) {
     int k = 1 + h->cv[h->nvel - ireq][Z];
     int mcount = field->nf*field_halo_size(h->rlim[ireq]);
     double * buf = h->recv[ireq];
-    if (have_gpu_aware_mpi_) buf = h->recv_d[ireq];
+    if (have_gpu_aware_mpi_()) buf = h->recv_d[ireq];
 
     h->request[ireq] = MPI_REQUEST_NULL;
 
@@ -1485,7 +1474,7 @@ int field_halo_post(const field_t * field, field_halo_t * h) {
     int k = 1 + h->cv[ireq][Z];
     int mcount = field->nf*field_halo_size(h->slim[ireq]);
     double * buf = h->send[ireq];
-    if (have_gpu_aware_mpi_) buf = h->send_d[ireq];
+    if (have_gpu_aware_mpi_()) buf = h->send_d[ireq];
 
     /* Skip messages to self ... */
     if (h->nbrrank[i][j][k] == h->nbrrank[1][1][1]) continue;
@@ -1747,7 +1736,7 @@ int field_graph_halo_send_create(const field_t * field, field_halo_t * h) {
     tdpAssert( tdpGraphAddKernelNode(&kernelNode, h->gsend.graph, NULL, 0,
 				     &kernelNodeParams) );
 
-    if (have_gpu_aware_mpi_) {
+    if (have_gpu_aware_mpi_()) {
       /* Don't need explicit device -> host copy */
     }
     else {
@@ -1803,7 +1792,7 @@ int field_graph_halo_recv_create(const field_t * field, field_halo_t * h) {
     int rcount = field->nf*field_halo_size(h->rlim[ireq]);
     tdpGraphNode_t memcpyNode = {0};
 
-    if (have_gpu_aware_mpi_) {
+    if (have_gpu_aware_mpi_()) {
       /* Don't need explicit copies */
     }
     else {
@@ -1851,7 +1840,7 @@ int field_graph_halo_recv_create(const field_t * field, field_halo_t * h) {
     kernelNodeParams.kernelParams   = (void **) kernelArgs;
     kernelNodeParams.extra          = NULL;
 
-    if (have_gpu_aware_mpi_) {
+    if (have_gpu_aware_mpi_()) {
       tdpAssert( tdpGraphAddKernelNode(&node, h->grecv.graph, NULL,
 				       0, &kernelNodeParams) );
     }

src/model_le.c

@@ -11,7 +11,7 @@
  *  Edinburgh Soft Matter and Statistical Physics Group and
  *  Edinburgh Parallel Computing Centre
  *
- *  (c) 2010-2022 The University of Edinburgh
+ *  (c) 2010-2025 The University of Edinburgh
  *
  *  Contributing authors:
  *  Kevin Stratford ([email protected])
@@ -36,17 +36,6 @@ static int le_reproject(lb_t * lb, lees_edw_t * le);
 static int le_displace_and_interpolate(lb_t * lb, lees_edw_t * le);
 static int le_displace_and_interpolate_parallel(lb_t * lb, lees_edw_t * le);
 
-#ifdef HAVE_OPENMPI_
-/* This provides MPIX_CUDA_AWARE_SUPPORT .. */
-#include "mpi-ext.h"
-#endif
-
-#if defined (MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT
-static const int have_gpu_aware_mpi_ = 1;
-#else
-static const int have_gpu_aware_mpi_ = 0;
-#endif
-
 /*****************************************************************************
  *
  *  lb_le_apply_boundary_conditions
@@ -825,7 +814,7 @@ int lb_data_apply_le_boundary_conditions(lb_t * lb, lees_edw_t * le) {
 
 
     /* Second, displacement. */
-    if (have_gpu_aware_mpi_ || mpi_cartsz[Y] > 1) {
+    if (have_gpu_aware_mpi_() || mpi_cartsz[Y] > 1) {
       lb_data_displace_communicate(lekh, lb, le, t);
     }
     else {
@@ -1139,7 +1128,7 @@ static int lb_data_displace_communicate(le_kernel_helper_t lekh,
   /* If there is GPU-aware MPI just communicate the GPU buffers; if
    * not, copy in relevant direction at the start and finish */
 
-  if (have_gpu_aware_mpi_) {
+  if (have_gpu_aware_mpi_()) {
     tdpAssert( tdpMemcpy(&sbuff, &lb->target->sbuff, sizeof(double *),
 			 tdpMemcpyDeviceToHost) );
     tdpAssert( tdpMemcpy(&rbuff, &lb->target->rbuff, sizeof(double *),
@@ -1253,7 +1242,7 @@ static int lb_data_displace_communicate(le_kernel_helper_t lekh,
   /* Complete */
   MPI_Waitall(8, req, MPI_STATUSES_IGNORE);
 
-  if (have_gpu_aware_mpi_) {
+  if (have_gpu_aware_mpi_()) {
     /* No further action */
   }
   else {

src/pe.c

@@ -15,7 +15,7 @@
  *  Edinburgh Soft Matter and Statistical Physics Group and
  *  Edinburgh Parallel Computing Centre
  *
- *  (c) 2010-2024 The University of Edinburgh
+ *  (c) 2010-2025 The University of Edinburgh
  *
  *  Contributing authors:
  *  Kevin Stratford ([email protected])
@@ -438,3 +438,28 @@ __host__ int pe_time(char * str, int bufsiz) {
 
   return ierr;
 }
+
+/*****************************************************************************
+ *
+ *  have_gpu_aware_mpi_
+ *
+ *  This is awkward; it might belong elsewhere on its own.
+ *
+ *****************************************************************************/
+
+#ifdef HAVE_OPENMPI_
+/* This provides MPIX_CUDA_AWARE_SUPPORT .. */
+#include "mpi-ext.h"
+#endif
+
+int have_gpu_aware_mpi_(void) {
+
+  int have_gpu_aware_mpi = 0;
+
+  /* OpenMPI */
+#if defined (MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT
+  have_gpu_aware_mpi = 1;
+#endif
+
+  return have_gpu_aware_mpi;
+}

src/pe.h

@@ -5,7 +5,7 @@
  *  Edinburgh Soft Matter and Statistical Physics Group and
  *  Edinburgh Parallel Computing Centre
  *
- *  (c) 2010-2023 The University of Edinburgh
+ *  (c) 2010-2025 The University of Edinburgh
  *
  *  Contribtuing authors:
  *  Kevin Stratford ([email protected])
@@ -40,5 +40,6 @@ __host__ int pe_warn(pe_t * pe, const char * fmt, ...);
 __host__ int pe_exit(pe_t * pe, const char * fmt, ...);
 
 __host__ int pe_time(char * strctime, int bufsiz);
+__host__ int have_gpu_aware_mpi_(void);
 
 #endif