Adding caliper annotations

src/EnergySpectrum.cc

@@ -6,6 +6,7 @@
 #include "MC_Processor_Info.hh"
 #include "Parameters.hh"
 #include <string>
+#include <stdint.h>
 
 using std::string;
 

src/EnergySpectrum.hh

@@ -2,6 +2,7 @@
 #define ENERGYSPECTRUM_HH
 #include <string>
 #include <vector>
+#include <stdint.h>
 
 class MonteCarlo;
 

src/MC_Fast_Timer.cc

@@ -4,6 +4,7 @@
 #include "MC_Processor_Info.hh"
 #include "Globals.hh"
 #include "portability.hh"
+#include <adiak.hpp>
 
 const char *mc_fast_timer_names[MC_Fast_Timer::Num_Timers] =
 {
@@ -101,6 +102,7 @@ void MC_Fast_Timer_Container::Cumulative_Report(int mpi_rank, int num_ranks, MPI
                         "Figure Of Merit",
                         (numSegments / (max_clock[cycleTracking_Index]*1e-6)),
                         "[Num Segments / Cycle Tracking Time]" );
+        adiak::value("FOM", numSegments / (max_clock[cycleTracking_Index]*1e-6));
     }
 }
 

src/MC_Fast_Timer.hh

@@ -6,6 +6,10 @@
 #include <chrono>
 #endif
 
+#ifdef USE_CALIPER
+#include <caliper/cali.h>
+#endif
+
 #include "portability.hh"   // needed for uint64_t in this file
 #include "utilsMpi.hh"      // needed for MPI_Comm type in this file
 
@@ -44,12 +48,20 @@ class MC_Fast_Timer
 class MC_Fast_Timer_Container
 {
 public:
-    MC_Fast_Timer_Container() {} ; // constructor
+    MC_Fast_Timer_Container()
+#ifdef USE_CALIPER
+        : cali_annotation("mc.timer", CALI_ATTR_SCOPE_PROCESS | CALI_ATTR_NESTED)
+#endif
+        {} ; // constructor
     void Cumulative_Report(int mpi_rank, int num_ranks, MPI_Comm comm_world, uint64_t numSegments);
     void Last_Cycle_Report(int report_time, int mpi_rank, int num_ranks, MPI_Comm comm_world);
     void Clear_Last_Cycle_Timers();
     MC_Fast_Timer  timers[MC_Fast_Timer::Num_Timers];  // timers for various routines
-
+
+#ifdef USE_CALIPER
+    cali::Annotation cali_annotation;
+#endif
+
 private:
     void Print_Cumulative_Heading(int mpi_rank);
     void Print_Last_Cycle_Heading(int mpi_rank);
@@ -87,27 +99,49 @@ extern const char *mc_fast_timer_names[MC_Fast_Timer::Num_Timers];
       #define MC_FASTTIMER_GET_LASTCYCLE(timerIndex) (float)mcco->fast_timer->timers[timerIndex].lastCycleClock / 1000000.
 
    #else // else CHRONO_MISSING is not defined, so high resolution clock is available
-
-      #define MC_FASTTIMER_START(timerIndex) \
-          if (omp_get_thread_num() == 0) { \
-              mcco->fast_timer->timers[timerIndex].startClock = std::chrono::high_resolution_clock::now(); \
-          }
-
-      #define MC_FASTTIMER_STOP(timerIndex) \
-          if ( omp_get_thread_num() == 0 ) { \
-              mcco->fast_timer->timers[timerIndex].stopClock = std::chrono::high_resolution_clock::now(); \
-              mcco->fast_timer->timers[timerIndex].lastCycleClock += \
-                std::chrono::duration_cast<std::chrono::microseconds> \
-		(mcco->fast_timer->timers[timerIndex].stopClock - mcco->fast_timer->timers[timerIndex].startClock).count(); \
-              mcco->fast_timer->timers[timerIndex].cumulativeClock += \
-	        std::chrono::duration_cast<std::chrono::microseconds> \
-	        (mcco->fast_timer->timers[timerIndex].stopClock - mcco->fast_timer->timers[timerIndex].startClock).count(); \
-              mcco->fast_timer->timers[timerIndex].numCalls++;		\
-          }
+      #ifdef USE_CALIPER
+         #define MC_FASTTIMER_START(timerIndex) \
+             if (omp_get_thread_num() == 0) { \
+                 mcco->fast_timer->timers[timerIndex].startClock = std::chrono::high_resolution_clock::now(); \
+             } \
+             mcco->fast_timer->cali_annotation.begin(mc_fast_timer_names[timerIndex]);
+
+         #define MC_FASTTIMER_STOP(timerIndex) \
+             if ( omp_get_thread_num() == 0 ) { \
+                 mcco->fast_timer->timers[timerIndex].stopClock = std::chrono::high_resolution_clock::now(); \
+                 mcco->fast_timer->timers[timerIndex].lastCycleClock += \
+                   std::chrono::duration_cast<std::chrono::microseconds> \
+                   (mcco->fast_timer->timers[timerIndex].stopClock - mcco->fast_timer->timers[timerIndex].startClock).count(); \
+                 mcco->fast_timer->timers[timerIndex].cumulativeClock += \
+                   std::chrono::duration_cast<std::chrono::microseconds> \
+                   (mcco->fast_timer->timers[timerIndex].stopClock - mcco->fast_timer->timers[timerIndex].startClock).count(); \
+                 mcco->fast_timer->timers[timerIndex].numCalls++;		\
+             } \
+             mcco->fast_timer->cali_annotation.end();
+
+      #else // not defined USE_CALIPER
+
+         #define MC_FASTTIMER_START(timerIndex) \
+             if (omp_get_thread_num() == 0) { \
+                 mcco->fast_timer->timers[timerIndex].startClock = std::chrono::high_resolution_clock::now(); \
+             }
+
+         #define MC_FASTTIMER_STOP(timerIndex) \
+             if ( omp_get_thread_num() == 0 ) { \
+                 mcco->fast_timer->timers[timerIndex].stopClock = std::chrono::high_resolution_clock::now(); \
+                 mcco->fast_timer->timers[timerIndex].lastCycleClock += \
+                   std::chrono::duration_cast<std::chrono::microseconds> \
+                   (mcco->fast_timer->timers[timerIndex].stopClock - mcco->fast_timer->timers[timerIndex].startClock).count(); \
+                 mcco->fast_timer->timers[timerIndex].cumulativeClock += \
+                   std::chrono::duration_cast<std::chrono::microseconds> \
+                   (mcco->fast_timer->timers[timerIndex].stopClock - mcco->fast_timer->timers[timerIndex].startClock).count(); \
+                 mcco->fast_timer->timers[timerIndex].numCalls++;		\
+             }
+
+      #endif // end ifdef USE_CALIPER else branch
 
       #define MC_FASTTIMER_GET_LASTCYCLE(timerIndex) (float)mcco->fast_timer->timers[timerIndex].lastCycleClock / 1000000.
 
-
    #endif // end ifdef CHRONO_MISSING else section
 #endif // end if DISABLE_TIMERS
 

src/Makefile

@@ -84,6 +84,13 @@
 #                   with some Clang compilers, some older Gnu compilers on BG/Q
 #                   and older Intel compilers.
 #         
+# -DUSE_CALIPER     Define this to enable Caliper instrumentation. Caliper
+#                   is a performance profiling / analysis library for
+#                   tracing, sampling, HW counter measurements, and much more.
+#                       When enabled, Quicksilver will link in the Caliper
+#                   library and export its timed regions to Caliper. See
+#                   https://github.com/LLNL/Caliper for more information.
+#         
 # -DUSE_NVTX        Define this for some extra NVProf profiling information.
 #                   It will create regions that can be visualized in NVVP.
 #
@@ -108,11 +115,11 @@ SHELL = /bin/bash
 #LDFLAGS = -L$(ROCM_ROOT)/lib -lamdhip64
 
 #AMD with HIP
-ROCM_ROOT = /opt/rocm-5.6.0
-CXX = /usr/tce/packages/cray-mpich/cray-mpich-8.1.26-rocmcc-5.6.0-cce-16.0.0a-magic/bin/mpicxx
-CXXFLAGS = -g 
-CPPFLAGS = -DHAVE_MPI -DHAVE_HIP -x hip --offload-arch=gfx90a -fgpu-rdc -Wno-unused-result
-LDFLAGS = -fgpu-rdc --hip-link --offload-arch=gfx90a
+#ROCM_ROOT = /opt/rocm-5.6.0
+#CXX = /usr/tce/packages/cray-mpich/cray-mpich-8.1.26-rocmcc-5.6.0-cce-16.0.0a-magic/bin/mpicxx
+#CXXFLAGS = -g 
+#CPPFLAGS = -DHAVE_MPI -DHAVE_HIP -x hip --offload-arch=gfx90a -fgpu-rdc -Wno-unused-result
+#LDFLAGS = -fgpu-rdc --hip-link --offload-arch=gfx90a
 
 
 
@@ -139,7 +146,6 @@ LDFLAGS = -fgpu-rdc --hip-link --offload-arch=gfx90a
 #CPPFLAGS = $(OPENMP_FLAGS)
 #LDFLAGS = $(OPENMP_LDFLAGS) 
 
-
 ###############################################################################
 ### GCC -- with MPI and OpenMP 
 ###############################################################################
@@ -154,6 +160,57 @@ LDFLAGS = -fgpu-rdc --hip-link --offload-arch=gfx90a
 #LDFLAGS = $(OPENMP_LDFLAGS) 
 
 
+###############################################################################
+### GCC -- with MPI and OpenMP and Caliper support
+###############################################################################
+CALIPER_DIR = $(spack location --install-dir caliper)
+ADIAK_DIR = $(spack location --install-dir adiak)
+#CALIPER_DIR = ${HOME}/local/caliper/toss3-release
+#CALIPER_DIR = /
+CALIPER_FLAGS = -I${CALIPER_DIR}/include -DUSE_CALIPER
+ADIAK_INCLUDE = -I${ADIAK_DIR}/include 
+#CALIPER_LDFLAGS = -Wl,-rpath ${CALIPER_DIR}/lib64 -L${CALIPER_DIR}/lib64 -lcaliper #-lcaliper-mpi
+ADIAK_LDFLAGS =  -L${ADIAK_DIR}/lib -ladiak
+CALIPER_LDFLAGS =  -L${CALIPER_DIR}/lib64 -lcaliper 
+OPENMP_FLAGS = -DHAVE_OPENMP -fopenmp
+OPENMP_LDFLAGS = -fopenmp
+MPI_FLAGS = -DHAVE_MPI
+OPTFLAGS = -g -O2
+
+CXX=mpicxx
+CXXFLAGS = -std=c++11 $(OPTFLAGS) #-Wpedantic
+CPPFLAGS = $(MPI_FLAGS) $(OPENMP_FLAGS) $(CALIPER_FLAGS) $(ADIAK_FLAGS)
+LDFLAGS = $(OPENMP_LDFLAGS) $(CALIPER_LDFLAGS) $(ADIAK_LDFLAGS)
+
+###############################################################################
+# Cuda on LLNL Lassen w/ Caliper
+###############################################################################
+## Choose one Cuda path
+# CUDA_PATH = /usr/local/cuda
+
+#CALIPER_DIR=${HOME}/local/caliper/lassen-cuda10
+#CUDA_PATH = /usr/tce/packages/cuda/cuda-10.1.168
+
+#HOST_COMPILER = mpicxx
+
+#CALIPER_FLAGS = -I${CALIPER_DIR}/include -DUSE_CALIPER
+#CALIPER_LDFLAGS = -L${CALIPER_DIR}/lib64 -lcaliper
+
+#OPTFLAGS = -O2 -g
+## Version below for debugging
+##OPTFLAGS = -DUSE_NVTX -g -G -lineinfo -O0
+
+#CUDA_FLAGS = -I${CUDA_PATH}/include/
+#CUDA_LDFLAGS = -L${CUDA_PATH}/lib64/ -lcuda -lcudart
+#
+#CXX=$(CUDA_PATH)/bin/nvcc
+#CXXFLAGS = -DHAVE_CUDA -std=c++11 $(OPTFLAGS) -Xptxas -v 
+#CXXFLAGS += -gencode=arch=compute_60,code=\"sm_60,compute_60\" 
+#CXXFLAGS += --compiler-bindir=$(HOST_COMPILER)
+#CPPFLAGS = -x cu -dc -DHAVE_MPI -DHAVE_ASYNC_MPI $(CALIPER_FLAGS)
+#LDFLAGS  = $(CUDA_LDFLAGS) $(CALIPER_LDFLAGS)
+##LDFLAGS += ${CUDA_PATH}/lib64/libnvToolsExt.so
+
 ###############################################################################
 # LLNL LC BG/Q Comilers                                                       #
 ###############################################################################

src/NuclearData.hh

@@ -9,6 +9,7 @@
 #include <algorithm>
 #include "qs_assert.hh"
 #include "DeclareMacro.hh"
+#include <stdint.h>
 
 class Polynomial
 {

src/Parameters.cc

@@ -226,6 +226,8 @@ namespace
       esName[0] = '\0';
       char xsec[1024];
       xsec[0] = '\0';
+      char calicfg[1024];
+      calicfg[0] = '\0';
 
       addArg("help",             'h', 0, 'i', &(help),           0,      "print this message");
       addArg("dt",               'D', 1, 'd', &(sp.dt),          0,      "time step (seconds)");
@@ -253,12 +255,14 @@ namespace
       addArg("bTally",           'B', 1, 'i', &(sp.balanceTallyReplications), 0, "number of balance tally replications");
       addArg("fTally",           'F', 1, 'i', &(sp.fluxTallyReplications),    0, "number of scalar flux tally replications");
       addArg("cTally",           'C', 1, 'i', &(sp.cellTallyReplications),    0, "number of scalar cell tally replications");
+      addArg("caliper-config",   'P', 1, 's', &(calicfg), sizeof(calicfg), "Caliper configuration");
 
       processArgs(argc, argv);
 
       sp.inputFile = name;
       sp.energySpectrum = esName;
       sp.crossSectionsOut = xsec;
+      sp.caliperConfig = calicfg;
 
       if (help)
       {

src/Parameters.hh

@@ -8,6 +8,7 @@
 #include <vector>
 #include <map>
 #include <iostream>
+#include <stdint.h>
 
 struct GeometryParameters
 {
@@ -164,6 +165,7 @@ struct SimulationParameters
    int fluxTallyReplications;    //!< Number of replications for the scalar flux tally
    int cellTallyReplications;    //!< Number of replications for the scalar cell tally
    int coralBenchmark;           //!< enable correctness check for Coral2 benchmark
+   std::string caliperConfig;    //!< Caliper configuration string
 };
 
 struct Parameters

src/READ.ME.HOW.TO.RUN

@@ -115,6 +115,16 @@ There is also, at the end of the run, a coarse breakdown of time spent overall
 in the above mentioned three code phases, as well as a few other sub timings
 from cycle tracking.
 
+-------------------------------------------------------------------------------
+A note on Caliper:
+
+Caliper is a powerful performance profiling/tracing library. When configured
+with Caliper support, Quicksilver adds Caliper annotations for its timed
+regions (cycleTracking, cycleTrackingKernel, etc.) as the "mc.timer"
+attribute. Performance measurements can be configured through environment
+variables or the caliper.config configuration file. For Caliper documentation,
+see https://github.com/LLNL/Caliper.
+
 -------------------------------------------------------------------------------
 A note on asserts: