Improved Particle Synchronization and Distributed Cancellation

CMakeLists.txt

@@ -143,7 +143,7 @@ target_compile_definitions(abeille PUBLIC ABEILLE_GIT_HASH=\"${ABEILLE_GIT_HASH_
 target_compile_definitions(abeille PUBLIC ABEILLE_COMPILER_NAME=\"${CMAKE_CXX_COMPILER_ID}\")
 target_compile_definitions(abeille PUBLIC ABEILLE_COMPILER_VERSION=\"${CMAKE_CXX_COMPILER_VERSION}\")
 
-if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") # Comile options for Windows
+if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") # Compile options for Windows
   target_compile_options(abeille PRIVATE /W3)
 elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") # Compile options for GCC
   target_compile_options(abeille PRIVATE -W -Wall -Wextra -Wconversion -Wpedantic)
@@ -158,7 +158,7 @@ if(ABEILLE_USE_OMP)
   find_package(OpenMP REQUIRED)
   if(OpenMP_CXX_FOUND)
     target_link_libraries(abeille PUBLIC OpenMP::OpenMP_CXX)
-    if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") # Comile options for Windows
+    if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") # Compile options for Windows
       target_compile_options(abeille PRIVATE /openmp:llvm) 
     endif()
     target_compile_definitions(abeille PUBLIC ABEILLE_USE_OMP)

include/simulation/approximate_mesh_cancelator.hpp

@@ -32,21 +32,27 @@
 class ApproximateMeshCancelator : public Cancelator {
  public:
   ApproximateMeshCancelator(Position low, Position hi, uint32_t Nx, uint32_t Ny,
-                            uint32_t Nz);
+                            uint32_t Nz, bool loop = false);
   ApproximateMeshCancelator(Position low, Position hi, uint32_t Nx, uint32_t Ny,
-                            uint32_t Nz, std::vector<double> energy_bounds);
+                            uint32_t Nz, std::vector<double> energy_bounds,
+                            bool loop = false);
 
   bool add_particle(BankedParticle& p) override final;
-  void perform_cancellation(pcg32& rng) override final;
+  void perform_cancellation() override final;
   std::vector<BankedParticle> get_new_particles(pcg32& rng) override final;
   void clear() override final;
 
  private:
-  Position r_low, r_hi;
+  std::unordered_map<int, std::vector<BankedParticle*>> bins;
   std::vector<double> energy_edges;
   std::array<uint32_t, 4> shape;
+  Position r_low, r_hi;
   double dx, dy, dz;
-  std::unordered_map<int, std::vector<BankedParticle*>> bins;
+  bool loop;
+
+  std::vector<int> sync_keys();
+  void perform_cancellation_loop();
+  void perform_cancellation_vector();
 };
 
 std::shared_ptr<ApproximateMeshCancelator> make_approximate_mesh_cancelator(

include/simulation/branchless_power_iterator.hpp

@@ -83,7 +83,7 @@ class BranchlessPowerIterator : public Simulation {
   void zero_entropy();
 
   // Pair-distance sqrd calculation
-  double compute_pair_dist_sqrd(const std::vector<BankedParticle>& next_gen);
+  void compute_pair_dist_sqrd(const std::vector<BankedParticle>& next_gen);
 
   void print_header();
 

include/simulation/cancelator.hpp

@@ -37,7 +37,7 @@ class Cancelator {
   virtual ~Cancelator() = default;
 
   virtual bool add_particle(BankedParticle& p) = 0;
-  virtual void perform_cancellation(pcg32& rng) = 0;
+  virtual void perform_cancellation() = 0;
   virtual std::vector<BankedParticle> get_new_particles(pcg32& rng) = 0;
   virtual void clear() = 0;
 };

include/simulation/exact_mg_cancelator.hpp

@@ -44,38 +44,19 @@ class ExactMGCancelator : public Cancelator {
                     uint32_t n_samples);
 
   bool add_particle(BankedParticle& p) override final;
-  void perform_cancellation(pcg32&) override final;
+  void perform_cancellation() override final;
   std::vector<BankedParticle> get_new_particles(pcg32& rng) override final;
   void clear() override final;
 
- private:
-  //==========================================================================
-  // Objects
-  struct CancelBin {
-    struct Averages {
-      double f = 0.;
-      double f_inv = 0.;
-    };
-
-    double uniform_wgt = 0.;
-    double uniform_wgt2 = 0.;
-    double W = 0.;
-    double W2 = 0.;
-    double sum_c = 0.;
-    double sum_c_wgt = 0.;
-    double sum_c_wgt2 = 0.;
-    bool can_cancel = true;
-    std::vector<BankedParticle*> particles;
-    std::vector<Averages> averages;
-  };
-
   // Key which represents a unique cancellation bin for a
   // given position and energy group.
+  // Key is public for MPI use
   struct Key {
-    Key(std::size_t i, std::size_t j, std::size_t k, std::size_t e)
+    Key(uint64_t i, uint64_t j, uint64_t k, uint64_t e)
         : i(i), j(j), k(k), e(e) {}
 
-    std::size_t i, j, k, e;
+    Key() : i(0), j(0), k(0), e(0) {}
+    uint64_t i, j, k, e;
 
     std::size_t hash_key() const {
       return e + shape[3] * (k + shape[2] * (j + shape[1] * i));
@@ -86,12 +67,14 @@ class ExactMGCancelator : public Cancelator {
               (e == other.e));
     }
 
+    auto operator<=>(const Key&) const = default;
+
     // Contains the shape of the cancellation region mesh.
     // shape[0] Number of regions in x
     // shape[1] Number of regions in y
     // shape[2] Number of regions in z
     // shape[3] Number of regions in energy
-    static std::array<std::size_t, 4> shape;
+    static std::array<uint64_t, 4> shape;
 
     // The width of each region in x, y, and z
     static std::array<double, 3> pitch;
@@ -105,6 +88,27 @@ class ExactMGCancelator : public Cancelator {
     static Position r_low, r_hi;
   };
 
+ private:
+  //==========================================================================
+  // Objects
+  struct CancelBin {
+    struct Averages {
+      double f = 0.;
+      double f_inv = 0.;
+    };
+
+    double uniform_wgt = 0.;
+    double uniform_wgt2 = 0.;
+    double W = 0.;
+    double W2 = 0.;
+    double sum_c = 0.;
+    double sum_c_wgt = 0.;
+    double sum_c_wgt2 = 0.;
+    bool can_cancel = true;
+    std::vector<BankedParticle*> particles;
+    std::vector<Averages> averages;
+  };
+
   struct KeyHash {
     std::size_t operator()(const Key& key) const { return key.hash_key(); }
   };
@@ -113,7 +117,7 @@ class ExactMGCancelator : public Cancelator {
   // Data Members
 
   // All cancellation bins, organized first by Key has, then by material.
-  std::unordered_map<Key, std::unordered_map<Material*, CancelBin>, KeyHash>
+  std::unordered_map<Key, std::unordered_map<uint32_t, CancelBin>, KeyHash>
       bins;
 
   // False if we only have MG materials with a chi vector.
@@ -133,6 +137,8 @@ class ExactMGCancelator : public Cancelator {
 
   std::optional<Key> get_key(const Position& r, std::size_t g);
 
+  std::vector<std::pair<ExactMGCancelator::Key, uint32_t>> sync_keys();
+
   // Get's a pointer to the material at r
   Material* get_material(const Position& r) const;
 
@@ -154,7 +160,7 @@ class ExactMGCancelator : public Cancelator {
   void compute_averages(const Key& key, Material* mat, MGNuclide* nuclide,
                         CancelBin& bin);
 
-  void cancel_bin(CancelBin& bin, MGNuclide* nuclide, bool first_wgt);
+  void cancel_bin(CancelBin& bin, MGNuclide* nuclide);
 };
 
 std::shared_ptr<ExactMGCancelator> make_exact_mg_cancelator(

include/simulation/noise.hpp

@@ -87,8 +87,7 @@ class Noise : public Simulation {
   void check_time(int gen);
 
   void normalize_weights(std::vector<BankedParticle>& next_gen);
-  void perform_regional_cancellation(std::vector<uint64_t>& nums,
-                                     std::vector<BankedParticle>& bank);
+  void perform_regional_cancellation(std::vector<BankedParticle>& bank);
 
   void power_iteration(bool sample_noise);
   void pi_generation_output();

include/simulation/power_iterator.hpp

@@ -79,7 +79,7 @@ class PowerIterator : public Simulation {
   void zero_entropy();
 
   // Pair-distance sqrd calculation
-  double compute_pair_dist_sqrd(const std::vector<BankedParticle>& next_gen);
+  void compute_pair_dist_sqrd(const std::vector<BankedParticle>& next_gen);
 
   void print_header();
 

include/simulation/simulation.hpp

@@ -106,10 +106,6 @@ class Simulation {
   void sync_banks(std::vector<uint64_t>& nums,
                   std::vector<BankedParticle>& bank);
 
-  void particles_to_master(std::vector<BankedParticle>& bank);
-  void distribute_particles(std::vector<uint64_t>& nums,
-                            std::vector<BankedParticle>& bank);
-
   void write_source(std::vector<Particle>& bank) const;
 
 };  // Simulation