From 240222e5ea5f74e5a2195d5405ee05f0b8fc8a5f Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Mon, 24 Feb 2025 17:14:18 -0500 Subject: [PATCH 01/21] refactor point_properties and property_container --- include/IO/property/reader.tpp | 18 +- include/IO/property/writer.tpp | 18 +- include/boundary_conditions/stacey/stacey.tpp | 20 +- .../acoustic/isotropic/frechet_derivative.hpp | 4 +- .../dim2/acoustic/isotropic/mass_matrix.tpp | 2 +- .../isotropic/properties_container.hpp | 218 +--------- .../medium/dim2/acoustic/isotropic/source.hpp | 2 +- .../medium/dim2/acoustic/isotropic/stress.hpp | 4 +- .../anisotropic/frechet_derivative.hpp | 14 +- .../anisotropic/properties_container.hpp | 341 +--------------- .../dim2/elastic/anisotropic/stress.hpp | 12 +- .../dim2/elastic/anisotropic/wavefield.hpp | 20 +- .../elastic/isotropic/frechet_derivative.hpp | 14 +- .../dim2/elastic/isotropic/mass_matrix.tpp | 6 +- .../isotropic/properties_container.hpp | 234 +---------- .../medium/dim2/elastic/isotropic/stress.hpp | 8 +- .../elastic/isotropic/stress_integrand.tpp | 10 +- .../dim2/elastic/isotropic/wavefield.hpp | 2 +- include/medium/properties_container.hpp | 159 ++++++++ include/point/properties.hpp | 382 +++++++----------- .../compute_wavefield/test_helper.hpp | 4 +- .../assembly/properties/properties.cpp | 245 ++++++----- 22 files changed, 563 insertions(+), 1174 deletions(-) diff --git a/include/IO/property/reader.tpp b/include/IO/property/reader.tpp index 60253ebb..5adf7216 100644 --- a/include/IO/property/reader.tpp +++ b/include/IO/property/reader.tpp @@ -23,31 +23,19 @@ void specfem::IO::property_reader::read(specfem::compute::assembly { typename InputLibrary::Group elastic = file.openGroup("/ElasticIsotropic"); - elastic.openDataset("rho", properties.elastic_isotropic.h_rho).read(); - elastic.openDataset("mu", properties.elastic_isotropic.h_mu).read(); - elastic.openDataset("lambdaplus2mu", properties.elastic_isotropic.h_lambdaplus2mu).read(); + elastic.openDataset("data", properties.elastic_isotropic.h_data).read(); } { typename InputLibrary::Group elastic = file.openGroup("/ElasticAnisotropic"); - elastic.openDataset("rho", properties.elastic_anisotropic.h_rho).read(); - elastic.openDataset("c11", properties.elastic_anisotropic.h_c11).read(); - elastic.openDataset("c13", properties.elastic_anisotropic.h_c13).read(); - elastic.openDataset("c15", properties.elastic_anisotropic.h_c15).read(); - elastic.openDataset("c33", properties.elastic_anisotropic.h_c33).read(); - elastic.openDataset("c35", properties.elastic_anisotropic.h_c35).read(); - elastic.openDataset("c55", properties.elastic_anisotropic.h_c55).read(); - elastic.openDataset("c12", properties.elastic_anisotropic.h_c12).read(); - elastic.openDataset("c23", properties.elastic_anisotropic.h_c23).read(); - elastic.openDataset("c25", properties.elastic_anisotropic.h_c25).read(); + elastic.openDataset("data", properties.elastic_anisotropic.h_data).read(); } { typename InputLibrary::Group acoustic = file.openGroup("/Acoustic"); - acoustic.openDataset("rho_inverse", properties.acoustic_isotropic.h_rho_inverse).read(); - acoustic.openDataset("kappa", properties.acoustic_isotropic.h_kappa).read(); + acoustic.openDataset("data", properties.acoustic_isotropic.h_data).read(); } std::cout << "Properties read from " << input_folder << "/Properties" diff --git a/include/IO/property/writer.tpp b/include/IO/property/writer.tpp index 563a2242..b0682f0d 100644 --- a/include/IO/property/writer.tpp +++ b/include/IO/property/writer.tpp @@ -57,9 +57,7 @@ void specfem::IO::property_writer::write(specfem::compute::assemb elastic.createDataset("X", x).write(); elastic.createDataset("Z", z).write(); - elastic.createDataset("rho", properties.elastic_isotropic.h_rho).write(); - elastic.createDataset("mu", properties.elastic_isotropic.h_mu).write(); - elastic.createDataset("lambdaplus2mu", properties.elastic_isotropic.h_lambdaplus2mu).write(); + elastic.createDataset("data", properties.elastic_isotropic.h_data).write(); } { @@ -86,16 +84,7 @@ void specfem::IO::property_writer::write(specfem::compute::assemb elastic.createDataset("X", x).write(); elastic.createDataset("Z", z).write(); - elastic.createDataset("rho", properties.elastic_anisotropic.h_rho).write(); - elastic.createDataset("c11", properties.elastic_anisotropic.h_c11).write(); - elastic.createDataset("c13", properties.elastic_anisotropic.h_c13).write(); - elastic.createDataset("c15", properties.elastic_anisotropic.h_c15).write(); - elastic.createDataset("c33", properties.elastic_anisotropic.h_c33).write(); - elastic.createDataset("c35", properties.elastic_anisotropic.h_c35).write(); - elastic.createDataset("c55", properties.elastic_anisotropic.h_c55).write(); - elastic.createDataset("c12", properties.elastic_anisotropic.h_c12).write(); - elastic.createDataset("c23", properties.elastic_anisotropic.h_c23).write(); - elastic.createDataset("c25", properties.elastic_anisotropic.h_c25).write(); + elastic.createDataset("data", properties.elastic_anisotropic.h_data).write(); } { @@ -120,8 +109,7 @@ void specfem::IO::property_writer::write(specfem::compute::assemb acoustic.createDataset("X", x).write(); acoustic.createDataset("Z", z).write(); - acoustic.createDataset("rho_inverse", properties.acoustic_isotropic.h_rho_inverse).write(); - acoustic.createDataset("kappa", properties.acoustic_isotropic.h_kappa).write(); + acoustic.createDataset("data", properties.acoustic_isotropic.h_data).write(); } assert(n_elastic_isotropic + n_elastic_anisotropic + n_acoustic == nspec); diff --git a/include/boundary_conditions/stacey/stacey.tpp b/include/boundary_conditions/stacey/stacey.tpp index 7f3010b0..b84b4a2e 100644 --- a/include/boundary_conditions/stacey/stacey.tpp +++ b/include/boundary_conditions/stacey/stacey.tpp @@ -57,7 +57,7 @@ impl_enforce_traction(const acoustic_type &, const isotropic_type &, // Apply Stacey boundary condition traction(0) += static_cast(-1.0) * factor * - property.rho_vpinverse * field.velocity(0); + property.rho_vpinverse() * field.velocity(0); // Do nothing return; @@ -101,7 +101,7 @@ impl_enforce_traction(const acoustic_type &, const isotropic_type &, // Apply Stacey boundary condition Kokkos::Experimental::where(mask, traction(0)) = traction(0) + static_cast(-1.0) * factor * - property.rho_vpinverse * field.velocity(0); + property.rho_vpinverse() * field.velocity(0); return; } @@ -146,8 +146,8 @@ impl_enforce_traction(const elastic_type &, const isotropic_type &, for (int icomp = 0; icomp < 2; ++icomp) { factor[icomp] = ((vn * dn(icomp) / (jacobian1d * jacobian1d)) * - (property.rho_vp - property.rho_vs)) + - field.velocity(icomp) * property.rho_vs; + (property.rho_vp() - property.rho_vs())) + + field.velocity(icomp) * property.rho_vs(); } traction(0) += static_cast(-1.0) * factor[0] * jacobian1d * @@ -203,8 +203,8 @@ impl_enforce_traction(const elastic_type &, const isotropic_type &, for (int icomp = 0; icomp < 2; ++icomp) { factor[icomp] = ((vn * dn(icomp) / (jacobian1d * jacobian1d)) * - (property.rho_vp - property.rho_vs)) + - field.velocity(icomp) * property.rho_vs; + (property.rho_vp() - property.rho_vs())) + + field.velocity(icomp) * property.rho_vs(); } Kokkos::Experimental::where(mask, traction(0)) = @@ -259,8 +259,8 @@ impl_enforce_traction(const elastic_type &, const anisotropic_type &, for (int icomp = 0; icomp < 2; ++icomp) { factor[icomp] = ((vn * dn(icomp) / (jacobian1d * jacobian1d)) * - (property.rho_vp - property.rho_vs)) + - field.velocity(icomp) * property.rho_vs; + (property.rho_vp() - property.rho_vs())) + + field.velocity(icomp) * property.rho_vs(); } traction(0) += static_cast(-1.0) * factor[0] * jacobian1d * @@ -316,8 +316,8 @@ impl_enforce_traction(const elastic_type &, const anisotropic_type &, for (int icomp = 0; icomp < 2; ++icomp) { factor[icomp] = ((vn * dn(icomp) / (jacobian1d * jacobian1d)) * - (property.rho_vp - property.rho_vs)) + - field.velocity(icomp) * property.rho_vs; + (property.rho_vp() - property.rho_vs())) + + field.velocity(icomp) * property.rho_vs(); } Kokkos::Experimental::where(mask, traction(0)) = diff --git a/include/medium/dim2/acoustic/isotropic/frechet_derivative.hpp b/include/medium/dim2/acoustic/isotropic/frechet_derivative.hpp index 25f48a34..59b2f215 100644 --- a/include/medium/dim2/acoustic/isotropic/frechet_derivative.hpp +++ b/include/medium/dim2/acoustic/isotropic/frechet_derivative.hpp @@ -30,11 +30,11 @@ impl_compute_frechet_derivatives( const auto rho_kl = (adjoint_derivatives.du(0, 0) * backward_derivatives.du(0, 0) + adjoint_derivatives.du(1, 0) * backward_derivatives.du(1, 0)) * - properties.rho_inverse * dt; + properties.rho_inverse() * dt; const auto kappa_kl = specfem::algorithms::dot(adjoint_field.acceleration, backward_field.displacement) * - static_cast(1.0) / properties.kappa * dt; + static_cast(1.0) / properties.kappa() * dt; return { rho_kl, kappa_kl }; } diff --git a/include/medium/dim2/acoustic/isotropic/mass_matrix.tpp b/include/medium/dim2/acoustic/isotropic/mass_matrix.tpp index 4533bf91..7af19384 100644 --- a/include/medium/dim2/acoustic/isotropic/mass_matrix.tpp +++ b/include/medium/dim2/acoustic/isotropic/mass_matrix.tpp @@ -14,5 +14,5 @@ specfem::medium::impl_mass_matrix_component( specfem::dimension::type::dim2, true, UseSIMD> &partial_derivatives) { return specfem::datatype::ScalarPointViewType( - partial_derivatives.jacobian / properties.kappa); + partial_derivatives.jacobian / properties.kappa()); } diff --git a/include/medium/dim2/acoustic/isotropic/properties_container.hpp b/include/medium/dim2/acoustic/isotropic/properties_container.hpp index 9c76c2b6..ee9381fe 100644 --- a/include/medium/dim2/acoustic/isotropic/properties_container.hpp +++ b/include/medium/dim2/acoustic/isotropic/properties_container.hpp @@ -9,211 +9,19 @@ namespace medium { template <> struct properties_container { - - constexpr static auto dimension = specfem::dimension::type::dim2; - constexpr static auto value_type = specfem::element::medium_tag::acoustic; - constexpr static auto property_type = - specfem::element::property_tag::isotropic; - - using ViewType = typename Kokkos::View; - - int nspec; ///< total number of acoustic spectral elements - int ngllz; ///< number of quadrature points in z dimension - int ngllx; ///< number of quadrature points in x dimension - ViewType rho_inverse; - ViewType::HostMirror h_rho_inverse; - ViewType kappa; - ViewType::HostMirror h_kappa; - - properties_container() = default; - - properties_container(const int nspec, const int ngllz, const int ngllx) - : nspec(nspec), ngllz(ngllz), ngllx(ngllx), - rho_inverse("specfem::compute::properties::rho_inverse", nspec, ngllz, - ngllx), - h_rho_inverse(Kokkos::create_mirror_view(rho_inverse)), - kappa("specfem::compute::properties::kappa", nspec, ngllz, ngllx), - h_kappa(Kokkos::create_mirror_view(kappa)) {} - - template < - typename PointProperties, - typename std::enable_if_t = 0> - KOKKOS_FORCEINLINE_FUNCTION void - load_device_properties(const specfem::point::index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - property.rho_inverse = rho_inverse(ispec, iz, ix); - property.kappa = kappa(ispec, iz, ix); - property.kappa_inverse = static_cast(1.0) / property.kappa; - property.rho_vpinverse = - sqrt(property.rho_inverse * property.kappa_inverse); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - KOKKOS_FORCEINLINE_FUNCTION void - load_device_properties(const specfem::point::simd_index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho_inverse) - .copy_from(&rho_inverse(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.kappa) - .copy_from(&kappa(ispec, iz, ix), tag_type()); - - property.kappa_inverse = static_cast(1.0) / property.kappa; - property.rho_vpinverse = - Kokkos::sqrt(property.rho_inverse * property.kappa_inverse); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void - load_host_properties(const specfem::point::index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - property.rho_inverse = h_rho_inverse(ispec, iz, ix); - property.kappa = h_kappa(ispec, iz, ix); - property.kappa_inverse = static_cast(1.0) / property.kappa; - property.rho_vpinverse = - sqrt(property.rho_inverse * property.kappa_inverse); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void - load_host_properties(const specfem::point::simd_index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho_inverse) - .copy_from(&h_rho_inverse(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.kappa) - .copy_from(&h_kappa(ispec, iz, ix), tag_type()); - - property.kappa_inverse = static_cast(1.0) / property.kappa; - property.rho_vpinverse = - Kokkos::sqrt(property.rho_inverse * property.kappa_inverse); - } - - void copy_to_device() { - Kokkos::deep_copy(rho_inverse, h_rho_inverse); - Kokkos::deep_copy(kappa, h_kappa); - } - - void copy_to_host() { - Kokkos::deep_copy(h_rho_inverse, rho_inverse); - Kokkos::deep_copy(h_kappa, kappa); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void assign(const specfem::point::index &index, - const PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho_inverse(ispec, iz, ix) = property.rho_inverse; - h_kappa(ispec, iz, ix) = property.kappa; - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void assign(const specfem::point::simd_index &index, - const PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho_inverse) - .copy_to(&h_rho_inverse(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.kappa) - .copy_to(&h_kappa(ispec, iz, ix), tag_type()); - } + specfem::element::property_tag::isotropic> + : public impl::impl_properties_container< + specfem::element::medium_tag::acoustic, + specfem::element::property_tag::isotropic, 2> { + using Base = + impl::impl_properties_container; + using Base::Base; + constexpr static int _counter = __COUNTER__; + + DEFINE_CONTAINER(rho_inverse) + DEFINE_CONTAINER(kappa) }; } // namespace medium diff --git a/include/medium/dim2/acoustic/isotropic/source.hpp b/include/medium/dim2/acoustic/isotropic/source.hpp index 6ba010ad..b5e7e43b 100644 --- a/include/medium/dim2/acoustic/isotropic/source.hpp +++ b/include/medium/dim2/acoustic/isotropic/source.hpp @@ -27,7 +27,7 @@ KOKKOS_INLINE_FUNCTION auto impl_compute_source_contribution( result.acceleration(0) = point_source.stf(0) * point_source.lagrange_interpolant(0) / - point_properties.kappa; + point_properties.kappa(); return result; } diff --git a/include/medium/dim2/acoustic/isotropic/stress.hpp b/include/medium/dim2/acoustic/isotropic/stress.hpp index 895a43e5..38f2c5d6 100644 --- a/include/medium/dim2/acoustic/isotropic/stress.hpp +++ b/include/medium/dim2/acoustic/isotropic/stress.hpp @@ -26,8 +26,8 @@ impl_compute_stress( specfem::datatype::VectorPointViewType T; - T(0, 0) = properties.rho_inverse * du(0, 0); - T(1, 0) = properties.rho_inverse * du(1, 0); + T(0, 0) = properties.rho_inverse() * du(0, 0); + T(1, 0) = properties.rho_inverse() * du(1, 0); return { T }; } diff --git a/include/medium/dim2/elastic/anisotropic/frechet_derivative.hpp b/include/medium/dim2/elastic/anisotropic/frechet_derivative.hpp index 2f83ec85..97ee5a04 100644 --- a/include/medium/dim2/elastic/anisotropic/frechet_derivative.hpp +++ b/include/medium/dim2/elastic/anisotropic/frechet_derivative.hpp @@ -79,13 +79,13 @@ impl_compute_frechet_derivatives( // Computing the rest of the integral. // rho from equation 14 - rho_kl = static_cast(-1.0) * properties.rho * dt * rho_kl; - c11_kl = static_cast(-1.0) * c11_kl * properties.c11 * dt; - c13_kl = static_cast(-1.0) * c13_kl * properties.c13 * dt; - c15_kl = static_cast(-1.0) * c15_kl * properties.c15 * dt; - c33_kl = static_cast(-1.0) * c33_kl * properties.c33 * dt; - c35_kl = static_cast(-1.0) * c35_kl * properties.c35 * dt; - c55_kl = static_cast(-1.0) * c55_kl * properties.c55 * dt; + rho_kl = static_cast(-1.0) * properties.rho() * dt * rho_kl; + c11_kl = static_cast(-1.0) * c11_kl * properties.c11() * dt; + c13_kl = static_cast(-1.0) * c13_kl * properties.c13() * dt; + c15_kl = static_cast(-1.0) * c15_kl * properties.c15() * dt; + c33_kl = static_cast(-1.0) * c33_kl * properties.c33() * dt; + c35_kl = static_cast(-1.0) * c35_kl * properties.c35() * dt; + c55_kl = static_cast(-1.0) * c55_kl * properties.c55() * dt; return { rho_kl, c11_kl, c13_kl, c15_kl, c33_kl, c35_kl, c55_kl }; diff --git a/include/medium/dim2/elastic/anisotropic/properties_container.hpp b/include/medium/dim2/elastic/anisotropic/properties_container.hpp index ca78ed05..b1307985 100644 --- a/include/medium/dim2/elastic/anisotropic/properties_container.hpp +++ b/include/medium/dim2/elastic/anisotropic/properties_container.hpp @@ -9,327 +9,26 @@ namespace medium { template <> struct properties_container { - - constexpr static auto dimension = specfem::dimension::type::dim2; - constexpr static auto value_type = specfem::element::medium_tag::elastic; - constexpr static auto property_type = - specfem::element::property_tag::anisotropic; - - using ViewType = typename Kokkos::View; - - int nspec; ///< total number of acoustic spectral elements - int ngllz; ///< number of quadrature points in z dimension - int ngllx; ///< number of quadrature points in x dimension - - ViewType rho; - ViewType::HostMirror h_rho; - ViewType c11; - ViewType::HostMirror h_c11; - ViewType c13; - ViewType::HostMirror h_c13; - ViewType c15; - ViewType::HostMirror h_c15; - ViewType c33; - ViewType::HostMirror h_c33; - ViewType c35; - ViewType::HostMirror h_c35; - ViewType c55; - ViewType::HostMirror h_c55; - ViewType c12; - ViewType::HostMirror h_c12; - ViewType c23; - ViewType::HostMirror h_c23; - ViewType c25; - ViewType::HostMirror h_c25; - - properties_container() = default; - - properties_container(const int nspec, const int ngllz, const int ngllx) - : nspec(nspec), ngllz(ngllz), ngllx(ngllx), - rho("specfem::compute::properties::rho", nspec, ngllz, ngllx), - h_rho(Kokkos::create_mirror_view(rho)), - c11("specfem::compute::properties::c11", nspec, ngllz, ngllx), - h_c11(Kokkos::create_mirror_view(c11)), - c12("specfem::compute::properties::c12", nspec, ngllz, ngllx), - h_c12(Kokkos::create_mirror_view(c12)), - c13("specfem::compute::properties::c13", nspec, ngllz, ngllx), - h_c13(Kokkos::create_mirror_view(c13)), - c15("specfem::compute::properties::c15", nspec, ngllz, ngllx), - h_c15(Kokkos::create_mirror_view(c15)), - c33("specfem::compute::properties::c33", nspec, ngllz, ngllx), - h_c33(Kokkos::create_mirror_view(c33)), - c35("specfem::compute::properties::c35", nspec, ngllz, ngllx), - h_c35(Kokkos::create_mirror_view(c35)), - c55("specfem::compute::properties::c55", nspec, ngllz, ngllx), - h_c55(Kokkos::create_mirror_view(c55)), - c23("specfem::compute::properties::c23", nspec, ngllz, ngllx), - h_c23(Kokkos::create_mirror_view(c23)), - c25("specfem::compute::properties::c25", nspec, ngllz, ngllx), - h_c25(Kokkos::create_mirror_view(c25)) {} - - template < - typename PointProperties, - typename std::enable_if_t = 0> - KOKKOS_FORCEINLINE_FUNCTION void - load_device_properties(const specfem::point::index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - property.rho = rho(ispec, iz, ix); - property.c11 = c11(ispec, iz, ix); - property.c12 = c12(ispec, iz, ix); - property.c13 = c13(ispec, iz, ix); - property.c15 = c15(ispec, iz, ix); - property.c33 = c33(ispec, iz, ix); - property.c35 = c35(ispec, iz, ix); - property.c55 = c55(ispec, iz, ix); - property.c23 = c23(ispec, iz, ix); - property.c25 = c25(ispec, iz, ix); - - property.rho_vp = sqrt(property.rho * property.c33); - property.rho_vs = sqrt(property.rho * property.c55); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - KOKKOS_FORCEINLINE_FUNCTION void - load_device_properties(const specfem::point::simd_index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho) - .copy_from(&rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c11) - .copy_from(&c11(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c12) - .copy_from(&c12(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c13) - .copy_from(&c13(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c15) - .copy_from(&c15(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c33) - .copy_from(&c33(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c35) - .copy_from(&c35(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c55) - .copy_from(&c55(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c23) - .copy_from(&c23(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c25) - .copy_from(&c25(ispec, iz, ix), tag_type()); - - property.rho_vp = Kokkos::sqrt(property.rho * property.c33); - property.rho_vs = Kokkos::sqrt(property.rho * property.c55); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void - load_host_properties(const specfem::point::index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - property.rho = h_rho(ispec, iz, ix); - property.c11 = h_c11(ispec, iz, ix); - property.c12 = h_c12(ispec, iz, ix); - property.c13 = h_c13(ispec, iz, ix); - property.c15 = h_c15(ispec, iz, ix); - property.c33 = h_c33(ispec, iz, ix); - property.c35 = h_c35(ispec, iz, ix); - property.c55 = h_c55(ispec, iz, ix); - property.c23 = h_c23(ispec, iz, ix); - property.c25 = h_c25(ispec, iz, ix); - - property.rho_vp = sqrt(property.rho * property.c33); - property.rho_vs = sqrt(property.rho * property.c55); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void - load_host_properties(const specfem::point::simd_index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho) - .copy_from(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c11) - .copy_from(&h_c11(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c13) - .copy_from(&h_c13(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c15) - .copy_from(&h_c15(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c33) - .copy_from(&h_c33(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c35) - .copy_from(&h_c35(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c55) - .copy_from(&h_c55(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c23) - .copy_from(&h_c23(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c25) - .copy_from(&h_c25(ispec, iz, ix), tag_type()); - - property.rho_vp = Kokkos::sqrt(property.rho * property.c33); - property.rho_vs = Kokkos::sqrt(property.rho * property.c55); - } - - void copy_to_device() { - Kokkos::deep_copy(rho, h_rho); - Kokkos::deep_copy(c11, h_c11); - Kokkos::deep_copy(c13, h_c13); - Kokkos::deep_copy(c15, h_c15); - Kokkos::deep_copy(c12, h_c12); - Kokkos::deep_copy(c33, h_c33); - Kokkos::deep_copy(c35, h_c35); - Kokkos::deep_copy(c55, h_c55); - Kokkos::deep_copy(c23, h_c23); - Kokkos::deep_copy(c25, h_c25); - } - - void copy_to_host() { - Kokkos::deep_copy(h_rho, rho); - Kokkos::deep_copy(h_c11, c11); - Kokkos::deep_copy(h_c13, c13); - Kokkos::deep_copy(h_c15, c15); - Kokkos::deep_copy(h_c12, c12); - Kokkos::deep_copy(h_c33, c33); - Kokkos::deep_copy(h_c35, c35); - Kokkos::deep_copy(h_c55, c55); - Kokkos::deep_copy(h_c23, c23); - Kokkos::deep_copy(h_c25, c25); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void assign(const specfem::point::index &index, - const PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho(ispec, iz, ix) = property.rho; - h_c11(ispec, iz, ix) = property.c11; - h_c13(ispec, iz, ix) = property.c13; - h_c15(ispec, iz, ix) = property.c15; - h_c12(ispec, iz, ix) = property.c12; - h_c33(ispec, iz, ix) = property.c33; - h_c35(ispec, iz, ix) = property.c35; - h_c55(ispec, iz, ix) = property.c55; - h_c23(ispec, iz, ix) = property.c23; - h_c25(ispec, iz, ix) = property.c25; - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void assign(const specfem::point::simd_index &index, - const PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho) - .copy_to(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c11) - .copy_to(&h_c11(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c13) - .copy_to(&h_c13(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c15) - .copy_to(&h_c15(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c12) - .copy_to(&h_c12(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c33) - .copy_to(&h_c33(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c35) - .copy_to(&h_c35(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c55) - .copy_to(&h_c55(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c23) - .copy_to(&h_c23(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c25) - .copy_to(&h_c25(ispec, iz, ix), tag_type()); - } + specfem::element::property_tag::anisotropic> + : public impl::impl_properties_container< + specfem::element::medium_tag::elastic, + specfem::element::property_tag::anisotropic, 10> { + using Base = impl::impl_properties_container< + specfem::element::medium_tag::elastic, + specfem::element::property_tag::anisotropic, 10>; + using Base::Base; + constexpr static int _counter = __COUNTER__; + + DEFINE_CONTAINER(c11) + DEFINE_CONTAINER(c13) + DEFINE_CONTAINER(c15) + DEFINE_CONTAINER(c33) + DEFINE_CONTAINER(c35) + DEFINE_CONTAINER(c55) + DEFINE_CONTAINER(c12) + DEFINE_CONTAINER(c23) + DEFINE_CONTAINER(c25) + DEFINE_CONTAINER(rho) }; } // namespace medium diff --git a/include/medium/dim2/elastic/anisotropic/stress.hpp b/include/medium/dim2/elastic/anisotropic/stress.hpp index 39ba0382..563cd2f4 100644 --- a/include/medium/dim2/elastic/anisotropic/stress.hpp +++ b/include/medium/dim2/elastic/anisotropic/stress.hpp @@ -28,16 +28,16 @@ impl_compute_stress( // P_SV case // sigma_xx - sigma_xx = properties.c11 * du(0, 0) + properties.c13 * du(1, 1) + - properties.c15 * (du(1, 0) + du(0, 1)); + sigma_xx = properties.c11() * du(0, 0) + properties.c13() * du(1, 1) + + properties.c15() * (du(1, 0) + du(0, 1)); // sigma_zz - sigma_zz = properties.c13 * du(0, 0) + properties.c33 * du(1, 1) + - properties.c35 * (du(1, 0) + du(0, 1)); + sigma_zz = properties.c13() * du(0, 0) + properties.c33() * du(1, 1) + + properties.c35() * (du(1, 0) + du(0, 1)); // sigma_xz - sigma_xz = properties.c15 * du(0, 0) + properties.c35 * du(1, 1) + - properties.c55 * (du(1, 0) + du(0, 1)); + sigma_xz = properties.c15() * du(0, 0) + properties.c35() * du(1, 1) + + properties.c55() * (du(1, 0) + du(0, 1)); specfem::datatype::VectorPointViewType T; diff --git a/include/medium/dim2/elastic/anisotropic/wavefield.hpp b/include/medium/dim2/elastic/anisotropic/wavefield.hpp index 4dfd2050..b0f4ff6f 100644 --- a/include/medium/dim2/elastic/anisotropic/wavefield.hpp +++ b/include/medium/dim2/elastic/anisotropic/wavefield.hpp @@ -66,7 +66,7 @@ KOKKOS_FUNCTION void impl_compute_wavefield( // cannot compute pressure for an anisotropic material if c12 or c23 // are zero - if (point_property.c12 < 1.e-7 || point_property.c23 < 1.e-7) { + if (point_property.c12() < 1.e-7 || point_property.c23() < 1.e-7) { Kokkos::abort("C_12 or C_23 are zero, cannot compute pressure. " "Check your material properties. Or, deactivate the " "pressure computation."); @@ -85,19 +85,19 @@ KOKKOS_FUNCTION void impl_compute_wavefield( // P_SV case // sigma_xx - const auto sigma_xx = point_property.c11 * du(0, 0) + - point_property.c13 * du(1, 1) + - point_property.c15 * (du(1, 0) + du(0, 1)); + const auto sigma_xx = point_property.c11() * du(0, 0) + + point_property.c13() * du(1, 1) + + point_property.c15() * (du(1, 0) + du(0, 1)); // sigma_zz - const auto sigma_zz = point_property.c13 * du(0, 0) + - point_property.c33 * du(1, 1) + - point_property.c35 * (du(1, 0) + du(0, 1)); + const auto sigma_zz = point_property.c13() * du(0, 0) + + point_property.c33() * du(1, 1) + + point_property.c35() * (du(1, 0) + du(0, 1)); // sigma_yy - const auto sigma_yy = point_property.c12 * du(0, 0) + - point_property.c23 * du(1, 1) + - point_property.c25 * (du(1, 0) + du(0, 1)); + const auto sigma_yy = point_property.c12() * du(0, 0) + + point_property.c23() * du(1, 1) + + point_property.c25() * (du(1, 0) + du(0, 1)); wavefield(iterator_index.ielement, index.iz, index.ix, 0) = -1.0 * (sigma_xx + sigma_zz + sigma_yy) / 3.0; diff --git a/include/medium/dim2/elastic/isotropic/frechet_derivative.hpp b/include/medium/dim2/elastic/isotropic/frechet_derivative.hpp index 3949495b..8853fd43 100644 --- a/include/medium/dim2/elastic/isotropic/frechet_derivative.hpp +++ b/include/medium/dim2/elastic/isotropic/frechet_derivative.hpp @@ -34,7 +34,7 @@ impl_compute_frechet_derivatives( specfem::globals::simulation_wave == specfem::wave::sh, "Only P-SV and SH waves are supported."); - const auto kappa = properties.lambdaplus2mu - properties.mu; + const auto kappa = properties.lambdaplus2mu() - properties.mu(); if (specfem::globals::simulation_wave == specfem::wave::p_sv) { @@ -121,8 +121,8 @@ impl_compute_frechet_derivatives( // Finishing the kernels kappa_kl = static_cast(-1.0) * kappa * dt * kappa_kl; - mu_kl = static_cast(-2.0) * properties.mu * dt * mu_kl; - rho_kl = static_cast(-1.0) * properties.rho * dt * rho_kl; + mu_kl = static_cast(-2.0) * properties.mu() * dt * mu_kl; + rho_kl = static_cast(-1.0) * properties.rho() * dt * rho_kl; // rho' kernel, first term in Equation 20 const auto rhop_kl = rho_kl + kappa_kl + mu_kl; @@ -130,14 +130,14 @@ impl_compute_frechet_derivatives( // beta (shear wave) kernel, second term in Equation 20 const auto beta_kl = static_cast(2.0) * (mu_kl - static_cast(4.0 / 3.0) * - properties.mu / kappa * kappa_kl); + properties.mu() / kappa * kappa_kl); // alpha (compressional wave) kernel, third and last term in Eq. 20 // of Tromp et al 2005. const auto alpha_kl = static_cast(2.0) * (static_cast(1.0) + - static_cast(4.0 / 3.0) * properties.mu / kappa) * + static_cast(4.0 / 3.0) * properties.mu() / kappa) * kappa_kl; return { rho_kl, mu_kl, kappa_kl, rhop_kl, alpha_kl, beta_kl }; @@ -164,13 +164,13 @@ impl_compute_frechet_derivatives( */ const auto mu_kl = - static_cast(-2.0) * properties.mu * dt * + static_cast(-2.0) * properties.mu() * dt * static_cast(0.5) * // du#y_dx * duy_dx + (adjoint_derivatives.du(0, 0) * backward_derivatives.du(0, 0) + // du#y_dz * duy_dz adjoint_derivatives.du(1, 0) * backward_derivatives.du(1, 0)); - const auto rho_kl = static_cast(-1.0) * properties.rho * dt * + const auto rho_kl = static_cast(-1.0) * properties.rho() * dt * specfem::algorithms::dot(adjoint_field.acceleration, backward_field.displacement); const auto kappa_kl = decltype(mu_kl)(0.0); diff --git a/include/medium/dim2/elastic/isotropic/mass_matrix.tpp b/include/medium/dim2/elastic/isotropic/mass_matrix.tpp index f49f3d8f..28b8ae7a 100644 --- a/include/medium/dim2/elastic/isotropic/mass_matrix.tpp +++ b/include/medium/dim2/elastic/isotropic/mass_matrix.tpp @@ -15,11 +15,11 @@ specfem::medium::impl_mass_matrix_component( if constexpr (specfem::globals::simulation_wave == specfem::wave::p_sv) { return specfem::datatype::ScalarPointViewType( - partial_derivatives.jacobian * properties.rho, - partial_derivatives.jacobian * properties.rho); + partial_derivatives.jacobian * properties.rho(), + partial_derivatives.jacobian * properties.rho()); } else if constexpr (specfem::globals::simulation_wave == specfem::wave::sh) { return specfem::datatype::ScalarPointViewType( - partial_derivatives.jacobian * properties.rho, 0); + partial_derivatives.jacobian * properties.rho(), 0); } else { static_assert("Unknown wave type"); return specfem::datatype::ScalarPointViewType( diff --git a/include/medium/dim2/elastic/isotropic/properties_container.hpp b/include/medium/dim2/elastic/isotropic/properties_container.hpp index 4be8eda7..1a4e8bdc 100644 --- a/include/medium/dim2/elastic/isotropic/properties_container.hpp +++ b/include/medium/dim2/elastic/isotropic/properties_container.hpp @@ -9,226 +9,20 @@ namespace medium { template <> struct properties_container { - - constexpr static auto dimension = specfem::dimension::type::dim2; - constexpr static auto value_type = specfem::element::medium_tag::elastic; - constexpr static auto property_type = - specfem::element::property_tag::isotropic; - - using ViewType = typename Kokkos::View; - - int nspec; ///< total number of acoustic spectral elements - int ngllz; ///< number of quadrature points in z dimension - int ngllx; ///< number of quadrature points in x dimension - ViewType rho; - ViewType::HostMirror h_rho; - ViewType mu; - ViewType::HostMirror h_mu; - ViewType lambdaplus2mu; - ViewType::HostMirror h_lambdaplus2mu; - - properties_container() = default; - - properties_container(const int nspec, const int ngllz, const int ngllx) - : nspec(nspec), ngllz(ngllz), ngllx(ngllx), - rho("specfem::compute::properties::rho", nspec, ngllz, ngllx), - h_rho(Kokkos::create_mirror_view(rho)), - mu("specfem::compute::properties::mu", nspec, ngllz, ngllx), - h_mu(Kokkos::create_mirror_view(mu)), - lambdaplus2mu("specfem::compute::properties::lambdaplus2mu", nspec, - ngllz, ngllx), - h_lambdaplus2mu(Kokkos::create_mirror_view(lambdaplus2mu)) {} - - template < - typename PointProperties, - typename std::enable_if_t = 0> - KOKKOS_FORCEINLINE_FUNCTION void - load_device_properties(const specfem::point::index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - property.rho = rho(ispec, iz, ix); - property.mu = mu(ispec, iz, ix); - property.lambdaplus2mu = lambdaplus2mu(ispec, iz, ix); - property.lambda = property.lambdaplus2mu - 2 * property.mu; - property.rho_vp = sqrt(property.rho * property.lambdaplus2mu); - property.rho_vs = sqrt(property.rho * property.mu); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - KOKKOS_FORCEINLINE_FUNCTION void - load_device_properties(const specfem::point::simd_index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho) - .copy_from(&rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.mu) - .copy_from(&mu(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.lambdaplus2mu) - .copy_from(&lambdaplus2mu(ispec, iz, ix), tag_type()); - - property.lambda = property.lambdaplus2mu - 2 * property.mu; - property.rho_vp = Kokkos::sqrt(property.rho * property.lambdaplus2mu); - property.rho_vs = Kokkos::sqrt(property.rho * property.mu); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void - load_host_properties(const specfem::point::index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - property.rho = h_rho(ispec, iz, ix); - property.mu = h_mu(ispec, iz, ix); - property.lambdaplus2mu = h_lambdaplus2mu(ispec, iz, ix); - property.lambda = property.lambdaplus2mu - 2 * property.mu; - property.rho_vp = sqrt(property.rho * property.lambdaplus2mu); - property.rho_vs = sqrt(property.rho * property.mu); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void - load_host_properties(const specfem::point::simd_index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho) - .copy_from(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.mu) - .copy_from(&h_mu(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.lambdaplus2mu) - .copy_from(&h_lambdaplus2mu(ispec, iz, ix), tag_type()); - - property.lambda = property.lambdaplus2mu - 2 * property.mu; - property.rho_vp = Kokkos::sqrt(property.rho * property.lambdaplus2mu); - property.rho_vs = Kokkos::sqrt(property.rho * property.mu); - } - - void copy_to_device() { - Kokkos::deep_copy(rho, h_rho); - Kokkos::deep_copy(mu, h_mu); - Kokkos::deep_copy(lambdaplus2mu, h_lambdaplus2mu); - } - - void copy_to_host() { - Kokkos::deep_copy(h_rho, rho); - Kokkos::deep_copy(h_mu, mu); - Kokkos::deep_copy(h_lambdaplus2mu, lambdaplus2mu); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void assign(const specfem::point::index &index, - const PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho(ispec, iz, ix) = property.rho; - h_mu(ispec, iz, ix) = property.mu; - h_lambdaplus2mu(ispec, iz, ix) = property.lambdaplus2mu; - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void assign(const specfem::point::simd_index &index, - const PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho) - .copy_to(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.mu) - .copy_to(&h_mu(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.lambdaplus2mu) - .copy_to(&h_lambdaplus2mu(ispec, iz, ix), tag_type()); - } + specfem::element::property_tag::isotropic> + : public impl::impl_properties_container< + specfem::element::medium_tag::elastic, + specfem::element::property_tag::isotropic, 3> { + using Base = + impl::impl_properties_container; + using Base::Base; + constexpr static int _counter = __COUNTER__; + + DEFINE_CONTAINER(lambdaplus2mu) + DEFINE_CONTAINER(mu) + DEFINE_CONTAINER(rho) }; } // namespace medium diff --git a/include/medium/dim2/elastic/isotropic/stress.hpp b/include/medium/dim2/elastic/isotropic/stress.hpp index a90f6cce..e7088a7f 100644 --- a/include/medium/dim2/elastic/isotropic/stress.hpp +++ b/include/medium/dim2/elastic/isotropic/stress.hpp @@ -29,13 +29,15 @@ impl_compute_stress( // P_SV case // sigma_xx - sigma_xx = properties.lambdaplus2mu * du(0, 0) + properties.lambda * du(1, 1); + sigma_xx = + properties.lambdaplus2mu() * du(0, 0) + properties.lambda() * du(1, 1); // sigma_zz - sigma_zz = properties.lambdaplus2mu * du(1, 1) + properties.lambda * du(0, 0); + sigma_zz = + properties.lambdaplus2mu() * du(1, 1) + properties.lambda() * du(0, 0); // sigma_xz - sigma_xz = properties.mu * (du(0, 1) + du(1, 0)); + sigma_xz = properties.mu() * (du(0, 1) + du(1, 0)); specfem::datatype::VectorPointViewType T; diff --git a/include/medium/dim2/elastic/isotropic/stress_integrand.tpp b/include/medium/dim2/elastic/isotropic/stress_integrand.tpp index e6d2f0d8..12f668a9 100644 --- a/include/medium/dim2/elastic/isotropic/stress_integrand.tpp +++ b/include/medium/dim2/elastic/isotropic/stress_integrand.tpp @@ -26,22 +26,22 @@ specfem::medium::impl_compute_stress_integrands( // P_SV case // sigma_xx sigma_xx = - properties.lambdaplus2mu * du(0, 0) + properties.lambda * du(1, 1); + properties.lambdaplus2mu() * du(0, 0) + properties.lambda() * du(1, 1); // sigma_zz sigma_zz = - properties.lambdaplus2mu * du(1, 1) + properties.lambda * du(0, 0); + properties.lambdaplus2mu() * du(1, 1) + properties.lambda() * du(0, 0); // sigma_xz - sigma_xz = properties.mu * (du(0, 1) + du(1, 0)); + sigma_xz = properties.mu() * (du(0, 1) + du(1, 0)); } else if (specfem::globals::simulation_wave == specfem::wave::sh) { // SH-case // sigma_xx - sigma_xx = properties.mu * du(0, 0); // would be sigma_xy in + sigma_xx = properties.mu() * du(0, 0); // would be sigma_xy in // CPU-version // sigma_xz - sigma_xz = properties.mu * du(1, 0); // sigma_zy + sigma_xz = properties.mu() * du(1, 0); // sigma_zy } specfem::datatype::VectorPointViewType F; diff --git a/include/medium/dim2/elastic/isotropic/wavefield.hpp b/include/medium/dim2/elastic/isotropic/wavefield.hpp index 5deb6b24..7f4da876 100644 --- a/include/medium/dim2/elastic/isotropic/wavefield.hpp +++ b/include/medium/dim2/elastic/isotropic/wavefield.hpp @@ -80,7 +80,7 @@ KOKKOS_FUNCTION void impl_compute_wavefield( // -1.0 * (sigma_xx + sigma_zz + sigma_yy) / 3.0; wavefield(iterator_index.ielement, index.iz, index.ix, 0) = -1.0 * - ((point_property.lambda + (2.0 / 3.0) * point_property.mu) * + ((point_property.lambda() + (2.0 / 3.0) * point_property.mu()) * (du(0, 0) + du(1, 1))); }); diff --git a/include/medium/properties_container.hpp b/include/medium/properties_container.hpp index 2b3c9d89..3fc831ce 100644 --- a/include/medium/properties_container.hpp +++ b/include/medium/properties_container.hpp @@ -2,9 +2,166 @@ #include "enumerations/medium.hpp" +#define DEFINE_CONTAINER(prop) \ + constexpr static int i_##prop = __COUNTER__ - _counter - 1; \ + KOKKOS_INLINE_FUNCTION type_real &prop(const int &ispec, const int &iz, \ + const int &ix) const { \ + return Base::data(ispec, iz, ix, i_##prop); \ + } \ + KOKKOS_INLINE_FUNCTION type_real &h_##prop(const int &ispec, const int &iz, \ + const int &ix) const { \ + return Base::h_data(ispec, iz, ix, i_##prop); \ + } + namespace specfem { namespace medium { +namespace impl { +template +struct impl_properties_container { + using view_type = typename Kokkos::View; + constexpr static auto nprops = N; + constexpr static auto dimension = specfem::dimension::type::dim2; + constexpr static auto medium_tag = MediumTag; + constexpr static auto property_tag = PropertyTag; + + int nspec; ///< total number of acoustic spectral elements + int ngllz; ///< number of quadrature points in z dimension + int ngllx; ///< number of quadrature points in x dimension + + view_type data; + view_type::HostMirror h_data; + + impl_properties_container() = default; + + impl_properties_container(const int nspec, const int ngllz, const int ngllx) + : nspec(nspec), ngllz(ngllz), ngllx(ngllx), + data("specfem::benchmarks::properties::data", nspec, ngllz, ngllx, N), + h_data(Kokkos::create_mirror_view(data)) {} + +private: + template + KOKKOS_FORCEINLINE_FUNCTION void + load_properties(const specfem::point::index &index, + PointProperties &property, const ViewType &target) const { + + static_assert(PointProperties::dimension == dimension, + "Dimension mismatch"); + static_assert(PointProperties::medium_tag == medium_tag, + "Medium tag mismatch"); + static_assert(PointProperties::property_tag == property_tag, + "Property tag mismatch"); + + const int ispec = index.ispec; + const int iz = index.iz; + const int ix = index.ix; + + for (int i = 0; i < nprops; i++) { + property.data[i] = target(ispec, iz, ix, i); + } + + property.compute(); + } + template + KOKKOS_FORCEINLINE_FUNCTION void + load_properties(const specfem::point::simd_index &index, + PointProperties &property, const ViewType &target) const { + + static_assert(PointProperties::dimension == dimension, + "Dimension mismatch"); + static_assert(PointProperties::medium_tag == medium_tag, + "Medium tag mismatch"); + static_assert(PointProperties::property_tag == property_tag, + "Property tag mismatch"); + + using simd = typename PointProperties::simd; + using mask_type = typename simd::mask_type; + using tag_type = typename simd::tag_type; + + const int ispec = index.ispec; + const int iz = index.iz; + const int ix = index.ix; + + mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); + + for (int i = 0; i < nprops; i++) { + Kokkos::Experimental::where(mask, property.data[i]) + .copy_from(&target(ispec, iz, ix, i), tag_type()); + } + + property.compute(); + } + +public: + template + KOKKOS_FORCEINLINE_FUNCTION void + load_device_properties(const IndexType &index, + PointProperties &property) const { + load_properties(index, property, data); + } + + template + KOKKOS_FORCEINLINE_FUNCTION void + load_host_properties(const IndexType &index, + PointProperties &property) const { + load_properties(index, property, h_data); + } + + template + inline void assign(const specfem::point::index &index, + const PointProperties &property) const { + + static_assert(PointProperties::dimension == dimension, + "Dimension mismatch"); + static_assert(PointProperties::medium_tag == medium_tag, + "Medium tag mismatch"); + static_assert(PointProperties::property_tag == property_tag, + "Property tag mismatch"); + + const int ispec = index.ispec; + const int iz = index.iz; + const int ix = index.ix; + + for (int i = 0; i < nprops; i++) { + h_data(ispec, iz, ix, i) = property.data[i]; + } + } + + template + inline void assign(const specfem::point::simd_index &index, + const PointProperties &property) const { + + static_assert(PointProperties::dimension == dimension, + "Dimension mismatch"); + static_assert(PointProperties::medium_tag == medium_tag, + "Medium tag mismatch"); + static_assert(PointProperties::property_tag == property_tag, + "Property tag mismatch"); + + using simd = typename PointProperties::simd; + using mask_type = typename simd::mask_type; + using tag_type = typename simd::tag_type; + + const int ispec = index.ispec; + const int iz = index.iz; + const int ix = index.ix; + + mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); + + for (int i = 0; i < nprops; i++) { + Kokkos::Experimental::where(mask, property.data[i]) + .copy_to(&h_data(ispec, iz, ix, i), tag_type()); + } + } + + void copy_to_device() { Kokkos::deep_copy(data, h_data); } + + void copy_to_host() { Kokkos::deep_copy(h_data, data); } +}; +} // namespace impl + template struct properties_container { @@ -19,3 +176,5 @@ struct properties_container { #include "dim2/acoustic/isotropic/properties_container.hpp" #include "dim2/elastic/anisotropic/properties_container.hpp" #include "dim2/elastic/isotropic/properties_container.hpp" + +#undef DEFINE_CONTAINER diff --git a/include/point/properties.hpp b/include/point/properties.hpp index a6409523..fab5902d 100644 --- a/include/point/properties.hpp +++ b/include/point/properties.hpp @@ -1,5 +1,4 @@ -#ifndef _POINT_PROPERTIES_HPP -#define _POINT_PROPERTIES_HPP +#pragma once #include "datatypes/simd.hpp" #include "enumerations/medium.hpp" @@ -7,6 +6,72 @@ namespace specfem { namespace point { +#define DEFINE_PROP(prop) \ + constexpr static int i_##prop = __COUNTER__ - _counter - 1; \ + KOKKOS_INLINE_FUNCTION value_type prop() const { \ + return Base::data[i_##prop]; \ + } \ + KOKKOS_INLINE_FUNCTION void prop(value_type val) { \ + Base::data[i_##prop] = val; \ + } + +namespace impl { +template struct impl_properties { + using simd = specfem::datatype::simd; ///< SIMD type + using value_type = typename simd::datatype; + constexpr static bool is_point_properties = true; + constexpr static auto nprops = N; + constexpr static auto nprops_extra = N_EX; + + value_type data[N_EX]; + + KOKKOS_FUNCTION + impl_properties() = default; + + /** + * @brief array constructor + * + */ + KOKKOS_FUNCTION + impl_properties(const value_type *value) { + for (int i = 0; i < N; ++i) { + data[i] = value[i]; + } + } + + /** + * @brief value constructor + * + */ + template = 0> + impl_properties(Args... args) : data{ args... } {} + + /** + * @brief Equality operator + * + */ + KOKKOS_FUNCTION + bool operator==(const impl_properties &rhs) const { + for (int i = 0; i < N_EX; ++i) { + if (data[i] != rhs.data[i]) { + return false; + } + } + return true; + } + + /** + * @brief Inequality operator + * + */ + KOKKOS_FUNCTION + bool operator!=(const impl_properties &rhs) const { + return !(*this == rhs); + } +}; +} // namespace impl + /** * @brief Store properties of the medium at a quadrature point * @@ -28,216 +93,110 @@ struct properties; template struct properties { + specfem::element::property_tag::isotropic, UseSIMD> + : public impl::impl_properties<3, 6, UseSIMD> { /** * @name Typedefs * */ ///@{ - constexpr static bool is_point_properties = true; - using simd = specfem::datatype::simd; ///< SIMD type + using Base = impl::impl_properties<3, 6, UseSIMD>; constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::elastic; constexpr static auto property_tag = specfem::element::property_tag::isotropic; - using value_type = - typename simd::datatype; ///< Value type to store properties - ///@} - value_type mu; ///< shear modulus @f$ \mu @f$ - value_type rho; ///< density @f$ \rho @f$ - - value_type rho_vp; ///< P-wave velocity @f$ \rho v_p @f$ - value_type rho_vs; ///< S-wave velocity @f$ \rho v_s @f$ - value_type lambda; ///< Lame's parameter @f$ \lambda @f$ - value_type lambdaplus2mu; ///< Lame's parameter @f$ \lambda + 2\mu @f$ - -private: - KOKKOS_FUNCTION - properties(const value_type &lambdaplus2mu, const value_type &mu, - const value_type &rho, std::false_type) - : lambdaplus2mu(lambdaplus2mu), mu(mu), rho(rho), - rho_vp(sqrt(rho * lambdaplus2mu)), rho_vs(sqrt(rho * mu)), - lambda(lambdaplus2mu - 2.0 * mu) {} - - KOKKOS_FUNCTION - properties(const value_type &lambdaplus2mu, const value_type &mu, - const value_type &rho, std::true_type) - : lambdaplus2mu(lambdaplus2mu), mu(mu), rho(rho), - rho_vp(Kokkos::sqrt(rho * lambdaplus2mu)), - rho_vs(Kokkos::sqrt(rho * mu)), - lambda(lambdaplus2mu - (static_cast(2.0)) * mu) {} - -public: - /** - * @name Constructors - * - */ - ///@{ - /** - * @brief Default constructor - * - */ + using simd = specfem::datatype::simd; + using value_type = typename simd::datatype; + constexpr static int _counter = __COUNTER__; + ///@} KOKKOS_FUNCTION properties() = default; - /** - * @brief Construct a new properties object - * - * @param lambdaplus2mu @f$ \lambda + 2\mu @f$ - * @param mu @f$ \mu @f$ - * @param rho @f$ \rho @f$ - */ - KOKKOS_FUNCTION - properties(const value_type &lambdaplus2mu, const value_type &mu, - const value_type &rho) - : properties(lambdaplus2mu, mu, rho, - std::integral_constant{}) {} - - /** - * @brief single value constructor - * - */ KOKKOS_FUNCTION - properties(const value_type value) - : properties(value, value, value, - std::integral_constant{}) {} - ///@} + properties(const value_type *value) : Base(value) { compute(); } - /** - * @brief Equality operator - * - */ - KOKKOS_FUNCTION - bool operator==(const properties &rhs) const { - return rho == rhs.rho && mu == rhs.mu && lambdaplus2mu == rhs.lambdaplus2mu; + template = 0> + KOKKOS_FUNCTION properties(Args... args) : Base(args...) { + compute(); } - /** - * @brief Inequality operator - * - */ - KOKKOS_FUNCTION - bool operator!=(const properties &rhs) const { return !(*this == rhs); } + DEFINE_PROP(lambdaplus2mu) ///< Lame's parameter @f$ \lambda + 2\mu @f$ + DEFINE_PROP(mu) ///< shear modulus @f$ \mu @f$ + DEFINE_PROP(rho) ///< density @f$ \rho @f$ - KOKKOS_FUNCTION - bool operator==(const value_type value) { - return rho == value && mu == value && lambdaplus2mu == value; - } + DEFINE_PROP(rho_vp) ///< P-wave velocity @f$ \rho v_p @f$ + DEFINE_PROP(rho_vs) ///< S-wave velocity @f$ \rho v_s @f$ + DEFINE_PROP(lambda) ///< Lame's parameter @f$ \lambda @f$ - KOKKOS_FUNCTION - bool operator!=(const value_type value) { return !(*this == value); } + KOKKOS_INLINE_FUNCTION + void compute() { + rho_vp(Kokkos::sqrt(rho() * lambdaplus2mu())); + rho_vs(Kokkos::sqrt(rho() * mu())); + lambda(lambdaplus2mu() - (static_cast(2.0)) * mu()); + } }; template struct properties { + specfem::element::property_tag::anisotropic, UseSIMD> + : public impl::impl_properties<10, 12, UseSIMD> { /** * @name Typedefs * */ ///@{ - constexpr static bool is_point_properties = true; - using simd = specfem::datatype::simd; ///< SIMD type + using Base = impl::impl_properties<10, 12, UseSIMD>; constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::elastic; constexpr static auto property_tag = specfem::element::property_tag::anisotropic; - using value_type = - typename simd::datatype; ///< Value type to store properties - ///@} - /** - * @name Elastic constants - * - */ - ///@{ - value_type c11; ///< @f$ c_{11} @f$ - value_type c13; ///< @f$ c_{13} @f$ - value_type c15; ///< @f$ c_{15} @f$ - value_type c33; ///< @f$ c_{33} @f$ - value_type c35; ///< @f$ c_{35} @f$ - value_type c55; ///< @f$ c_{55} @f$ - value_type c12; ///< @f$ c_{12} @f$ - value_type c23; ///< @f$ c_{23} @f$ - value_type c25; ///< @f$ c_{25} @f$ + using simd = specfem::datatype::simd; + using value_type = typename simd::datatype; + constexpr static int _counter = __COUNTER__; ///@} - - value_type rho; ///< Density @f$ \rho @f$ - value_type rho_vp; ///< P-wave velocity @f$ \rho v_p @f$ - value_type rho_vs; ///< S-wave velocity @f$ \rho v_s @f$ - -private: - KOKKOS_FUNCTION - properties(const value_type &c11, const value_type &c13, - const value_type &c15, const value_type &c33, - const value_type &c35, const value_type &c55, - const value_type &c12, const value_type &c23, - const value_type &c25, const value_type &rho, std::false_type) - : c11(c11), c13(c13), c15(c15), c33(c33), c35(c35), c55(c55), c12(c12), - c23(c23), c25(c25), rho(rho), rho_vp(sqrt(rho * c33)), - rho_vs(sqrt(rho * c55)) {} - - KOKKOS_FUNCTION - properties(const value_type &c11, const value_type &c13, - const value_type &c15, const value_type &c33, - const value_type &c35, const value_type &c55, - const value_type &c12, const value_type &c23, - const value_type &c25, const value_type &rho, std::true_type) - : c11(c11), c13(c13), c15(c15), c33(c33), c35(c35), c55(c55), c12(c12), - c23(c23), c25(c25), rho(rho), rho_vp(Kokkos::sqrt(rho * c33)), - rho_vs(Kokkos::sqrt(rho * c55)) {} - -public: KOKKOS_FUNCTION properties() = default; KOKKOS_FUNCTION - properties(const value_type &c11, const value_type &c13, - const value_type &c15, const value_type &c33, - const value_type &c35, const value_type &c55, - const value_type &c12, const value_type &c23, - const value_type &c25, const type_real &rho) - : properties(c11, c13, c15, c33, c35, c55, c12, c23, c25, rho, - std::integral_constant{}) {} - - /** - * @brief single value constructor - * - */ - KOKKOS_FUNCTION - properties(const value_type value) - : properties(value, value, value, value, value, value, value, value, - value, value, std::integral_constant{}) {} + properties(const value_type *value) : Base(value) { compute(); } - /** - * @brief Equality operator - * - */ - KOKKOS_FUNCTION - bool operator==(const properties &rhs) const { - return rho == rhs.rho && c11 == rhs.c11 && c13 == rhs.c13 && - c15 == rhs.c15 && c33 == rhs.c33 && c35 == rhs.c35 && c55 == rhs.c55; + template = 0> + KOKKOS_FUNCTION properties(Args... args) : Base(args...) { + compute(); } /** - * @brief Inequality operator + * @name Elastic constants * */ - KOKKOS_FUNCTION - bool operator!=(const properties &rhs) const { return !(*this == rhs); } + ///@{ + DEFINE_PROP(c11) ///< @f$ c_{11} @f$ + DEFINE_PROP(c13) ///< @f$ c_{13} @f$ + DEFINE_PROP(c15) ///< @f$ c_{15} @f$ + DEFINE_PROP(c33) ///< @f$ c_{33} @f$ + DEFINE_PROP(c35) ///< @f$ c_{35} @f$ + DEFINE_PROP(c55) ///< @f$ c_{55} @f$ + DEFINE_PROP(c12) ///< @f$ c_{12} @f$ + DEFINE_PROP(c23) ///< @f$ c_{23} @f$ + DEFINE_PROP(c25) ///< @f$ c_{25} @f$ + ///@} - KOKKOS_FUNCTION - bool operator==(const value_type value) { - return rho == value && c11 == value && c13 == value && c15 == value && - c33 == value && c35 == value && c55 == value; - } + DEFINE_PROP(rho) ///< Density @f$ \rho @f$ + DEFINE_PROP(rho_vp) ///< P-wave velocity @f$ \rho v_p @f$ + DEFINE_PROP(rho_vs) ///< S-wave velocity @f$ \rho v_s @f$ - KOKKOS_FUNCTION - bool operator!=(const value_type value) { return !(*this == value); } + KOKKOS_INLINE_FUNCTION + void compute() { + rho_vp(Kokkos::sqrt(rho() * c33())); + rho_vs(Kokkos::sqrt(rho() * c55())); + } }; /** @@ -248,103 +207,50 @@ struct properties struct properties { + specfem::element::property_tag::isotropic, UseSIMD> + : public impl::impl_properties<2, 4, UseSIMD> { /** * @name Typedefs * */ ///@{ - constexpr static bool is_point_properties = true; - using simd = specfem::datatype::simd; ///< SIMD type + using Base = impl::impl_properties<2, 4, UseSIMD>; constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::acoustic; constexpr static auto property_tag = specfem::element::property_tag::isotropic; - using value_type = - typename simd::datatype; ///< Value type to store properties + using simd = specfem::datatype::simd; + using value_type = typename simd::datatype; + constexpr static int _counter = __COUNTER__; ///@} - - value_type kappa_inverse; ///< @f$ \frac{1}{\lambda + 2\mu} @f$ - value_type rho_inverse; ///< @f$ \frac{1}{\rho} @f$ - value_type kappa; ///< Bulk modulus @f$ \kappa @f$ - - value_type rho_vpinverse; ///< @f$ \frac{1}{\rho v_p} @f$ - -private: - KOKKOS_FUNCTION - properties(const value_type &rho_inverse, const value_type &kappa, - std::false_type) - : kappa_inverse(1.0 / kappa), rho_inverse(rho_inverse), kappa(kappa), - rho_vpinverse(sqrt(rho_inverse * kappa_inverse)) {} - - KOKKOS_FUNCTION - properties(const value_type &rho_inverse, const value_type &kappa, - std::true_type) - : kappa_inverse((static_cast(1.0)) / kappa), - rho_inverse(rho_inverse), kappa(kappa), - rho_vpinverse(Kokkos::sqrt(rho_inverse * kappa_inverse)) {} - -public: - /** - * @name Constructors - * - */ - ///@{ - - /** - * @brief Default constructor - * - */ KOKKOS_FUNCTION properties() = default; - /** - * @brief Construct a new properties object - * - * @param rho_inverse @f$ \frac{1}{\rho} @f$ - * @param kappa Bulk modulus @f$ \kappa @f$ - */ - KOKKOS_FUNCTION - properties(const value_type &rho_inverse, const value_type &kappa) - : properties(rho_inverse, kappa, - std::integral_constant{}) {} - /** - * @brief single value constructor - * - */ KOKKOS_FUNCTION - properties(const value_type value) - : properties(value, value, std::integral_constant{}) {} - ///@} + properties(const value_type *value) : Base(value) { compute(); } - /** - * @brief Equality operator - * - */ - KOKKOS_FUNCTION - bool operator==(const properties &rhs) const { - return rho_inverse == rhs.rho_inverse && kappa == rhs.kappa; + template = 0> + KOKKOS_FUNCTION properties(Args... args) : Base(args...) { + compute(); } - /** - * @brief Inequality operator - * - */ - KOKKOS_FUNCTION - bool operator!=(const properties &rhs) const { return !(*this == rhs); } + DEFINE_PROP(rho_inverse) ///< @f$ \frac{1}{\rho} @f$ + DEFINE_PROP(kappa) ///< Bulk modulus @f$ \kappa @f$ + DEFINE_PROP(kappa_inverse) ///< @f$ \frac{1}{\lambda + 2\mu} @f$ + DEFINE_PROP(rho_vpinverse) ///< @f$ \frac{1}{\rho v_p} @f$ - KOKKOS_FUNCTION - bool operator==(const value_type value) { - return rho_inverse == value && kappa == value; + KOKKOS_INLINE_FUNCTION + void compute() { + kappa_inverse((static_cast(1.0)) / kappa()); + rho_vpinverse(Kokkos::sqrt(rho_inverse() * kappa_inverse())); } - - KOKKOS_FUNCTION - bool operator!=(const value_type value) { return !(*this == value); } }; +#undef DEFINE_PROP_CONSTRUCTORS +#undef DEFINE_PROP + } // namespace point } // namespace specfem - -#endif diff --git a/tests/unit-tests/assembly/compute_wavefield/test_helper.hpp b/tests/unit-tests/assembly/compute_wavefield/test_helper.hpp index 52d45d06..58ad19e6 100644 --- a/tests/unit-tests/assembly/compute_wavefield/test_helper.hpp +++ b/tests/unit-tests/assembly/compute_wavefield/test_helper.hpp @@ -111,7 +111,7 @@ class test_helper 1.0e-4) { diff --git a/tests/unit-tests/assembly/properties/properties.cpp b/tests/unit-tests/assembly/properties/properties.cpp index c047bb02..abe828ac 100644 --- a/tests/unit-tests/assembly/properties/properties.cpp +++ b/tests/unit-tests/assembly/properties/properties.cpp @@ -38,9 +38,9 @@ std::string get_error_message( std::ostringstream message; error_message_header(message, value, mode); - message << "\t\trho = " << point_property.rho << "\n"; - message << "\t\tmu = " << point_property.mu << "\n"; - message << "\t\tkappa = " << point_property.lambdaplus2mu << "\n"; + message << "\t\trho = " << point_property.rho() << "\n"; + message << "\t\tmu = " << point_property.mu() << "\n"; + message << "\t\tlambdaplus2mu = " << point_property.lambdaplus2mu() << "\n"; return message.str(); } @@ -54,13 +54,13 @@ std::string get_error_message( std::ostringstream message; error_message_header(message, value, mode); - message << "\t\trho = " << point_property.rho << "\n"; - message << "\t\tc11 = " << point_property.c11 << "\n"; - message << "\t\tc13 = " << point_property.c13 << "\n"; - message << "\t\tc15 = " << point_property.c15 << "\n"; - message << "\t\tc33 = " << point_property.c33 << "\n"; - message << "\t\tc35 = " << point_property.c35 << "\n"; - message << "\t\tc55 = " << point_property.c55 << "\n"; + message << "\t\trho = " << point_property.rho() << "\n"; + message << "\t\tc11 = " << point_property.c11() << "\n"; + message << "\t\tc13 = " << point_property.c13() << "\n"; + message << "\t\tc15 = " << point_property.c15() << "\n"; + message << "\t\tc33 = " << point_property.c33() << "\n"; + message << "\t\tc35 = " << point_property.c35() << "\n"; + message << "\t\tc55 = " << point_property.c55() << "\n"; return message.str(); } @@ -74,8 +74,8 @@ std::string get_error_message( std::ostringstream message; error_message_header(message, value, mode); - message << "\t\trho_inverse = " << point_property.rho_inverse << "\n"; - message << "\t\tkappa = " << point_property.kappa << "\n"; + message << "\t\trho_inverse = " << point_property.rho_inverse() << "\n"; + message << "\t\tkappa = " << point_property.kappa() << "\n"; return message.str(); } @@ -112,10 +112,10 @@ get_point_property(const int ispec, const int iz, const int ix, specfem::element::property_tag::isotropic, false> point_property; - point_property.rho = elastic_isotropic.h_rho(ispec_l, iz, ix); - point_property.mu = elastic_isotropic.h_mu(ispec_l, iz, ix); - point_property.lambdaplus2mu = - elastic_isotropic.h_lambdaplus2mu(ispec_l, iz, ix); + point_property.rho(elastic_isotropic.h_rho(ispec_l, iz, ix)); + point_property.mu(elastic_isotropic.h_mu(ispec_l, iz, ix)); + point_property.lambdaplus2mu( + elastic_isotropic.h_lambdaplus2mu(ispec_l, iz, ix)); return point_property; } @@ -134,9 +134,9 @@ get_point_property( specfem::element::property_tag::isotropic, false> point_property_l; - point_property_l.rho = point_property.rho[lane]; - point_property_l.mu = point_property.mu[lane]; - point_property_l.lambdaplus2mu = point_property.lambdaplus2mu[lane]; + point_property_l.rho(point_property.rho()[lane]); + point_property_l.mu(point_property.mu()[lane]); + point_property_l.lambdaplus2mu(point_property.lambdaplus2mu()[lane]); return point_property_l; } @@ -157,13 +157,16 @@ get_point_property(const int ispec, const int iz, const int ix, specfem::element::property_tag::anisotropic, false> point_property; - point_property.rho = elastic_anisotropic.h_rho(ispec_l, iz, ix); - point_property.c11 = elastic_anisotropic.h_c11(ispec_l, iz, ix); - point_property.c13 = elastic_anisotropic.h_c13(ispec_l, iz, ix); - point_property.c15 = elastic_anisotropic.h_c15(ispec_l, iz, ix); - point_property.c33 = elastic_anisotropic.h_c33(ispec_l, iz, ix); - point_property.c35 = elastic_anisotropic.h_c35(ispec_l, iz, ix); - point_property.c55 = elastic_anisotropic.h_c55(ispec_l, iz, ix); + point_property.rho(elastic_anisotropic.h_rho(ispec_l, iz, ix)); + point_property.c11(elastic_anisotropic.h_c11(ispec_l, iz, ix)); + point_property.c13(elastic_anisotropic.h_c13(ispec_l, iz, ix)); + point_property.c15(elastic_anisotropic.h_c15(ispec_l, iz, ix)); + point_property.c33(elastic_anisotropic.h_c33(ispec_l, iz, ix)); + point_property.c35(elastic_anisotropic.h_c35(ispec_l, iz, ix)); + point_property.c55(elastic_anisotropic.h_c55(ispec_l, iz, ix)); + point_property.c12(elastic_anisotropic.h_c12(ispec_l, iz, ix)); + point_property.c23(elastic_anisotropic.h_c23(ispec_l, iz, ix)); + point_property.c25(elastic_anisotropic.h_c25(ispec_l, iz, ix)); return point_property; } @@ -182,13 +185,16 @@ get_point_property( specfem::element::property_tag::anisotropic, false> point_property_l; - point_property_l.rho = point_property.rho[lane]; - point_property_l.c11 = point_property.c11[lane]; - point_property_l.c13 = point_property.c13[lane]; - point_property_l.c15 = point_property.c15[lane]; - point_property_l.c33 = point_property.c33[lane]; - point_property_l.c35 = point_property.c35[lane]; - point_property_l.c55 = point_property.c55[lane]; + point_property_l.rho(point_property.rho()[lane]); + point_property_l.c11(point_property.c11()[lane]); + point_property_l.c13(point_property.c13()[lane]); + point_property_l.c15(point_property.c15()[lane]); + point_property_l.c33(point_property.c33()[lane]); + point_property_l.c35(point_property.c35()[lane]); + point_property_l.c55(point_property.c55()[lane]); + point_property_l.c12(point_property.c12()[lane]); + point_property_l.c23(point_property.c23()[lane]); + point_property_l.c25(point_property.c25()[lane]); return point_property_l; } @@ -209,9 +215,8 @@ get_point_property(const int ispec, const int iz, const int ix, specfem::element::property_tag::isotropic, false> point_property; - point_property.rho_inverse = - acoustic_isotropic.h_rho_inverse(ispec_l, iz, ix); - point_property.kappa = acoustic_isotropic.h_kappa(ispec_l, iz, ix); + point_property.rho_inverse(acoustic_isotropic.h_rho_inverse(ispec_l, iz, ix)); + point_property.kappa(acoustic_isotropic.h_kappa(ispec_l, iz, ix)); return point_property; } @@ -230,8 +235,8 @@ get_point_property( specfem::element::property_tag::isotropic, false> point_property_l; - point_property_l.rho_inverse = point_property.rho_inverse[lane]; - point_property_l.kappa = point_property.kappa[lane]; + point_property_l.rho_inverse(point_property.rho_inverse()[lane]); + point_property_l.kappa(point_property.kappa()[lane]); return point_property_l; } @@ -240,13 +245,13 @@ template void check_eq( const typename specfem::datatype::simd::datatype &p1, const typename specfem::datatype::simd::datatype &p2, - const int &n_simd_elements) { + const int &n_simd_elements, const std::string &msg) { if constexpr (using_simd) { for (int i = 0; i < n_simd_elements; i++) { if (p1[i] != p2[i]) { std::ostringstream message; - message << "\n \t Error in function load_on_host"; + message << "\n \t Error in function load_on_host (SIMD) | " << msg; message << "\n\t Expected: " << p1[i]; message << "\n\t Got: " << p2[i]; @@ -257,7 +262,7 @@ void check_eq( if (p1 != p2) { std::ostringstream message; - message << "\n \t Error in function load_on_host"; + message << "\n \t Error in function load_on_host | " << msg; message << "\n\t Expected: " << p1; message << "\n\t Got: " << p2; @@ -284,19 +289,20 @@ void check_point_properties( specfem::dimension::type::dim2, specfem::element::medium_tag::elastic, specfem::element::property_tag::isotropic, using_simd> &p2, const int &n_simd_elements) { - check_eq(p1.rho, p2.rho, n_simd_elements); - check_eq(p1.mu, p2.mu, n_simd_elements); - check_eq(p1.lambdaplus2mu, p2.lambdaplus2mu, n_simd_elements); - check_eq(p1.lambda, - p2.lambdaplus2mu - + check_eq(p1.rho(), p2.rho(), n_simd_elements, "rho"); + check_eq(p1.mu(), p2.mu(), n_simd_elements, ".mu"); + check_eq(p1.lambdaplus2mu(), p2.lambdaplus2mu(), n_simd_elements, + "lambdaplus2mu"); + check_eq(p1.lambda(), + p2.lambdaplus2mu() - (static_cast::datatype>(2.0)) * - p2.mu, - n_simd_elements); - check_eq(p1.rho_vp, Kokkos::sqrt(p2.rho * p2.lambdaplus2mu), - n_simd_elements); - check_eq(p1.rho_vs, Kokkos::sqrt(p2.rho * p2.mu), - n_simd_elements); + p2.mu(), + n_simd_elements, "lambda"); + check_eq(p1.rho_vp(), Kokkos::sqrt(p2.rho() * p2.lambdaplus2mu()), + n_simd_elements, "rho_vp"); + check_eq(p1.rho_vs(), Kokkos::sqrt(p2.rho() * p2.mu()), + n_simd_elements, "rho_vp"); } template @@ -308,17 +314,17 @@ void check_point_properties( specfem::dimension::type::dim2, specfem::element::medium_tag::elastic, specfem::element::property_tag::anisotropic, using_simd> &p2, const int &n_simd_elements) { - check_eq(p1.rho, p2.rho, n_simd_elements); - check_eq(p1.c11, p2.c11, n_simd_elements); - check_eq(p1.c13, p2.c13, n_simd_elements); - check_eq(p1.c15, p2.c15, n_simd_elements); - check_eq(p1.c33, p2.c33, n_simd_elements); - check_eq(p1.c35, p2.c35, n_simd_elements); - check_eq(p1.c55, p2.c55, n_simd_elements); - check_eq(p1.rho_vp, Kokkos::sqrt(p2.rho * p2.c33), - n_simd_elements); - check_eq(p1.rho_vs, Kokkos::sqrt(p2.rho * p2.c55), - n_simd_elements); + check_eq(p1.rho(), p2.rho(), n_simd_elements, "rho"); + check_eq(p1.c11(), p2.c11(), n_simd_elements, "c11"); + check_eq(p1.c13(), p2.c13(), n_simd_elements, "c13"); + check_eq(p1.c15(), p2.c15(), n_simd_elements, "c15"); + check_eq(p1.c33(), p2.c33(), n_simd_elements, "c33"); + check_eq(p1.c35(), p2.c35(), n_simd_elements, "c35"); + check_eq(p1.c55(), p2.c55(), n_simd_elements, "c55"); + check_eq(p1.rho_vp(), Kokkos::sqrt(p2.rho() * p2.c33()), + n_simd_elements, "rho_vp"); + check_eq(p1.rho_vs(), Kokkos::sqrt(p2.rho() * p2.c55()), + n_simd_elements, "rho_vs"); } template @@ -330,18 +336,23 @@ void check_point_properties( specfem::dimension::type::dim2, specfem::element::medium_tag::acoustic, specfem::element::property_tag::isotropic, using_simd> &p2, const int &n_simd_elements) { - check_eq(p1.rho_inverse, p2.rho_inverse, n_simd_elements); - check_eq(p1.kappa, p2.kappa, n_simd_elements); + check_eq(p1.rho_inverse(), p2.rho_inverse(), n_simd_elements, + "rho_inverse"); + check_eq(p1.kappa(), p2.kappa(), n_simd_elements, "kappa"); check_eq( - p1.kappa_inverse, + p1.kappa_inverse(), (static_cast< typename specfem::datatype::simd::datatype>( 1.0)) / - p2.kappa, - n_simd_elements); - check_eq(p1.rho_vpinverse, - Kokkos::sqrt(p2.rho_inverse * p2.kappa_inverse), - n_simd_elements); + p2.kappa(), + n_simd_elements, "kappa_inverse"); + check_eq( + p1.rho_vpinverse(), + Kokkos::sqrt(p2.rho_inverse() * + (static_cast::datatype>(1.0) / + p2.kappa())), + n_simd_elements, "rho_vpinverse"); } template ; + constexpr int simd_size = specfem::datatype::simd::size(); @@ -379,14 +394,19 @@ void check_to_value(const specfem::compute::properties properties, get_point_property(ielement + j, iz, ix, properties); const type_real value = values_to_store(i); - if (point_property != value) { - std::ostringstream message; + for (int l = 0; l < PointType::nprops; l++) { + if (point_property.data[l] != value) { + std::ostringstream message; - message << "\n \t Error at ispec = " << ielement + j - << ", iz = " << iz << ", ix = " << ix; - message << get_error_message(point_property, value); + message << "\n \t Error in function check_to_value"; - throw std::runtime_error(message.str()); + message << "\n \t Error at ispec = " << ielement + j + << ", iz = " << iz << ", ix = " << ix + << ", iprop = " << l; + message << get_error_message(point_property, value); + + throw std::runtime_error(message.str()); + } } } } @@ -411,6 +431,9 @@ void check_compute_to_mesh( const int ngllz = properties.ngllz; std::vector elements; + using PointType = specfem::point::properties; + for (int ispec = 0; ispec < nspec; ispec++) { if ((element_types.get_medium_tag(ispec) == MediumTag) && (element_types.get_property_tag(ispec) == PropertyTag)) { @@ -445,17 +468,31 @@ void check_compute_to_mesh( std::get >( materials[ispec_mesh]); auto value = material.get_properties(); - if (point_property != value) { - std::ostringstream message; + for (int l = 0; l < PointType::nprops; l++) { + if (point_property.data[l] != value.data[l]) { + std::ostringstream message; - message << "\n \t Error at ispec = " << ielement << ", iz = " << iz - << ", ix = " << ix; + message << "\n \t Error in function check_compute_to_mesh"; - message << get_error_message(value, 0.0, 1); - message << get_error_message(point_property, 0.0, 2); + message << "\n \t Error at ispec = " << ielement << ", iz = " << iz + << ", ix = " << ix << ", iprop = " << l; + message << get_error_message(value, 0.0, 1); + message << get_error_message(point_property, 0.0, 2); - throw std::runtime_error(message.str()); + throw std::runtime_error(message.str()); + } } + // if (point_property != value) { + // std::ostringstream message; + + // message << "\n \t Error at ispec = " << ielement << ", iz = " << iz + // << ", ix = " << ix; + + // message << get_error_message(value, 0.0, 1); + // message << get_error_message(point_property, 0.0, 2); + + // throw std::runtime_error(message.str()); + // } } } } @@ -515,11 +552,14 @@ void check_store_on_host(specfem::compute::properties &properties, const auto index = get_index(ielement, n_simd_elements, iz, ix); const type_real value = values_to_store_h(i); - PointType point(value); + PointType point; + for (int l = 0; l < PointType::nprops; l++) { + point.data[l] = value; + } PointType point_loaded; specfem::compute::store_on_host(index, properties, point); specfem::compute::load_on_host(index, properties, point_loaded); - check_point_properties(point, point_loaded, + check_point_properties(point_loaded, point, n_simd_elements); } } @@ -614,28 +654,33 @@ void check_load_on_device(specfem::compute::properties &properties, for (int lane = 0; lane < n_simd_elements; lane++) { const auto point_property_l = get_point_property(lane, point_property); - if (point_property_l != value_l) { - std::ostringstream message; + for (int l = 0; l < PointType::nprops; l++) { + if (point_property_l.data[l] != value_l) { + std::ostringstream message; - message << "\n \t Error in function load_on_device"; + message << "\n \t Error in function load_on_device"; - message << "\n \t Error at ispec = " << ielement << ", iz = " << 0 - << ", ix = " << 0; - message << get_error_message(point_property_l, value_l); + message << "\n \t Error at ispec = " << ielement + << ", iz = " << 0 << ", ix = " << 0 + << ", iprop = " << l; + message << get_error_message(point_property_l, value_l); - throw std::runtime_error(message.str()); + throw std::runtime_error(message.str()); + } } } } else if constexpr (!using_simd) { - if (point_property != value_l) { - std::ostringstream message; - message << "\n \t Error in function load_on_device"; + for (int l = 0; l < PointType::nprops; l++) { + if (point_property.data[l] != value_l) { + std::ostringstream message; + message << "\n \t Error in function load_on_device"; - message << "\n \t Error at ispec = " << ielement << ", iz = " << 0 - << ", ix = " << 0; - message << get_error_message(point_property, value_l); + message << "\n \t Error at ispec = " << ielement << ", iz = " << 0 + << ", ix = " << 0 << ", iprop = " << l; + message << get_error_message(point_property, value_l); - throw std::runtime_error(message.str()); + throw std::runtime_error(message.str()); + } } } } From e778c3bb932dd31b1650c1c7abe88a7376ec0661 Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Tue, 25 Feb 2025 13:32:42 -0500 Subject: [PATCH 02/21] Compute derivated perperties on the fly. --- include/medium/properties_container.hpp | 4 - include/point/properties.hpp | 111 +++++++++--------------- 2 files changed, 41 insertions(+), 74 deletions(-) diff --git a/include/medium/properties_container.hpp b/include/medium/properties_container.hpp index 3fc831ce..89ec9bb9 100644 --- a/include/medium/properties_container.hpp +++ b/include/medium/properties_container.hpp @@ -61,8 +61,6 @@ struct impl_properties_container { for (int i = 0; i < nprops; i++) { property.data[i] = target(ispec, iz, ix, i); } - - property.compute(); } template KOKKOS_FORCEINLINE_FUNCTION void @@ -90,8 +88,6 @@ struct impl_properties_container { Kokkos::Experimental::where(mask, property.data[i]) .copy_from(&target(ispec, iz, ix, i), tag_type()); } - - property.compute(); } public: diff --git a/include/point/properties.hpp b/include/point/properties.hpp index fab5902d..6b703ddc 100644 --- a/include/point/properties.hpp +++ b/include/point/properties.hpp @@ -8,22 +8,21 @@ namespace point { #define DEFINE_PROP(prop) \ constexpr static int i_##prop = __COUNTER__ - _counter - 1; \ - KOKKOS_INLINE_FUNCTION value_type prop() const { \ + KOKKOS_INLINE_FUNCTION constexpr value_type prop() const { \ return Base::data[i_##prop]; \ } \ - KOKKOS_INLINE_FUNCTION void prop(value_type val) { \ + KOKKOS_INLINE_FUNCTION constexpr void prop(value_type &val) { \ Base::data[i_##prop] = val; \ } namespace impl { -template struct impl_properties { +template struct impl_properties { using simd = specfem::datatype::simd; ///< SIMD type using value_type = typename simd::datatype; constexpr static bool is_point_properties = true; constexpr static auto nprops = N; - constexpr static auto nprops_extra = N_EX; - value_type data[N_EX]; + value_type data[N]; KOKKOS_FUNCTION impl_properties() = default; @@ -52,8 +51,8 @@ template struct impl_properties { * */ KOKKOS_FUNCTION - bool operator==(const impl_properties &rhs) const { - for (int i = 0; i < N_EX; ++i) { + bool operator==(const impl_properties &rhs) const { + for (int i = 0; i < N; ++i) { if (data[i] != rhs.data[i]) { return false; } @@ -66,7 +65,7 @@ template struct impl_properties { * */ KOKKOS_FUNCTION - bool operator!=(const impl_properties &rhs) const { + bool operator!=(const impl_properties &rhs) const { return !(*this == rhs); } }; @@ -94,14 +93,13 @@ template struct properties - : public impl::impl_properties<3, 6, UseSIMD> { + : public impl::impl_properties<3, UseSIMD> { /** * @name Typedefs * */ ///@{ - using Base = impl::impl_properties<3, 6, UseSIMD>; constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::elastic; constexpr static auto property_tag = @@ -111,31 +109,25 @@ struct properties = 0> - KOKKOS_FUNCTION properties(Args... args) : Base(args...) { - compute(); - } + using Base = impl::impl_properties<3, UseSIMD>; + using Base::Base; DEFINE_PROP(lambdaplus2mu) ///< Lame's parameter @f$ \lambda + 2\mu @f$ DEFINE_PROP(mu) ///< shear modulus @f$ \mu @f$ DEFINE_PROP(rho) ///< density @f$ \rho @f$ - DEFINE_PROP(rho_vp) ///< P-wave velocity @f$ \rho v_p @f$ - DEFINE_PROP(rho_vs) ///< S-wave velocity @f$ \rho v_s @f$ - DEFINE_PROP(lambda) ///< Lame's parameter @f$ \lambda @f$ + KOKKOS_INLINE_FUNCTION constexpr value_type rho_vp() const { + return Kokkos::sqrt(rho() * lambdaplus2mu()); ///< P-wave velocity @f$ \rho + ///< v_p @f$ + } - KOKKOS_INLINE_FUNCTION - void compute() { - rho_vp(Kokkos::sqrt(rho() * lambdaplus2mu())); - rho_vs(Kokkos::sqrt(rho() * mu())); - lambda(lambdaplus2mu() - (static_cast(2.0)) * mu()); + KOKKOS_INLINE_FUNCTION constexpr value_type rho_vs() const { + return Kokkos::sqrt(rho() * mu()); ///< S-wave velocity @f$ \rho v_s @f$ + } + + KOKKOS_INLINE_FUNCTION constexpr value_type lambda() const { + return lambdaplus2mu() - (static_cast(2.0)) * + mu(); ///< Lame's parameter @f$ \lambda @f$ } }; @@ -143,14 +135,13 @@ template struct properties - : public impl::impl_properties<10, 12, UseSIMD> { + : public impl::impl_properties<10, UseSIMD> { /** * @name Typedefs * */ ///@{ - using Base = impl::impl_properties<10, 12, UseSIMD>; constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::elastic; constexpr static auto property_tag = @@ -160,17 +151,8 @@ struct properties = 0> - KOKKOS_FUNCTION properties(Args... args) : Base(args...) { - compute(); - } + using Base = impl::impl_properties<10, UseSIMD>; + using Base::Base; /** * @name Elastic constants @@ -186,16 +168,15 @@ struct properties struct properties - : public impl::impl_properties<2, 4, UseSIMD> { - + : public impl::impl_properties<2, UseSIMD> { /** * @name Typedefs * */ ///@{ - using Base = impl::impl_properties<2, 4, UseSIMD>; constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::acoustic; constexpr static auto property_tag = @@ -225,31 +204,23 @@ struct properties; + using Base::Base; - KOKKOS_FUNCTION - properties(const value_type *value) : Base(value) { compute(); } + DEFINE_PROP(rho_inverse) ///< @f$ \frac{1}{\rho} @f$ + DEFINE_PROP(kappa) ///< Bulk modulus @f$ \kappa @f$ - template = 0> - KOKKOS_FUNCTION properties(Args... args) : Base(args...) { - compute(); + KOKKOS_INLINE_FUNCTION constexpr value_type kappa_inverse() const { + return (static_cast(1.0)) / + kappa(); ///< @f$ \frac{1}{\lambda + 2\mu} @f$ } - DEFINE_PROP(rho_inverse) ///< @f$ \frac{1}{\rho} @f$ - DEFINE_PROP(kappa) ///< Bulk modulus @f$ \kappa @f$ - DEFINE_PROP(kappa_inverse) ///< @f$ \frac{1}{\lambda + 2\mu} @f$ - DEFINE_PROP(rho_vpinverse) ///< @f$ \frac{1}{\rho v_p} @f$ - - KOKKOS_INLINE_FUNCTION - void compute() { - kappa_inverse((static_cast(1.0)) / kappa()); - rho_vpinverse(Kokkos::sqrt(rho_inverse() * kappa_inverse())); + KOKKOS_INLINE_FUNCTION constexpr value_type rho_vpinverse() const { + return Kokkos::sqrt(rho_inverse() * kappa_inverse()); ///< @f$ \frac{1}{\rho + ///< v_p} @f$ } }; -#undef DEFINE_PROP_CONSTRUCTORS #undef DEFINE_PROP } // namespace point From c09796f51858d29bc9d34a20aeb98edc5eda9872 Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Tue, 25 Feb 2025 14:46:50 -0500 Subject: [PATCH 03/21] Update macro location. --- include/point/properties.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/point/properties.hpp b/include/point/properties.hpp index 6b703ddc..dfa3c901 100644 --- a/include/point/properties.hpp +++ b/include/point/properties.hpp @@ -3,9 +3,6 @@ #include "datatypes/simd.hpp" #include "enumerations/medium.hpp" -namespace specfem { -namespace point { - #define DEFINE_PROP(prop) \ constexpr static int i_##prop = __COUNTER__ - _counter - 1; \ KOKKOS_INLINE_FUNCTION constexpr value_type prop() const { \ @@ -15,6 +12,9 @@ namespace point { Base::data[i_##prop] = val; \ } +namespace specfem { +namespace point { + namespace impl { template struct impl_properties { using simd = specfem::datatype::simd; ///< SIMD type @@ -221,7 +221,7 @@ struct properties Date: Wed, 26 Feb 2025 10:18:16 -0500 Subject: [PATCH 04/21] Rename Base to base_type --- .../isotropic/properties_container.hpp | 4 +-- .../anisotropic/properties_container.hpp | 4 +-- .../isotropic/properties_container.hpp | 4 +-- include/medium/properties_container.hpp | 4 +-- include/point/properties.hpp | 16 ++++----- include/policies/chunk.hpp | 34 +++++++++---------- 6 files changed, 33 insertions(+), 33 deletions(-) diff --git a/include/medium/dim2/acoustic/isotropic/properties_container.hpp b/include/medium/dim2/acoustic/isotropic/properties_container.hpp index ee9381fe..6a788138 100644 --- a/include/medium/dim2/acoustic/isotropic/properties_container.hpp +++ b/include/medium/dim2/acoustic/isotropic/properties_container.hpp @@ -13,11 +13,11 @@ struct properties_container { - using Base = + using base_type = impl::impl_properties_container; - using Base::Base; + using base_type::base_type; constexpr static int _counter = __COUNTER__; DEFINE_CONTAINER(rho_inverse) diff --git a/include/medium/dim2/elastic/anisotropic/properties_container.hpp b/include/medium/dim2/elastic/anisotropic/properties_container.hpp index b1307985..b78debd9 100644 --- a/include/medium/dim2/elastic/anisotropic/properties_container.hpp +++ b/include/medium/dim2/elastic/anisotropic/properties_container.hpp @@ -13,10 +13,10 @@ struct properties_container { - using Base = impl::impl_properties_container< + using base_type = impl::impl_properties_container< specfem::element::medium_tag::elastic, specfem::element::property_tag::anisotropic, 10>; - using Base::Base; + using base_type::base_type; constexpr static int _counter = __COUNTER__; DEFINE_CONTAINER(c11) diff --git a/include/medium/dim2/elastic/isotropic/properties_container.hpp b/include/medium/dim2/elastic/isotropic/properties_container.hpp index 1a4e8bdc..998d8877 100644 --- a/include/medium/dim2/elastic/isotropic/properties_container.hpp +++ b/include/medium/dim2/elastic/isotropic/properties_container.hpp @@ -13,11 +13,11 @@ struct properties_container { - using Base = + using base_type = impl::impl_properties_container; - using Base::Base; + using base_type::base_type; constexpr static int _counter = __COUNTER__; DEFINE_CONTAINER(lambdaplus2mu) diff --git a/include/medium/properties_container.hpp b/include/medium/properties_container.hpp index 89ec9bb9..9a762252 100644 --- a/include/medium/properties_container.hpp +++ b/include/medium/properties_container.hpp @@ -6,11 +6,11 @@ constexpr static int i_##prop = __COUNTER__ - _counter - 1; \ KOKKOS_INLINE_FUNCTION type_real &prop(const int &ispec, const int &iz, \ const int &ix) const { \ - return Base::data(ispec, iz, ix, i_##prop); \ + return base_type::data(ispec, iz, ix, i_##prop); \ } \ KOKKOS_INLINE_FUNCTION type_real &h_##prop(const int &ispec, const int &iz, \ const int &ix) const { \ - return Base::h_data(ispec, iz, ix, i_##prop); \ + return base_type::h_data(ispec, iz, ix, i_##prop); \ } namespace specfem { diff --git a/include/point/properties.hpp b/include/point/properties.hpp index dfa3c901..34befd8b 100644 --- a/include/point/properties.hpp +++ b/include/point/properties.hpp @@ -6,10 +6,10 @@ #define DEFINE_PROP(prop) \ constexpr static int i_##prop = __COUNTER__ - _counter - 1; \ KOKKOS_INLINE_FUNCTION constexpr value_type prop() const { \ - return Base::data[i_##prop]; \ + return base_type::data[i_##prop]; \ } \ KOKKOS_INLINE_FUNCTION constexpr void prop(value_type &val) { \ - Base::data[i_##prop] = val; \ + base_type::data[i_##prop] = val; \ } namespace specfem { @@ -109,8 +109,8 @@ struct properties; - using Base::Base; + using base_type = impl::impl_properties<3, UseSIMD>; + using base_type::base_type; DEFINE_PROP(lambdaplus2mu) ///< Lame's parameter @f$ \lambda + 2\mu @f$ DEFINE_PROP(mu) ///< shear modulus @f$ \mu @f$ @@ -151,8 +151,8 @@ struct properties; - using Base::Base; + using base_type = impl::impl_properties<10, UseSIMD>; + using base_type::base_type; /** * @name Elastic constants @@ -204,8 +204,8 @@ struct properties; - using Base::Base; + using base_type = impl::impl_properties<2, UseSIMD>; + using base_type::base_type; DEFINE_PROP(rho_inverse) ///< @f$ \frac{1}{\rho} @f$ DEFINE_PROP(kappa) ///< Bulk modulus @f$ \kappa @f$ diff --git a/include/policies/chunk.hpp b/include/policies/chunk.hpp index 018b99a6..a112f5f5 100644 --- a/include/policies/chunk.hpp +++ b/include/policies/chunk.hpp @@ -63,14 +63,14 @@ struct chunk_index_type { template struct mapped_chunk_index_type : public chunk_index_type { - using Base = chunk_index_type; + using base_type = chunk_index_type; int imap; ///< Index of the mapped element KOKKOS_INLINE_FUNCTION mapped_chunk_index_type(const int ielement, const specfem::point::index index, const int imap) - : Base(ielement, index), imap(imap) {} + : base_type(ielement, index), imap(imap) {} }; } // namespace impl @@ -252,20 +252,20 @@ class mapped_chunk; template class mapped_chunk : public chunk { - using Base = chunk; + using base_type = chunk; using mapped_index_type = - typename impl::mapped_chunk_index_type; ///< Index + typename impl::mapped_chunk_index_type; ///< Index public: KOKKOS_INLINE_FUNCTION mapped_chunk(const ViewType &indices, const ViewType &mapping, const int ngllz, const int ngllx) - : Base(indices, ngllz, ngllx), mapping(mapping) {} + : base_type(indices, ngllz, ngllx), mapping(mapping) {} KOKKOS_INLINE_FUNCTION mapped_index_type operator()(const int i) const { - const auto base_index = Base::operator()(i); + const auto base_index = base_type::operator()(i); return mapped_index_type(base_index.ielement, base_index.index, mapping(base_index.ielement)); } @@ -405,8 +405,8 @@ struct element_chunk template struct mapped_element_chunk : public element_chunk { using simd = typename ParallelConfig::simd; - using Base = element_chunk; - using IndexViewType = typename Base::IndexViewType; + using base_type = element_chunk; + using IndexViewType = typename base_type::IndexViewType; using mapped_iterator_type = specfem::iterator::mapped_chunk { mapped_element_chunk(const IndexViewType &view, const IndexViewType &mapping, int ngllz, int ngllx) - : Base(view, ngllz, ngllx), mapping(mapping) {} + : base_type(view, ngllz, ngllx), mapping(mapping) {} KOKKOS_INLINE_FUNCTION mapped_iterator_type mapped_league_iterator(const int start_index) const { const int start = start_index; - const int end = - (start + Base::chunk_size * Base::simd_size > Base::elements.extent(0)) - ? Base::elements.extent(0) - : start + Base::chunk_size * Base::simd_size; + const int end = (start + base_type::chunk_size * base_type::simd_size > + base_type::elements.extent(0)) + ? base_type::elements.extent(0) + : start + base_type::chunk_size * base_type::simd_size; const auto elem_indices = - Kokkos::subview(Base::elements, Kokkos::make_pair(start, end)); + Kokkos::subview(base_type::elements, Kokkos::make_pair(start, end)); const auto map_indices = Kokkos::subview(mapping, Kokkos::make_pair(start, end)); - return mapped_iterator_type(elem_indices, map_indices, Base::ngllz, - Base::ngllx); + return mapped_iterator_type(elem_indices, map_indices, base_type::ngllz, + base_type::ngllx); } protected: From b09b9c2c4bbe31b59c0f58eda847ffdc86574623 Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Wed, 26 Feb 2025 16:50:28 -0500 Subject: [PATCH 05/21] Merge host and device access functions. --- include/compute/compute_mesh.hpp | 68 +- .../compute/compute_partial_derivatives.hpp | 109 +- include/compute/fields/data_access.tpp | 1628 ++++------------- include/compute/fields/impl/field_impl.hpp | 45 +- include/compute/fields/impl/field_impl.tpp | 4 +- include/compute/fields/simulation_field.hpp | 24 +- include/medium/properties_container.hpp | 20 +- 7 files changed, 501 insertions(+), 1397 deletions(-) diff --git a/include/compute/compute_mesh.hpp b/include/compute/compute_mesh.hpp index d87e236c..e2aea2bf 100644 --- a/include/compute/compute_mesh.hpp +++ b/include/compute/compute_mesh.hpp @@ -229,7 +229,7 @@ struct mesh { */ /** - * @brief Load quadrature data for a spectral element on the device + * @brief Load quadrature data for a spectral element on host or device * * @ingroup QuadratureDataAccess * @@ -239,11 +239,11 @@ struct mesh { * @param quadrature Quadrature data * @param element_quadrature Quadrature data for the element (output) */ -template -KOKKOS_FUNCTION void -load_on_device(const MemberType &team, - const specfem::compute::quadrature &quadrature, - ViewType &element_quadrature) { +template +KOKKOS_INLINE_FUNCTION void +impl_load(const MemberType &team, + const specfem::compute::quadrature &quadrature, + ViewType &element_quadrature) { constexpr bool store_hprime_gll = ViewType::store_hprime_gll; @@ -260,15 +260,45 @@ load_on_device(const MemberType &team, int ix, iz; sub2ind(xz, NGLL, iz, ix); if constexpr (store_hprime_gll) { - element_quadrature.hprime_gll(iz, ix) = quadrature.gll.hprime(iz, ix); + element_quadrature.hprime_gll(iz, ix) = + on_device ? quadrature.gll.hprime(iz, ix) + : quadrature.gll.h_hprime(iz, ix); } if constexpr (store_weight_times_hprime_gll) { element_quadrature.hprime_wgll(ix, iz) = - quadrature.gll.hprime(iz, ix) * quadrature.gll.weights(iz); + on_device + ? quadrature.gll.hprime(iz, ix) * quadrature.gll.weights(iz) + : quadrature.gll.h_hprime(iz, ix) * + quadrature.gll.h_weights(iz); } }); } +/** + * @defgroup QuadratureDataAccess + * + */ + +/** + * @brief Load quadrature data for a spectral element on the device + * + * @ingroup QuadratureDataAccess + * + * @tparam MemberType Member type. Needs to be a Kokkos::TeamPolicy member type + * @tparam ViewType View type. Needs to be of @ref specfem::element::quadrature + * @param team Team member + * @param quadrature Quadrature data + * @param element_quadrature Quadrature data for the element (output) + */ +template +KOKKOS_FUNCTION void +load_on_device(const MemberType &team, + const specfem::compute::quadrature &quadrature, + ViewType &element_quadrature) { + + impl_load(team, quadrature, element_quadrature); +} + /** * @brief Load quadrature data for a spectral element on the host * @@ -284,27 +314,7 @@ template void load_on_host(const MemberType &team, const specfem::compute::quadrature &quadrature, ViewType &element_quadrature) { - - constexpr bool store_hprime_gll = ViewType::store_hprime_gll; - constexpr bool store_weight_times_hprime_gll = - ViewType::store_weight_times_hprime_gll; - constexpr int NGLL = ViewType::ngll; - - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, NGLL * NGLL), [=](const int &xz) { - int ix, iz; - sub2ind(xz, NGLL, iz, ix); - if constexpr (store_hprime_gll) { - element_quadrature.hprime_gll(iz, ix) = - quadrature.gll.h_hprime(iz, ix); - } - if constexpr (store_weight_times_hprime_gll) { - element_quadrature.hprime_wgll(ix, iz) = - quadrature.gll.h_hprime(iz, ix) * quadrature.gll.h_weights(iz); - } - }); - - return; + impl_load(team, quadrature, element_quadrature); } } // namespace compute diff --git a/include/compute/compute_partial_derivatives.hpp b/include/compute/compute_partial_derivatives.hpp index 700eb79f..0244fc7d 100644 --- a/include/compute/compute_partial_derivatives.hpp +++ b/include/compute/compute_partial_derivatives.hpp @@ -71,10 +71,10 @@ struct partial_derivatives { * */ -template = 0> -KOKKOS_FORCEINLINE_FUNCTION void impl_load_on_device( +KOKKOS_FORCEINLINE_FUNCTION void impl_load( const specfem::point::simd_index &index, const specfem::compute::partial_derivatives &derivatives, @@ -95,84 +95,33 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_load_on_device( mask_type mask([&](std::size_t lane) { return index.mask(lane); }); Kokkos::Experimental::where(mask, partial_derivatives.xix) - .copy_from(&derivatives.xix(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, partial_derivatives.gammax) - .copy_from(&derivatives.gammax(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, partial_derivatives.xiz) - .copy_from(&derivatives.xiz(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, partial_derivatives.gammaz) - .copy_from(&derivatives.gammaz(ispec, iz, ix), tag_type()); - if constexpr (StoreJacobian) { - Kokkos::Experimental::where(mask, partial_derivatives.jacobian) - .copy_from(&derivatives.jacobian(ispec, iz, ix), tag_type()); - } -} - -template = 0> -KOKKOS_FORCEINLINE_FUNCTION void impl_load_on_device( - const specfem::point::index &index, - const specfem::compute::partial_derivatives &derivatives, - PointPartialDerivativesType &partial_derivatives) { - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - constexpr static bool StoreJacobian = - PointPartialDerivativesType::store_jacobian; - - partial_derivatives.xix = derivatives.xix(ispec, iz, ix); - partial_derivatives.gammax = derivatives.gammax(ispec, iz, ix); - partial_derivatives.xiz = derivatives.xiz(ispec, iz, ix); - partial_derivatives.gammaz = derivatives.gammaz(ispec, iz, ix); - if constexpr (StoreJacobian) { - partial_derivatives.jacobian = derivatives.jacobian(ispec, iz, ix); - } -} - -template = 0> -inline void -impl_load_on_host(const specfem::point::simd_index< - PointPointPartialDerivativesType::dimension> &index, - const specfem::compute::partial_derivatives &derivatives, - PointPointPartialDerivativesType &partial_derivatives) { - - const int ispec = index.ispec; - const int nspec = derivatives.nspec; - const int iz = index.iz; - const int ix = index.ix; - - constexpr static bool StoreJacobian = - PointPointPartialDerivativesType::store_jacobian; - - using simd = typename PointPointPartialDerivativesType::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, partial_derivatives.xix) - .copy_from(&derivatives.h_xix(ispec, iz, ix), tag_type()); + .copy_from(on_device ? &derivatives.xix(ispec, iz, ix) + : &derivatives.h_xix(ispec, iz, ix), + tag_type()); Kokkos::Experimental::where(mask, partial_derivatives.gammax) - .copy_from(&derivatives.h_gammax(ispec, iz, ix), tag_type()); + .copy_from(on_device ? &derivatives.gammax(ispec, iz, ix) + : &derivatives.h_gammax(ispec, iz, ix), + tag_type()); Kokkos::Experimental::where(mask, partial_derivatives.xiz) - .copy_from(&derivatives.h_xiz(ispec, iz, ix), tag_type()); + .copy_from(on_device ? &derivatives.xiz(ispec, iz, ix) + : &derivatives.h_xiz(ispec, iz, ix), + tag_type()); Kokkos::Experimental::where(mask, partial_derivatives.gammaz) - .copy_from(&derivatives.h_gammaz(ispec, iz, ix), tag_type()); + .copy_from(on_device ? &derivatives.gammaz(ispec, iz, ix) + : &derivatives.h_gammaz(ispec, iz, ix), + tag_type()); if constexpr (StoreJacobian) { Kokkos::Experimental::where(mask, partial_derivatives.jacobian) - .copy_from(&derivatives.h_jacobian(ispec, iz, ix), tag_type()); + .copy_from(on_device ? &derivatives.jacobian(ispec, iz, ix) + : &derivatives.h_jacobian(ispec, iz, ix), + tag_type()); } } -template = 0> -inline void impl_load_on_host( +KOKKOS_FORCEINLINE_FUNCTION void impl_load( const specfem::point::index &index, const specfem::compute::partial_derivatives &derivatives, PointPartialDerivativesType &partial_derivatives) { @@ -184,12 +133,18 @@ inline void impl_load_on_host( constexpr static bool StoreJacobian = PointPartialDerivativesType::store_jacobian; - partial_derivatives.xix = derivatives.h_xix(ispec, iz, ix); - partial_derivatives.gammax = derivatives.h_gammax(ispec, iz, ix); - partial_derivatives.xiz = derivatives.h_xiz(ispec, iz, ix); - partial_derivatives.gammaz = derivatives.h_gammaz(ispec, iz, ix); + partial_derivatives.xix = on_device ? derivatives.xix(ispec, iz, ix) + : derivatives.h_xix(ispec, iz, ix); + partial_derivatives.gammax = on_device ? derivatives.gammax(ispec, iz, ix) + : derivatives.h_gammax(ispec, iz, ix); + partial_derivatives.xiz = on_device ? derivatives.xiz(ispec, iz, ix) + : derivatives.h_xiz(ispec, iz, ix); + partial_derivatives.gammaz = on_device ? derivatives.gammaz(ispec, iz, ix) + : derivatives.h_gammaz(ispec, iz, ix); if constexpr (StoreJacobian) { - partial_derivatives.jacobian = derivatives.h_jacobian(ispec, iz, ix); + partial_derivatives.jacobian = on_device + ? derivatives.jacobian(ispec, iz, ix) + : derivatives.h_jacobian(ispec, iz, ix); } } @@ -276,7 +231,7 @@ KOKKOS_FORCEINLINE_FUNCTION void load_on_device(const IndexType &index, const specfem::compute::partial_derivatives &derivatives, PointPartialDerivativesType &partial_derivatives) { - impl_load_on_device(index, derivatives, partial_derivatives); + impl_load(index, derivatives, partial_derivatives); } /** @@ -302,7 +257,7 @@ inline void load_on_host(const IndexType &index, const specfem::compute::partial_derivatives &derivatives, PointPartialDerivativesType &partial_derivatives) { - impl_load_on_host(index, derivatives, partial_derivatives); + impl_load(index, derivatives, partial_derivatives); } /** diff --git a/include/compute/fields/data_access.tpp b/include/compute/fields/data_access.tpp index 94f27af4..f8aea0ec 100644 --- a/include/compute/fields/data_access.tpp +++ b/include/compute/fields/data_access.tpp @@ -6,11 +6,11 @@ namespace specfem { namespace compute { -template < +template = 0> -KOKKOS_FORCEINLINE_FUNCTION void impl_load_on_device(const int iglob, +KOKKOS_FORCEINLINE_FUNCTION void impl_load(const int iglob, const WavefieldType &field, ViewType &point_field) { @@ -25,40 +25,40 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_load_on_device(const int iglob, if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < ViewType::components; ++icomp) { point_field.displacement(icomp) = - curr_field.field(iglob, icomp); + curr_field.template get_field(iglob, icomp); } } if constexpr (StoreVelocity) { for (int icomp = 0; icomp < ViewType::components; ++icomp) { point_field.velocity(icomp) = - curr_field.field_dot(iglob, icomp); + curr_field.template get_field_dot(iglob, icomp); } } if constexpr (StoreAcceleration) { for (int icomp = 0; icomp < ViewType::components; ++icomp) { point_field.acceleration(icomp) = - curr_field.field_dot_dot(iglob, icomp); + curr_field.template get_field_dot_dot(iglob, icomp); } } if constexpr (StoreMassMatrix) { for (int icomp = 0; icomp < ViewType::components; ++icomp) { point_field.mass_matrix(icomp) = - curr_field.mass_inverse(iglob, icomp); + curr_field.template get_mass_inverse(iglob, icomp); } } return; } -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_load_on_device(const typename ViewType::simd::mask_type &mask, +impl_load(const typename ViewType::simd::mask_type &mask, const int *iglob, const WavefieldType &field, ViewType &point_field) { @@ -81,28 +81,28 @@ impl_load_on_device(const typename ViewType::simd::mask_type &mask, if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { point_field.displacement(icomp)[lane] = - curr_field.field(iglob_l, icomp); + curr_field.template get_field(iglob_l, icomp); } } if constexpr (StoreVelocity) { for (int icomp = 0; icomp < components; ++icomp) { point_field.velocity(icomp)[lane] = - curr_field.field_dot(iglob_l, icomp); + curr_field.template get_field_dot(iglob_l, icomp); } } if constexpr (StoreAcceleration) { for (int icomp = 0; icomp < components; ++icomp) { point_field.acceleration(icomp)[lane] = - curr_field.field_dot_dot(iglob_l, icomp); + curr_field.template get_field_dot_dot(iglob_l, icomp); } } if constexpr (StoreMassMatrix) { for (int icomp = 0; icomp < components; ++icomp) { point_field.mass_matrix(icomp)[lane] = - curr_field.mass_inverse(iglob_l, icomp); + curr_field.template get_mass_inverse(iglob_l, icomp); } } } @@ -110,12 +110,12 @@ impl_load_on_device(const typename ViewType::simd::mask_type &mask, return; } -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_load_on_device(const specfem::point::simd_assembly_index &index, +impl_load(const specfem::point::simd_assembly_index &index, const WavefieldType &field, ViewType &point_field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; @@ -137,40 +137,98 @@ impl_load_on_device(const specfem::point::simd_assembly_index &index, if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.displacement(icomp)) - .copy_from(&curr_field.field(iglob, icomp), tag_type()); + .copy_from(&(curr_field.template get_field(iglob, icomp)), tag_type()); } } if constexpr (StoreVelocity) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.velocity(icomp)) - .copy_from(&curr_field.field_dot(iglob, icomp), tag_type()); + .copy_from(&(curr_field.template get_field_dot(iglob, icomp)), tag_type()); } } if constexpr (StoreAcceleration) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.acceleration(icomp)) - .copy_from(&curr_field.field_dot_dot(iglob, icomp), tag_type()); + .copy_from(&(curr_field.template get_field_dot_dot(iglob, icomp)), tag_type()); } } if constexpr (StoreMassMatrix) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.mass_matrix(icomp)) - .copy_from(&curr_field.mass_inverse(iglob, icomp), tag_type()); + .copy_from(&(curr_field.template get_mass_inverse(iglob, icomp)), tag_type()); } } return; } -template < +template = 0> -inline void impl_load_on_host(const int iglob, const WavefieldType &field, - ViewType &point_field) { +KOKKOS_FORCEINLINE_FUNCTION void +impl_load(const specfem::point::index &index, + const WavefieldType &field, ViewType &point_field) { + constexpr static auto MediumType = ViewType::medium_tag; + const int iglob_l = on_device ? field.index_mapping(index.ispec, index.iz, index.ix) : field.h_index_mapping(index.ispec, index.iz, index.ix); + const int iglob = on_device ? + field.assembly_index_mapping(iglob_l, static_cast(MediumType)) : field.h_assembly_index_mapping(iglob_l, static_cast(MediumType)); + impl_load(iglob, field, point_field); +} + +template = 0> +KOKKOS_FORCEINLINE_FUNCTION void +impl_load(const specfem::point::assembly_index &index, + const WavefieldType &field, ViewType &point_field) { + constexpr static auto MediumType = ViewType::medium_tag; + const int iglob = index.iglob; + impl_load(iglob, field, point_field); +} + +template = 0> +KOKKOS_FORCEINLINE_FUNCTION void impl_load( + const specfem::point::simd_index &index, + const WavefieldType &field, ViewType &point_field) { + constexpr static auto MediumType = ViewType::medium_tag; + int iglob[ViewType::simd::size()]; + + using mask_type = typename ViewType::simd::mask_type; + + mask_type mask([&](std::size_t lane) { return index.mask(lane); }); + + for (int lane = 0; lane < ViewType::simd::size(); ++lane) { + iglob[lane] = on_device ? + ((index.mask(std::size_t(lane))) + ? field.assembly_index_mapping( + field.index_mapping(index.ispec + lane, index.iz, index.ix), + static_cast(MediumType)) + : field.nglob + 1) : + ((index.mask(std::size_t(lane))) + ? field.h_assembly_index_mapping( + field.h_index_mapping(index.ispec + lane, index.iz, index.ix), + static_cast(MediumType)) + : field.nglob + 1); + } + + impl_load(mask, iglob, field, point_field); +} + +template = 0> +KOKKOS_FORCEINLINE_FUNCTION void +impl_store(const int iglob, const ViewType &point_field, + const WavefieldType &field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; @@ -180,44 +238,32 @@ inline void impl_load_on_host(const int iglob, const WavefieldType &field, const auto &curr_field = field.template get_field(); - if constexpr (StoreDisplacement) { - for (int icomp = 0; icomp < ViewType::components; ++icomp) { - point_field.displacement(icomp) = - curr_field.h_field(iglob, icomp); + for (int icomp = 0; icomp < ViewType::components; ++icomp) { + if constexpr (StoreDisplacement) { + curr_field.template get_field(iglob, icomp) = point_field.displacement(icomp); } - } - - if constexpr (StoreVelocity) { - for (int icomp = 0; icomp < ViewType::components; ++icomp) { - point_field.velocity(icomp) = - curr_field.h_field_dot(iglob, icomp); + if constexpr (StoreVelocity) { + curr_field.template get_field_dot(iglob, icomp) = point_field.velocity(icomp); } - } - - if constexpr (StoreAcceleration) { - for (int icomp = 0; icomp < ViewType::components; ++icomp) { - point_field.acceleration(icomp) = - curr_field.h_field_dot_dot(iglob, icomp); + if constexpr (StoreAcceleration) { + curr_field.template get_field_dot_dot(iglob, icomp) = point_field.acceleration(icomp); } - } - - if constexpr (StoreMassMatrix) { - for (int icomp = 0; icomp < ViewType::components; ++icomp) { - point_field.mass_matrix(icomp) = - curr_field.h_mass_inverse(iglob, icomp); + if constexpr (StoreMassMatrix) { + curr_field.template get_mass_inverse(iglob, icomp) = point_field.mass_matrix(icomp); } } return; } -template < +template = 0> -inline void impl_load_on_host(const typename ViewType::simd::mask_type &mask, - const int *iglob, const WavefieldType &field, - ViewType &point_field) { + typename std::enable_if_t< + ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> +KOKKOS_FORCEINLINE_FUNCTION void +impl_store(const typename ViewType::simd::mask_type &mask, + const int *iglob, const ViewType &point_field, + const WavefieldType &field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; @@ -227,7 +273,6 @@ inline void impl_load_on_host(const typename ViewType::simd::mask_type &mask, constexpr static int components = ViewType::components; const auto &curr_field = field.template get_field(); - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { if (!mask[lane]) { continue; @@ -237,29 +282,29 @@ inline void impl_load_on_host(const typename ViewType::simd::mask_type &mask, if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { - point_field.displacement(icomp)[lane] = - curr_field.h_field(iglob_l, icomp); + curr_field.template get_field(iglob_l, icomp) = + point_field.displacement(icomp)[lane]; } } if constexpr (StoreVelocity) { for (int icomp = 0; icomp < components; ++icomp) { - point_field.velocity(icomp)[lane] = - curr_field.h_field_dot(iglob_l, icomp); + curr_field.template get_field_dot(iglob_l, icomp) = + point_field.velocity(icomp)[lane]; } } if constexpr (StoreAcceleration) { for (int icomp = 0; icomp < components; ++icomp) { - point_field.acceleration(icomp)[lane] = - curr_field.h_field_dot_dot(iglob_l, icomp); + curr_field.template get_field_dot_dot(iglob_l, icomp) = + point_field.acceleration(icomp)[lane]; } } if constexpr (StoreMassMatrix) { for (int icomp = 0; icomp < components; ++icomp) { - point_field.mass_matrix(icomp)[lane] = - curr_field.h_mass_inverse(iglob_l, icomp); + curr_field.template get_mass_inverse(iglob_l, icomp) = + point_field.mass_matrix(icomp)[lane]; } } } @@ -267,12 +312,13 @@ inline void impl_load_on_host(const typename ViewType::simd::mask_type &mask, return; } -template < +template = 0> -inline void impl_load_on_host(const specfem::point::simd_assembly_index &index, - const WavefieldType &field, ViewType &point_field) { + typename std::enable_if_t< + ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> +KOKKOS_FORCEINLINE_FUNCTION void +impl_store(const specfem::point::simd_assembly_index &index, + const ViewType &point_field, const WavefieldType &field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; @@ -289,124 +335,78 @@ inline void impl_load_on_host(const specfem::point::simd_assembly_index &index, mask_type mask([&](std::size_t lane) { return index.mask(lane); }); const auto &curr_field = field.template get_field(); - if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.displacement(icomp)) - .copy_from(&curr_field.h_field(iglob, icomp), tag_type()); + .copy_to(&(curr_field.template get_field(iglob, icomp)), tag_type()); } } if constexpr (StoreVelocity) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.velocity(icomp)) - .copy_from(&curr_field.h_field_dot(iglob, icomp), tag_type()); + .copy_to(&(curr_field.template get_field_dot(iglob, icomp)), tag_type()); } } if constexpr (StoreAcceleration) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.acceleration(icomp)) - .copy_from(&curr_field.h_field_dot_dot(iglob, icomp), tag_type()); + .copy_to(&(curr_field.template get_field_dot_dot(iglob, icomp)), tag_type()); } } if constexpr (StoreMassMatrix) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.mass_matrix(icomp)) - .copy_from(&curr_field.h_mass_inverse(iglob, icomp), tag_type()); + .copy_to(&(curr_field.template get_mass_inverse(iglob, icomp)), tag_type()); } } return; } -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_load_on_device(const specfem::point::index &index, - const WavefieldType &field, ViewType &point_field) { - constexpr static auto MediumType = ViewType::medium_tag; - const int iglob_l = field.index_mapping(index.ispec, index.iz, index.ix); - const int iglob = - field.assembly_index_mapping(iglob_l, static_cast(MediumType)); - impl_load_on_device(iglob, field, point_field); -} - -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_load_on_device(const specfem::point::assembly_index &index, - const WavefieldType &field, ViewType &point_field) { - constexpr static auto MediumType = ViewType::medium_tag; - const int iglob = index.iglob; - impl_load_on_device(iglob, field, point_field); -} +impl_store(const specfem::point::index &index, + const ViewType &point_field, const WavefieldType &field) { -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void impl_load_on_device( - const specfem::point::simd_index &index, - const WavefieldType &field, ViewType &point_field) { constexpr static auto MediumType = ViewType::medium_tag; - int iglob[ViewType::simd::size()]; - - using mask_type = typename ViewType::simd::mask_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = - (index.mask(std::size_t(lane))) - ? field.assembly_index_mapping( - field.index_mapping(index.ispec + lane, index.iz, index.ix), - static_cast(MediumType)) - : field.nglob + 1; - } + const int iglob = on_device ? field.assembly_index_mapping( + field.index_mapping(index.ispec, index.iz, index.ix), + static_cast(MediumType)) : field.h_assembly_index_mapping( + field.h_index_mapping(index.ispec, index.iz, index.ix), + static_cast(MediumType)); - impl_load_on_device(mask, iglob, field, point_field); + impl_store(iglob, point_field, field); } -template < +template = 0> -inline void impl_load_on_host(const specfem::point::index &index, - const WavefieldType &field, ViewType &point_field) { +KOKKOS_FORCEINLINE_FUNCTION void +impl_store(const specfem::point::assembly_index &index, + const ViewType &point_field, const WavefieldType &field) { constexpr static auto MediumType = ViewType::medium_tag; - const int iglob = field.h_assembly_index_mapping( - field.h_index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); - - impl_load_on_host(iglob, field, point_field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_load_on_host(const specfem::point::assembly_index &index, - const WavefieldType &field, ViewType &point_field) { - constexpr static auto MediumType = ViewType::medium_tag; const int iglob = index.iglob; - impl_load_on_host(iglob, field, point_field); + + impl_store(iglob, point_field, field); } -template < +template = 0> -inline void impl_load_on_host( +KOKKOS_FORCEINLINE_FUNCTION void impl_store( const specfem::point::simd_index &index, - const WavefieldType &field, ViewType &point_field) { + const ViewType &point_field, const WavefieldType &field) { constexpr static auto MediumType = ViewType::medium_tag; int iglob[ViewType::simd::size()]; @@ -416,24 +416,29 @@ inline void impl_load_on_host( mask_type mask([&](std::size_t lane) { return index.mask(lane); }); for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = - (index.mask(std::size_t(lane))) - ? field.h_assembly_index_mapping( - field.h_index_mapping(index.ispec + lane, index.iz, index.ix), + iglob[lane] = on_device ? + ((index.mask(std::size_t(lane))) + ? field.assembly_index_mapping( + field.index_mapping(index.ispec + lane, index.iz, index.ix), + static_cast(MediumType)) + : field.nglob + 1) : + ((index.mask(std::size_t(lane))) + ? field.assembly_index_mapping( + field.index_mapping(index.ispec + lane, index.iz, index.ix), static_cast(MediumType)) - : field.nglob + 1; + : field.nglob + 1); } - impl_load_on_host(mask, iglob, field, point_field); + impl_store(mask, iglob, point_field, field); } -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_store_on_device(const int iglob, const ViewType &point_field, - const WavefieldType &field) { +impl_add(const int iglob, const ViewType &point_field, + const WavefieldType &field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; @@ -442,33 +447,32 @@ impl_store_on_device(const int iglob, const ViewType &point_field, constexpr static auto MediumType = ViewType::medium_tag; const auto &curr_field = field.template get_field(); - for (int icomp = 0; icomp < ViewType::components; ++icomp) { if constexpr (StoreDisplacement) { - curr_field.field(iglob, icomp) = point_field.displacement(icomp); + curr_field.template get_field(iglob, icomp) += point_field.displacement(icomp); } if constexpr (StoreVelocity) { - curr_field.field_dot(iglob, icomp) = point_field.velocity(icomp); + curr_field.template get_field_dot(iglob, icomp) += point_field.velocity(icomp); } if constexpr (StoreAcceleration) { - curr_field.field_dot_dot(iglob, icomp) = point_field.acceleration(icomp); + curr_field.template get_field_dot_dot(iglob, icomp) += point_field.acceleration(icomp); } if constexpr (StoreMassMatrix) { - curr_field.mass_inverse(iglob, icomp) = point_field.mass_matrix(icomp); + curr_field.template get_mass_inverse(iglob, icomp) += point_field.mass_matrix(icomp); } } return; } -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_store_on_device(const typename ViewType::simd::mask_type &mask, - const int *iglob, const ViewType &point_field, - const WavefieldType &field) { +impl_add(const typename ViewType::simd::mask_type &mask, + const int *iglob, const ViewType &point_field, + const WavefieldType &field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; @@ -487,28 +491,28 @@ impl_store_on_device(const typename ViewType::simd::mask_type &mask, if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { - curr_field.field(iglob_l, icomp) = + curr_field.template get_field(iglob_l, icomp) += point_field.displacement(icomp)[lane]; } } if constexpr (StoreVelocity) { for (int icomp = 0; icomp < components; ++icomp) { - curr_field.field_dot(iglob_l, icomp) = + curr_field.template get_field_dot(iglob_l, icomp) += point_field.velocity(icomp)[lane]; } } if constexpr (StoreAcceleration) { for (int icomp = 0; icomp < components; ++icomp) { - curr_field.field_dot_dot(iglob_l, icomp) = + curr_field.template get_field_dot_dot(iglob_l, icomp) += point_field.acceleration(icomp)[lane]; } } if constexpr (StoreMassMatrix) { for (int icomp = 0; icomp < components; ++icomp) { - curr_field.mass_inverse(iglob_l, icomp) = + curr_field.template get_mass_inverse(iglob_l, icomp) += point_field.mass_matrix(icomp)[lane]; } } @@ -517,13 +521,13 @@ impl_store_on_device(const typename ViewType::simd::mask_type &mask, return; } -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_store_on_device(const specfem::point::simd_assembly_index &index, - const ViewType &point_field, const WavefieldType &field) { +impl_add(const specfem::point::simd_assembly_index &index, + const ViewType &point_field, const WavefieldType &field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; @@ -542,41 +546,128 @@ impl_store_on_device(const specfem::point::simd_assembly_index &index, const auto &curr_field = field.template get_field(); if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::Experimental::where(mask, point_field.displacement(icomp)) - .copy_to(&curr_field.field(iglob, icomp), tag_type()); + typename ViewType::simd::datatype lhs; + Kokkos::Experimental::where(mask, lhs).copy_from( + &(curr_field.template get_field(iglob, icomp)), tag_type()); + + lhs += point_field.displacement(icomp); + Kokkos::Experimental::where(mask, lhs).copy_to( + &(curr_field.template get_field(iglob, icomp)), tag_type()); } } if constexpr (StoreVelocity) { for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::Experimental::where(mask, point_field.velocity(icomp)) - .copy_to(&curr_field.field_dot(iglob, icomp), tag_type()); + typename ViewType::simd::datatype lhs; + Kokkos::Experimental::where(mask, lhs).copy_from( + &(curr_field.template get_field_dot(iglob, icomp)), tag_type()); + + lhs += point_field.velocity(icomp); + Kokkos::Experimental::where(mask, lhs).copy_to( + &(curr_field.template get_field_dot(iglob, icomp)), tag_type()); } } if constexpr (StoreAcceleration) { for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::Experimental::where(mask, point_field.acceleration(icomp)) - .copy_to(&curr_field.field_dot_dot(iglob, icomp), tag_type()); + typename ViewType::simd::datatype lhs; + Kokkos::Experimental::where(mask, lhs).copy_from( + &(curr_field.template get_field_dot_dot(iglob, icomp)), tag_type()); + + lhs += point_field.acceleration(icomp); + Kokkos::Experimental::where(mask, lhs).copy_to( + &(curr_field.template get_field_dot_dot(iglob, icomp)), tag_type()); } } if constexpr (StoreMassMatrix) { for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::Experimental::where(mask, point_field.mass_matrix(icomp)) - .copy_to(&curr_field.mass_inverse(iglob, icomp), tag_type()); + typename ViewType::simd::datatype lhs; + Kokkos::Experimental::where(mask, lhs).copy_from( + &(curr_field.template get_mass_inverse(iglob, icomp)), tag_type()); + + lhs += point_field.mass_matrix(icomp); + Kokkos::Experimental::where(mask, lhs).copy_to( + &(curr_field.template get_mass_inverse(iglob, icomp)), tag_type()); } } return; } -template < +template = 0> +KOKKOS_FORCEINLINE_FUNCTION void +impl_add(const specfem::point::index &index, + const ViewType &point_field, const WavefieldType &field) { + + constexpr static auto MediumType = ViewType::medium_tag; + + const int iglob = on_device ? field.assembly_index_mapping( + field.index_mapping(index.ispec, index.iz, index.ix), + static_cast(MediumType)) : field.h_assembly_index_mapping( + field.h_index_mapping(index.ispec, index.iz, index.ix), + static_cast(MediumType)); + + impl_add(iglob, point_field, field); +} + +template = 0> +KOKKOS_FORCEINLINE_FUNCTION void +impl_add(const specfem::point::assembly_index &index, + const ViewType &point_field, const WavefieldType &field) { + + constexpr static auto MediumType = ViewType::medium_tag; + + const int iglob = index.iglob; + + impl_add(iglob, point_field, field); +} + +template = 0> +KOKKOS_FORCEINLINE_FUNCTION void +impl_add(const specfem::point::simd_index &index, + const ViewType &point_field, const WavefieldType &field) { + + constexpr static auto MediumType = ViewType::medium_tag; + int iglob[ViewType::simd::size()]; + + using mask_type = typename ViewType::simd::mask_type; + + mask_type mask([&](std::size_t lane) { return index.mask(lane); }); + + for (int lane = 0; lane < ViewType::simd::size(); ++lane) { + iglob[lane] = on_device ? + ((index.mask(std::size_t(lane))) + ? field.assembly_index_mapping( + field.index_mapping(index.ispec + lane, index.iz, index.ix), + static_cast(MediumType)) + : field.nglob + 1) : + ((index.mask(std::size_t(lane))) + ? field.h_assembly_index_mapping( + field.h_index_mapping(index.ispec + lane, index.iz, index.ix), + static_cast(MediumType)) + : field.nglob + 1); + } + + impl_add(mask, iglob, point_field, field); +} + +template = 0> -inline void impl_store_on_host(const int iglob, const ViewType &point_field, - const WavefieldType &field) { +KOKKOS_FORCEINLINE_FUNCTION void +impl_atomic_add(const int iglob, const ViewType &point_field, + const WavefieldType &field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; @@ -587,30 +678,34 @@ inline void impl_store_on_host(const int iglob, const ViewType &point_field, const auto &curr_field = field.template get_field(); for (int icomp = 0; icomp < ViewType::components; ++icomp) { if constexpr (StoreDisplacement) { - curr_field.h_field(iglob, icomp) = point_field.displacement(icomp); + Kokkos::atomic_add(&(curr_field.template get_field(iglob, icomp)), + point_field.displacement(icomp)); } if constexpr (StoreVelocity) { - curr_field.h_field_dot(iglob, icomp) = point_field.velocity(icomp); + Kokkos::atomic_add(&(curr_field.template get_field_dot(iglob, icomp)), + point_field.velocity(icomp)); } if constexpr (StoreAcceleration) { - curr_field.h_field_dot_dot(iglob, icomp) = - point_field.acceleration(icomp); + Kokkos::atomic_add(&(curr_field.template get_field_dot_dot(iglob, icomp)), + point_field.acceleration(icomp)); } if constexpr (StoreMassMatrix) { - curr_field.h_mass_inverse(iglob, icomp) = point_field.mass_matrix(icomp); + Kokkos::atomic_add(&(curr_field.template get_mass_inverse(iglob, icomp)), + point_field.mass_matrix(icomp)); } } return; } -template < +template = 0> -inline void impl_store_on_host(const typename ViewType::simd::mask_type &mask, - const int *iglob, const ViewType &point_field, - const WavefieldType &field) { +KOKKOS_FORCEINLINE_FUNCTION void +impl_atomic_add(const typename ViewType::simd::mask_type &mask, + const int *iglob, const ViewType &point_field, + const WavefieldType &field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; @@ -629,29 +724,29 @@ inline void impl_store_on_host(const typename ViewType::simd::mask_type &mask, if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { - curr_field.h_field(iglob_l, icomp) = - point_field.displacement(icomp)[lane]; + Kokkos::atomic_add(&(curr_field.template get_field(iglob_l, icomp)), + point_field.displacement(icomp)[lane]); } } if constexpr (StoreVelocity) { for (int icomp = 0; icomp < components; ++icomp) { - curr_field.h_field_dot(iglob_l, icomp) = - point_field.velocity(icomp)[lane]; + Kokkos::atomic_add(&(curr_field.template get_field_dot(iglob_l, icomp)), + point_field.velocity(icomp)[lane]); } } if constexpr (StoreAcceleration) { for (int icomp = 0; icomp < components; ++icomp) { - curr_field.h_field_dot_dot(iglob_l, icomp) = - point_field.acceleration(icomp)[lane]; + Kokkos::atomic_add(&(curr_field.template get_field_dot_dot(iglob_l, icomp)), + point_field.acceleration(icomp)[lane]); } } if constexpr (StoreMassMatrix) { for (int icomp = 0; icomp < components; ++icomp) { - curr_field.h_mass_inverse(iglob_l, icomp) = - point_field.mass_matrix(icomp)[lane]; + Kokkos::atomic_add(&(curr_field.template get_mass_inverse(iglob_l, icomp)), + point_field.mass_matrix(icomp)[lane]); } } } @@ -659,890 +754,63 @@ inline void impl_store_on_host(const typename ViewType::simd::mask_type &mask, return; } -template < +template = 0> +KOKKOS_FORCEINLINE_FUNCTION void impl_atomic_add( + const specfem::point::index &index, + const ViewType &point_field, const WavefieldType &field) { + + constexpr static auto MediumType = ViewType::medium_tag; + + const int iglob = on_device ? field.assembly_index_mapping( + field.index_mapping(index.ispec, index.iz, index.ix), + static_cast(MediumType)) : field.h_assembly_index_mapping( + field.h_index_mapping(index.ispec, index.iz, index.ix), + static_cast(MediumType)); + + impl_atomic_add(iglob, point_field, field); +} + +template = 0> -inline void impl_store_on_host(const specfem::point::simd_assembly_index &index, - const ViewType &point_field, - const WavefieldType &field) { +KOKKOS_FORCEINLINE_FUNCTION void impl_atomic_add( + const specfem::point::simd_index &index, + const ViewType &point_field, const WavefieldType &field) { - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - using mask_type = typename ViewType::simd::mask_type; - using tag_type = typename ViewType::simd::tag_type; - - const int iglob = index.iglob; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const auto &curr_field = field.template get_field(); - if constexpr (StoreDisplacement) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::Experimental::where(mask, point_field.displacement(icomp)) - .copy_to(&curr_field.h_field(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreVelocity) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::Experimental::where(mask, point_field.velocity(icomp)) - .copy_to(&curr_field.h_field_dot(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreAcceleration) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::Experimental::where(mask, point_field.acceleration(icomp)) - .copy_to(&curr_field.h_field_dot_dot(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreMassMatrix) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::Experimental::where(mask, point_field.acceleration(icomp)) - .copy_to(&curr_field.h_mass_matrix(iglob, icomp), tag_type()); - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_store_on_device(const specfem::point::index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = field.assembly_index_mapping( - field.index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); - - impl_store_on_device(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_store_on_device(const specfem::point::assembly_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = index.iglob; - - impl_store_on_device(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void impl_store_on_device( - const specfem::point::simd_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - int iglob[ViewType::simd::size()]; - - using mask_type = typename ViewType::simd::mask_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = - (index.mask(std::size_t(lane))) - ? field.assembly_index_mapping( - field.index_mapping(index.ispec + lane, index.iz, index.ix), - static_cast(MediumType)) - : field.nglob + 1; - } - - impl_store_on_device(mask, iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_store_on_host(const specfem::point::index &index, - const ViewType &point_field, - const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = field.h_assembly_index_mapping( - field.h_index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); - - impl_store_on_host(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_store_on_host(const specfem::point::assembly_index &index, - const ViewType &point_field, - const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = index.iglob; - - impl_store_on_host(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -inline void impl_store_on_host( - const specfem::point::simd_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - int iglob[ViewType::simd::size()]; - - using mask_type = typename ViewType::simd::mask_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = - (index.mask(std::size_t(lane))) - ? field.h_index_mapping(index.ispec + lane, index.iz, index.ix) - : field.nglob + 1; - } - - impl_store_on_host(mask, iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_add_on_device(const int iglob, const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - - const auto &curr_field = field.template get_field(); - for (int icomp = 0; icomp < ViewType::components; ++icomp) { - if constexpr (StoreDisplacement) { - curr_field.field(iglob, icomp) += point_field.displacement(icomp); - } - if constexpr (StoreVelocity) { - curr_field.field_dot(iglob, icomp) += point_field.velocity(icomp); - } - if constexpr (StoreAcceleration) { - curr_field.field_dot_dot(iglob, icomp) += point_field.acceleration(icomp); - } - if constexpr (StoreMassMatrix) { - curr_field.mass_inverse(iglob, icomp) += point_field.mass_matrix(icomp); - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_add_on_device(const typename ViewType::simd::mask_type &mask, - const int *iglob, const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - const auto &curr_field = field.template get_field(); - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - if (!mask[lane]) { - continue; - } - - const int iglob_l = iglob[lane]; - - if constexpr (StoreDisplacement) { - for (int icomp = 0; icomp < components; ++icomp) { - curr_field.field(iglob_l, icomp) += - point_field.displacement(icomp)[lane]; - } - } - - if constexpr (StoreVelocity) { - for (int icomp = 0; icomp < components; ++icomp) { - curr_field.field_dot(iglob_l, icomp) += - point_field.velocity(icomp)[lane]; - } - } - - if constexpr (StoreAcceleration) { - for (int icomp = 0; icomp < components; ++icomp) { - curr_field.field_dot_dot(iglob_l, icomp) += - point_field.acceleration(icomp)[lane]; - } - } - - if constexpr (StoreMassMatrix) { - for (int icomp = 0; icomp < components; ++icomp) { - curr_field.mass_inverse(iglob_l, icomp) += - point_field.mass_matrix(icomp)[lane]; - } - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_add_on_device(const specfem::point::simd_assembly_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - using mask_type = typename ViewType::simd::mask_type; - using tag_type = typename ViewType::simd::tag_type; - - const int iglob = index.iglob; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const auto &curr_field = field.template get_field(); - if constexpr (StoreDisplacement) { - for (int icomp = 0; icomp < components; ++icomp) { - typename ViewType::simd::datatype lhs; - Kokkos::Experimental::where(mask, lhs).copy_from( - &curr_field.field(iglob, icomp), tag_type()); - - lhs += point_field.displacement(icomp); - Kokkos::Experimental::where(mask, lhs).copy_to( - &curr_field.field(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreVelocity) { - for (int icomp = 0; icomp < components; ++icomp) { - typename ViewType::simd::datatype lhs; - Kokkos::Experimental::where(mask, lhs).copy_from( - &curr_field.field_dot(iglob, icomp), tag_type()); - - lhs += point_field.velocity(icomp); - Kokkos::Experimental::where(mask, lhs).copy_to( - &curr_field.field_dot(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreAcceleration) { - for (int icomp = 0; icomp < components; ++icomp) { - typename ViewType::simd::datatype lhs; - Kokkos::Experimental::where(mask, lhs).copy_from( - &curr_field.field_dot_dot(iglob, icomp), tag_type()); - - lhs += point_field.acceleration(icomp); - Kokkos::Experimental::where(mask, lhs).copy_to( - &curr_field.field_dot_dot(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreMassMatrix) { - for (int icomp = 0; icomp < components; ++icomp) { - typename ViewType::simd::datatype lhs; - Kokkos::Experimental::where(mask, lhs).copy_from( - &curr_field.mass_inverse(iglob, icomp), tag_type()); - - lhs += point_field.mass_matrix(icomp); - Kokkos::Experimental::where(mask, lhs).copy_to( - &curr_field.mass_inverse(iglob, icomp), tag_type()); - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_add_on_host(const int iglob, const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - - const auto &curr_field = field.template get_field(); - for (int icomp = 0; icomp < ViewType::components; ++icomp) { - if constexpr (StoreDisplacement) { - curr_field.h_field(iglob, icomp) += point_field.displacement(icomp); - } - if constexpr (StoreVelocity) { - curr_field.h_field_dot(iglob, icomp) += point_field.velocity(icomp); - } - if constexpr (StoreAcceleration) { - curr_field.h_field_dot_dot(iglob, icomp) += - point_field.acceleration(icomp); - } - if constexpr (StoreMassMatrix) { - curr_field.h_mass_inverse(iglob, icomp) += point_field.mass_matrix(icomp); - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -inline void impl_add_on_host(const typename ViewType::simd::mask_type &mask, - const int *iglob, const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - const auto &curr_field = field.template get_field(); - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - if (!mask[lane]) { - continue; - } - - const int iglob_l = iglob[lane]; - - if constexpr (StoreDisplacement) { - for (int icomp = 0; icomp < components; ++icomp) { - curr_field.h_field(iglob_l, icomp) += - point_field.displacement(icomp)[lane]; - } - } - - if constexpr (StoreVelocity) { - for (int icomp = 0; icomp < components; ++icomp) { - curr_field.h_field_dot(iglob_l, icomp) += - point_field.velocity(icomp)[lane]; - } - } - - if constexpr (StoreAcceleration) { - for (int icomp = 0; icomp < components; ++icomp) { - curr_field.h_field_dot_dot(iglob_l, icomp) += - point_field.acceleration(icomp)[lane]; - } - } - - if constexpr (StoreMassMatrix) { - for (int icomp = 0; icomp < components; ++icomp) { - curr_field.h_mass_inverse(iglob_l, icomp) += - point_field.mass_matrix(icomp)[lane]; - } - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -inline void impl_add_on_host(const specfem::point::simd_assembly_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - using mask_type = typename ViewType::simd::mask_type; - using tag_type = typename ViewType::simd::tag_type; - - const int iglob = index.iglob; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const auto &curr_field = field.template get_field(); - if constexpr (StoreDisplacement) { - for (int icomp = 0; icomp < components; ++icomp) { - typename ViewType::simd::datatype lhs; - Kokkos::Experimental::where(mask, lhs).copy_from( - &curr_field.h_field(iglob, icomp), tag_type()); - - lhs += point_field.displacement(icomp); - Kokkos::Experimental::where(mask, lhs).copy_to( - &curr_field.h_field(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreVelocity) { - for (int icomp = 0; icomp < components; ++icomp) { - typename ViewType::simd::datatype lhs; - Kokkos::Experimental::where(mask, lhs).copy_from( - &curr_field.h_field_dot(iglob, icomp), tag_type()); - - lhs += point_field.velocity(icomp); - Kokkos::Experimental::where(mask, lhs).copy_to( - &curr_field.h_field_dot(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreAcceleration) { - for (int icomp = 0; icomp < components; ++icomp) { - typename ViewType::simd::datatype lhs; - Kokkos::Experimental::where(mask, lhs).copy_from( - &curr_field.h_field_dot_dot(iglob, icomp), tag_type()); - - lhs += point_field.acceleration(icomp); - Kokkos::Experimental::where(mask, lhs).copy_to( - &curr_field.h_field_dot_dot(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreMassMatrix) { - for (int icomp = 0; icomp < components; ++icomp) { - typename ViewType::simd::datatype lhs; - Kokkos::Experimental::where(mask, lhs).copy_from( - &curr_field.h_mass_inverse(iglob, icomp), tag_type()); - - lhs += point_field.mass_matrix(icomp); - Kokkos::Experimental::where(mask, lhs).copy_to( - &curr_field.h_mass_inverse(iglob, icomp), tag_type()); - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_add_on_device(const specfem::point::index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = field.assembly_index_mapping( - field.index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); - - impl_add_on_device(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_add_on_device(const specfem::point::assembly_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = index.iglob; - - impl_add_on_device(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_add_on_device(const specfem::point::simd_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - int iglob[ViewType::simd::size()]; - - using mask_type = typename ViewType::simd::mask_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = - (index.mask(std::size_t(lane))) - ? field.assembly_index_mapping( - field.index_mapping(index.ispec + lane, index.iz, index.ix), - static_cast(MediumType)) - : field.nglob + 1; - } - - impl_add_on_device(mask, iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_add_on_host(const specfem::point::index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = field.h_assembly_index_mapping( - field.h_index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); - - impl_add_on_host(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_add_on_host(const specfem::point::assembly_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = index.iglob; - - impl_add_on_host(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -inline void impl_add_on_host( - const specfem::point::simd_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - int iglob[ViewType::simd::size()]; - - using mask_type = typename ViewType::simd::mask_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = - (index.mask(std::size_t(lane))) - ? field.h_index_mapping(index.ispec + lane, index.iz, index.ix) - : field.nglob + 1; - } - - impl_add_on_host(mask, iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_atomic_add_on_device(const int iglob, const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - - const auto &curr_field = field.template get_field(); - for (int icomp = 0; icomp < ViewType::components; ++icomp) { - if constexpr (StoreDisplacement) { - Kokkos::atomic_add(&curr_field.field(iglob, icomp), - point_field.displacement(icomp)); - } - if constexpr (StoreVelocity) { - Kokkos::atomic_add(&curr_field.field_dot(iglob, icomp), - point_field.velocity(icomp)); - } - if constexpr (StoreAcceleration) { - Kokkos::atomic_add(&curr_field.field_dot_dot(iglob, icomp), - point_field.acceleration(icomp)); - } - if constexpr (StoreMassMatrix) { - Kokkos::atomic_add(&curr_field.mass_inverse(iglob, icomp), - point_field.mass_matrix(icomp)); - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_atomic_add_on_device(const typename ViewType::simd::mask_type &mask, - const int *iglob, const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - const auto &curr_field = field.template get_field(); - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - if (!mask[lane]) { - continue; - } - - const int iglob_l = iglob[lane]; - - if constexpr (StoreDisplacement) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::atomic_add(&curr_field.field(iglob_l, icomp), - point_field.displacement(icomp)[lane]); - } - } - - if constexpr (StoreVelocity) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::atomic_add(&curr_field.field_dot(iglob_l, icomp), - point_field.velocity(icomp)[lane]); - } - } - - if constexpr (StoreAcceleration) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::atomic_add(&curr_field.field_dot_dot(iglob_l, icomp), - point_field.acceleration(icomp)[lane]); - } - } - - if constexpr (StoreMassMatrix) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::atomic_add(&curr_field.mass_inverse(iglob_l, icomp), - point_field.mass_matrix(icomp)[lane]); - } - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_atomic_add_on_host(const int iglob, const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - - const auto &curr_field = field.template get_field(); - for (int icomp = 0; icomp < ViewType::components; ++icomp) { - if constexpr (StoreDisplacement) { - Kokkos::atomic_add(&curr_field.h_field(iglob, icomp), - point_field.displacement(icomp)); - } - if constexpr (StoreVelocity) { - Kokkos::atomic_add(&curr_field.h_field_dot(iglob, icomp), - point_field.velocity(icomp)); - } - if constexpr (StoreAcceleration) { - Kokkos::atomic_add(&curr_field.h_field_dot_dot(iglob, icomp), - point_field.acceleration(icomp)); - } - if constexpr (StoreMassMatrix) { - Kokkos::atomic_add(&curr_field.h_mass_inverse(iglob, icomp), - point_field.mass_matrix(icomp)); - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -inline void impl_atomic_add_on_host(const typename ViewType::simd::mask_type &mask, - const int *iglob, const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - const auto &curr_field = field.template get_field(); - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - if (!mask[lane]) { - continue; - } - - const int iglob_l = iglob[lane]; - - if constexpr (StoreDisplacement) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::atomic_add(&curr_field.h_field(iglob_l, icomp), - point_field.displacement(icomp)[lane]); - } - } - - if constexpr (StoreVelocity) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::atomic_add(&curr_field.h_field_dot(iglob_l, icomp), - point_field.velocity(icomp)[lane]); - } - } - - if constexpr (StoreAcceleration) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::atomic_add(&curr_field.h_field_dot_dot(iglob_l, icomp), - point_field.acceleration(icomp)[lane]); - } - } - - if constexpr (StoreMassMatrix) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::atomic_add(&curr_field.h_mass_inverse(iglob_l, icomp), - point_field.mass_matrix(icomp)[lane]); - } - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void impl_atomic_add_on_device( - const specfem::point::index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = field.assembly_index_mapping( - field.index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); - - impl_atomic_add_on_device(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void impl_atomic_add_on_device( - const specfem::point::simd_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - int iglob[ViewType::simd::size()]; + int iglob[ViewType::simd::size()]; using mask_type = typename ViewType::simd::mask_type; mask_type mask([&](std::size_t lane) { return index.mask(lane); }); for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = - (index.mask(std::size_t(lane))) + iglob[lane] = on_device ? + ((index.mask(std::size_t(lane))) ? field.assembly_index_mapping( field.index_mapping(index.ispec + lane, index.iz, index.ix), static_cast(MediumType)) - : field.nglob + 1; - } - - impl_atomic_add_on_device(mask, iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_atomic_add_on_host( - const specfem::point::index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = field.h_assembly_index_mapping( - field.h_index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); - - impl_atomic_add_on_host(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -inline void impl_atomic_add_on_host( - const specfem::point::simd_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - int iglob[ViewType::simd::size()]; - - using mask_type = typename ViewType::simd::mask_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = - (index.mask(std::size_t(lane))) - ? field.h_index_mapping(index.ispec + lane, index.iz, index.ix) - : field.nglob + 1; + : field.nglob + 1): + ((index.mask(std::size_t(lane))) + ? field.assembly_index_mapping( + field.index_mapping(index.ispec + lane, index.iz, index.ix), + static_cast(MediumType)) + : field.nglob + 1); } - impl_atomic_add_on_host(mask, iglob, point_field, field); + impl_atomic_add(mask, iglob, point_field, field); } -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_load_on_device(const MemberType &team, const int ispec, +impl_load(const MemberType &team, const int ispec, const WavefieldType &field, ViewType &element_field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; @@ -1554,85 +822,40 @@ impl_load_on_device(const MemberType &team, const int ispec, constexpr static int NGLL = ViewType::ngll; - static_assert(std::is_same_v, - "Calling team must have a device execution space"); - - const auto &curr_field = field.template get_field(); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, NGLL * NGLL), [&](const int &xz) { - int iz, ix; - sub2ind(xz, NGLL, iz, ix); - const int iglob = field.assembly_index_mapping( - field.index_mapping(ispec, iz, ix), static_cast(MediumType)); - - for (int icomp = 0; icomp < components; ++icomp) { - if constexpr (StoreDisplacement) { - element_field.displacement(iz, ix, icomp) = - curr_field.field(iglob, icomp); - } - if constexpr (StoreVelocity) { - element_field.velocity(iz, ix, icomp) = - curr_field.field_dot(iglob, icomp); - } - if constexpr (StoreAcceleration) { - element_field.acceleration(iz, ix, icomp) = - curr_field.field_dot_dot(iglob, icomp); - } - if constexpr (StoreMassMatrix) { - element_field.mass_matrix(iz, ix, icomp) = - curr_field.mass_inverse(iglob, icomp); - } - } - }); - - return; -} - -template < - typename MemberType, typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isElementFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_load_on_host(const MemberType &team, const int ispec, - const WavefieldType &field, ViewType &element_field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - constexpr static int NGLL = ViewType::ngll; - - static_assert( - std::is_same_v, - "Calling team must have a host execution space"); + if constexpr (on_device) { + static_assert(std::is_same_v, + "Calling team must have a device execution space"); + } + else { + static_assert(std::is_same_v, + "Calling team must have a device execution space"); + } const auto &curr_field = field.template get_field(); Kokkos::parallel_for( Kokkos::TeamThreadRange(team, NGLL * NGLL), [&](const int &xz) { int iz, ix; sub2ind(xz, NGLL, iz, ix); - const int iglob = field.h_assembly_index_mapping( - field.h_index_mapping(ispec, iz, ix), static_cast(MediumType)); + const int iglob = on_device ? field.assembly_index_mapping( + field.index_mapping(ispec, iz, ix), static_cast(MediumType)) : field.h_assembly_index_mapping( + field.h_index_mapping(ispec, iz, ix), static_cast(MediumType)); for (int icomp = 0; icomp < components; ++icomp) { if constexpr (StoreDisplacement) { element_field.displacement(iz, ix, icomp) = - curr_field.h_field(iglob, icomp); + curr_field.template get_field(iglob, icomp); } if constexpr (StoreVelocity) { element_field.velocity(iz, ix, icomp) = - curr_field.h_field_dot(iglob, icomp); + curr_field.template get_field_dot(iglob, icomp); } if constexpr (StoreAcceleration) { element_field.acceleration(iz, ix, icomp) = - curr_field.h_field_dot_dot(iglob, icomp); + curr_field.template get_field_dot_dot(iglob, icomp); } if constexpr (StoreMassMatrix) { element_field.mass_matrix(iz, ix, icomp) = - curr_field.h_mass_inverse(iglob, icomp); + curr_field.template get_mass_inverse(iglob, icomp); } } }); @@ -1640,13 +863,13 @@ inline void impl_load_on_host(const MemberType &team, const int ispec, return; } -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_load_on_device(const MemberType &team, const IteratorType &iterator, +impl_load(const MemberType &team, const IteratorType &iterator, const WavefieldType &field, ViewType &chunk_field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; @@ -1663,9 +886,14 @@ impl_load_on_device(const MemberType &team, const IteratorType &iterator, std::is_same_v, "Iterator and View must have the same simd type"); - static_assert(std::is_same_v, - "Calling team must have a device execution space"); + if constexpr (on_device) { + static_assert(std::is_same_v, + "Calling team must have a device execution space"); + } + else { + static_assert(std::is_same_v, + "Calling team must have a device execution space"); + } static_assert( Kokkos::SpaceAccessibility(MediumType)); + const int iglob = on_device ? field.assembly_index_mapping( + field.index_mapping(ispec, iz, ix), static_cast(MediumType)) : field.h_assembly_index_mapping( + field.h_index_mapping(ispec, iz, ix), static_cast(MediumType)); for (int icomp = 0; icomp < components; ++icomp) { if constexpr (StoreDisplacement) { chunk_field.displacement(ielement, iz, ix, icomp) = - curr_field.field(iglob, icomp); + curr_field.template get_field(iglob, icomp); } if constexpr (StoreVelocity) { chunk_field.velocity(ielement, iz, ix, icomp) = - curr_field.field_dot(iglob, icomp); + curr_field.template get_field_dot(iglob, icomp); } if constexpr (StoreAcceleration) { chunk_field.acceleration(ielement, iz, ix, icomp) = - curr_field.field_dot_dot(iglob, icomp); + curr_field.template get_field_dot_dot(iglob, icomp); } if constexpr (StoreMassMatrix) { chunk_field.mass_matrix(ielement, iz, ix, icomp) = - curr_field.mass_inverse(iglob, icomp); + curr_field.template get_mass_inverse(iglob, icomp); } } }); @@ -1707,13 +936,13 @@ impl_load_on_device(const MemberType &team, const IteratorType &iterator, return; } -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_load_on_device(const MemberType &team, const IteratorType &iterator, +impl_load(const MemberType &team, const IteratorType &iterator, const WavefieldType &field, ViewType &chunk_field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; @@ -1734,9 +963,14 @@ impl_load_on_device(const MemberType &team, const IteratorType &iterator, typename ViewType::memory_space>::accessible, "Calling team must have access to the memory space of the view"); - static_assert(std::is_same_v, - "Calling team must have a device execution space"); + if constexpr (on_device) { + static_assert(std::is_same_v, + "Calling team must have a device execution space"); + } + else { + static_assert(std::is_same_v, + "Calling team must have a device execution space"); + } const auto &curr_field = field.template get_field(); Kokkos::parallel_for( @@ -1752,162 +986,28 @@ impl_load_on_device(const MemberType &team, const IteratorType &iterator, continue; } - const int iglob = field.assembly_index_mapping( + const int iglob = on_device ? field.assembly_index_mapping( field.index_mapping(ispec + lane, iz, ix), - static_cast(MediumType)); - - for (int icomp = 0; icomp < components; ++icomp) { - if constexpr (StoreDisplacement) { - chunk_field.displacement(ielement, iz, ix, icomp)[lane] = - curr_field.field(iglob, icomp); - } - if constexpr (StoreVelocity) { - chunk_field.velocity(ielement, iz, ix, icomp)[lane] = - curr_field.field_dot(iglob, icomp); - } - if constexpr (StoreAcceleration) { - chunk_field.acceleration(ielement, iz, ix, icomp)[lane] = - curr_field.field_dot_dot(iglob, icomp); - } - if constexpr (StoreMassMatrix) { - chunk_field.mass_matrix(ielement, iz, ix, icomp)[lane] = - curr_field.mass_inverse(iglob, icomp); - } - } - } - }); - - return; -} - -template < - typename MemberType, typename WavefieldType, typename IteratorType, - typename ViewType, - typename std::enable_if_t< - ViewType::isChunkFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_load_on_host(const MemberType &team, const IteratorType &iterator, - const WavefieldType &field, ViewType &chunk_field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - constexpr static int NGLL = ViewType::ngll; - - static_assert( - std::is_same_v, - "Calling team must have a host execution space"); - - static_assert( - std::is_same_v, - "Iterator and View must have the same simd type"); - - static_assert( - Kokkos::SpaceAccessibility::accessible, - "Calling team must have access to the memory space of the view"); - - const auto &curr_field = field.template get_field(); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, iterator.chunk_size()), [&](const int &i) { - const auto iterator_index = iterator(i); - const int ielement = iterator_index.ielement; - const int ispec = iterator_index.index.ispec; - const int iz = iterator_index.index.iz; - const int ix = iterator_index.index.ix; - - const int iglob = field.h_assembly_index_mapping( - field.h_index_mapping(ispec, iz, ix), static_cast(MediumType)); - - for (int icomp = 0; icomp < components; ++icomp) { - if constexpr (StoreDisplacement) { - chunk_field.displacement(ielement, iz, ix, icomp) = - curr_field.h_field(iglob, icomp); - } - if constexpr (StoreVelocity) { - chunk_field.velocity(ielement, iz, ix, icomp) = - curr_field.h_field_dot(iglob, icomp); - } - if constexpr (StoreAcceleration) { - chunk_field.acceleration(ielement, iz, ix, icomp); - } - if constexpr (StoreMassMatrix) { - chunk_field.mass_matrix(ielement, iz, ix, icomp) = - curr_field.h_mass_inverse(iglob, icomp); - } - } - }); - - return; -} - -template < - typename MemberType, typename WavefieldType, typename IteratorType, - typename ViewType, - typename std::enable_if_t< - ViewType::isChunkFieldType && ViewType::simd::using_simd, int> = 0> -inline void impl_load_on_host(const MemberType &team, const IteratorType &iterator, - const WavefieldType &field, ViewType &chunk_field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - constexpr static int NGLL = ViewType::ngll; - - static_assert( - std::is_same_v, - "Calling team must have a host execution space"); - - static_assert( - std::is_same_v, - "Iterator and View must have the same simd type"); - - static_assert( - Kokkos::SpaceAccessibility::accessible, - "Calling team must have access to the memory space of the view"); - - const auto &curr_field = field.template get_field(); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, iterator.chunk_size()), [&](const int &i) { - const auto iterator_index = iterator(i); - const int ielement = iterator_index.ielement; - const int ispec = iterator_index.index.ispec; - const int iz = iterator_index.index.iz; - const int ix = iterator_index.index.ix; - - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - if (!iterator_index.index.mask(lane)) { - continue; - } - - const int iglob = field.h_assembly_index_mapping( - field.h_index_mapping(ispec + lane, iz, ix), - static_cast(MediumType)); + static_cast(MediumType)) : field.h_assembly_index_mapping( + field.h_index_mapping(ispec + lane, iz, ix), + static_cast(MediumType)); for (int icomp = 0; icomp < components; ++icomp) { if constexpr (StoreDisplacement) { chunk_field.displacement(ielement, iz, ix, icomp)[lane] = - curr_field.h_field(iglob, icomp); + curr_field.template get_field(iglob, icomp); } if constexpr (StoreVelocity) { chunk_field.velocity(ielement, iz, ix, icomp)[lane] = - curr_field.h_field_dot(iglob, icomp); + curr_field.template get_field_dot(iglob, icomp); } if constexpr (StoreAcceleration) { chunk_field.acceleration(ielement, iz, ix, icomp)[lane] = - curr_field.h_field_dot_dot(iglob, icomp); + curr_field.template get_field_dot_dot(iglob, icomp); } if constexpr (StoreMassMatrix) { chunk_field.mass_matrix(ielement, iz, ix, icomp)[lane] = - curr_field.h_mass_inverse(iglob, icomp); + curr_field.template get_mass_inverse(iglob, icomp); } } } diff --git a/include/compute/fields/impl/field_impl.hpp b/include/compute/fields/impl/field_impl.hpp index 1da50d73..888398b5 100644 --- a/include/compute/fields/impl/field_impl.hpp +++ b/include/compute/fields/impl/field_impl.hpp @@ -1,5 +1,4 @@ -#ifndef _COMPUTE_FIELDS_IMPL_FIELD_IMPL_HPP_ -#define _COMPUTE_FIELDS_IMPL_FIELD_IMPL_HPP_ +#pragma once #include "compute/compute_mesh.hpp" #include "compute/element_types/element_types.hpp" @@ -37,6 +36,46 @@ class field_impl { specfem::kokkos::HostMirror2d h_field_dot_dot; specfem::kokkos::DeviceView2d mass_inverse; specfem::kokkos::HostMirror2d h_mass_inverse; + + template + KOKKOS_FORCEINLINE_FUNCTION constexpr type_real & + get_field(const int &iglob, const int &icomp) const { + if constexpr (on_device) { + return field(iglob, icomp); + } else { + return h_field(iglob, icomp); + } + } + + template + KOKKOS_FORCEINLINE_FUNCTION constexpr type_real & + get_field_dot(const int &iglob, const int &icomp) const { + if constexpr (on_device) { + return field_dot(iglob, icomp); + } else { + return h_field_dot(iglob, icomp); + } + } + + template + KOKKOS_FORCEINLINE_FUNCTION constexpr type_real & + get_field_dot_dot(const int &iglob, const int &icomp) const { + if constexpr (on_device) { + return field_dot_dot(iglob, icomp); + } else { + return h_field_dot_dot(iglob, icomp); + } + } + + template + KOKKOS_FORCEINLINE_FUNCTION constexpr type_real & + get_mass_inverse(const int &iglob, const int &icomp) const { + if constexpr (on_device) { + return mass_inverse(iglob, icomp); + } else { + return h_mass_inverse(iglob, icomp); + } + } }; } // namespace impl @@ -54,5 +93,3 @@ void deep_copy(impl::field_impl &dst, } // namespace compute } // namespace specfem - -#endif /* _COMPUTE_FIELDS_IMPL_FIELD_IMPL_HPP_ */ diff --git a/include/compute/fields/impl/field_impl.tpp b/include/compute/fields/impl/field_impl.tpp index ebb9db43..eb2391f2 100644 --- a/include/compute/fields/impl/field_impl.tpp +++ b/include/compute/fields/impl/field_impl.tpp @@ -1,5 +1,4 @@ -#ifndef _COMPUTE_FIELDS_IMPL_FIELD_IMPL_TPP_ -#define _COMPUTE_FIELDS_IMPL_FIELD_IMPL_TPP_ +#pragma once #include "compute/fields/impl/field_impl.hpp" #include "compute/element_types/element_types.hpp" @@ -111,7 +110,6 @@ void specfem::compute::impl::field_impl::sync_fields() } } -#endif /* _COMPUTE_FIELDS_IMPL_FIELD_IMPL_TPP_ */ // template // KOKKOS_INLINE_FUNCTION type_real &specfem::compute::(const int &iglob, diff --git a/include/compute/fields/simulation_field.hpp b/include/compute/fields/simulation_field.hpp index 2fe7fb9b..bd72ee7e 100644 --- a/include/compute/fields/simulation_field.hpp +++ b/include/compute/fields/simulation_field.hpp @@ -172,7 +172,7 @@ template (index, field, point_field); } /** @@ -195,7 +195,7 @@ template (index, field, point_field); } /** @@ -218,7 +218,7 @@ template (index, point_field, field); } /** @@ -240,7 +240,7 @@ template = 0> inline void store_on_host(const IndexType &index, const ViewType &point_field, const WavefieldContainer &field) { - impl_store_on_host(index, point_field, field); + impl_store(index, point_field, field); } /** @@ -263,7 +263,7 @@ template (index, point_field, field); } /** @@ -285,7 +285,7 @@ template = 0> inline void add_on_host(const IndexType &index, const ViewType &point_field, const WavefieldContainer &field) { - impl_add_on_host(index, point_field, field); + impl_add(index, point_field, field); } /** @@ -308,7 +308,7 @@ template (index, point_field, field); } /** @@ -331,7 +331,7 @@ template (index, point_field, field); } /** @@ -353,7 +353,7 @@ template (member, index, field, element_field); } /** @@ -375,7 +375,7 @@ template (member, index, field, element_field); } /** @@ -400,7 +400,7 @@ template (member, iterator, field, chunk_field); } /** @@ -425,7 +425,7 @@ template (member, iterator, field, chunk_field); } } // namespace compute diff --git a/include/medium/properties_container.hpp b/include/medium/properties_container.hpp index 9a762252..94ae23aa 100644 --- a/include/medium/properties_container.hpp +++ b/include/medium/properties_container.hpp @@ -42,10 +42,10 @@ struct impl_properties_container { h_data(Kokkos::create_mirror_view(data)) {} private: - template + template KOKKOS_FORCEINLINE_FUNCTION void load_properties(const specfem::point::index &index, - PointProperties &property, const ViewType &target) const { + PointProperties &property) const { static_assert(PointProperties::dimension == dimension, "Dimension mismatch"); @@ -59,13 +59,15 @@ struct impl_properties_container { const int ix = index.ix; for (int i = 0; i < nprops; i++) { - property.data[i] = target(ispec, iz, ix, i); + property.data[i] = + on_device ? data(ispec, iz, ix, i) : h_data(ispec, iz, ix, i); } } - template + + template KOKKOS_FORCEINLINE_FUNCTION void load_properties(const specfem::point::simd_index &index, - PointProperties &property, const ViewType &target) const { + PointProperties &property) const { static_assert(PointProperties::dimension == dimension, "Dimension mismatch"); @@ -86,7 +88,9 @@ struct impl_properties_container { for (int i = 0; i < nprops; i++) { Kokkos::Experimental::where(mask, property.data[i]) - .copy_from(&target(ispec, iz, ix, i), tag_type()); + .copy_from(on_device ? &data(ispec, iz, ix, i) + : &h_data(ispec, iz, ix, i), + tag_type()); } } @@ -95,14 +99,14 @@ struct impl_properties_container { KOKKOS_FORCEINLINE_FUNCTION void load_device_properties(const IndexType &index, PointProperties &property) const { - load_properties(index, property, data); + load_properties(index, property); } template KOKKOS_FORCEINLINE_FUNCTION void load_host_properties(const IndexType &index, PointProperties &property) const { - load_properties(index, property, h_data); + load_properties(index, property); } template From 635457714e2989d49d73d3e1f917260d0b767546 Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Thu, 27 Feb 2025 09:43:12 -0500 Subject: [PATCH 06/21] Rename medium::properties to medium::impl_material --- .../{properties.hpp => material.hpp} | 24 +++++++------- .../{properties.hpp => material.hpp} | 33 ++++++++++--------- .../{properties.hpp => material.hpp} | 22 ++++++------- .../{properties.hpp => impl/material.hpp} | 2 +- include/medium/material.hpp | 11 +++---- 5 files changed, 47 insertions(+), 45 deletions(-) rename include/medium/dim2/acoustic/isotropic/{properties.hpp => material.hpp} (82%) rename include/medium/dim2/elastic/anisotropic/{properties.hpp => material.hpp} (83%) rename include/medium/dim2/elastic/isotropic/{properties.hpp => material.hpp} (86%) rename include/medium/{properties.hpp => impl/material.hpp} (95%) diff --git a/include/medium/dim2/acoustic/isotropic/properties.hpp b/include/medium/dim2/acoustic/isotropic/material.hpp similarity index 82% rename from include/medium/dim2/acoustic/isotropic/properties.hpp rename to include/medium/dim2/acoustic/isotropic/material.hpp index de132117..9d98a760 100644 --- a/include/medium/dim2/acoustic/isotropic/properties.hpp +++ b/include/medium/dim2/acoustic/isotropic/material.hpp @@ -1,7 +1,7 @@ #pragma once #include "enumerations/medium.hpp" -#include "medium/properties.hpp" +#include "medium/impl/material.hpp" #include "point/properties.hpp" #include "specfem_setup.hpp" #include @@ -17,8 +17,8 @@ namespace medium { * */ template <> -class properties { +class impl_material { public: constexpr static auto dimension = @@ -41,9 +41,9 @@ class propertiesQkappa <= 0.0 || this->Qmu <= 0.0) { @@ -59,8 +59,8 @@ class properties + bool operator==(const impl_material &other) const { return (std::abs(this->density - other.density) < 1e-6 && @@ -76,8 +76,8 @@ class properties + bool operator!=(const impl_material &other) const { return !(*this == other); } @@ -86,10 +86,10 @@ class properties @@ -17,8 +17,8 @@ namespace medium { * */ template <> -class properties { +class impl_material { public: constexpr static auto dimension = specfem::dimension::type::dim2; ///< Dimension of the material @@ -46,11 +46,12 @@ class properties - &other) const { + bool operator==( + const impl_material &other) + const { return (std::abs(this->density - other.density) < 1e-6 && std::abs(this->c11 - other.c11) < 1e-6 && std::abs(this->c13 - other.c13) < 1e-6 && @@ -100,9 +102,10 @@ class properties - &other) const { + bool operator!=( + const impl_material &other) + const { return !(*this == other); } diff --git a/include/medium/dim2/elastic/isotropic/properties.hpp b/include/medium/dim2/elastic/isotropic/material.hpp similarity index 86% rename from include/medium/dim2/elastic/isotropic/properties.hpp rename to include/medium/dim2/elastic/isotropic/material.hpp index 6dc7162d..b420d518 100644 --- a/include/medium/dim2/elastic/isotropic/properties.hpp +++ b/include/medium/dim2/elastic/isotropic/material.hpp @@ -1,7 +1,7 @@ #pragma once #include "enumerations/medium.hpp" -#include "medium/properties.hpp" +#include "medium/impl/material.hpp" #include "point/properties.hpp" #include "specfem_setup.hpp" #include @@ -17,8 +17,8 @@ namespace medium { * */ template <> -class properties { +class impl_material { public: constexpr static auto dimension = specfem::dimension::type::dim2; ///< Dimension of the material @@ -41,9 +41,9 @@ class properties + bool operator==(const impl_material &other) const { return (std::abs(this->density - other.density) < 1e-6 && @@ -91,8 +91,8 @@ class properties + bool operator!=(const impl_material &other) const { return !(*this == other); } diff --git a/include/medium/properties.hpp b/include/medium/impl/material.hpp similarity index 95% rename from include/medium/properties.hpp rename to include/medium/impl/material.hpp index 33d3d44d..ea483e29 100644 --- a/include/medium/properties.hpp +++ b/include/medium/impl/material.hpp @@ -12,6 +12,6 @@ namespace medium { */ template -class properties; +class impl_material; } // namespace medium } // namespace specfem diff --git a/include/medium/material.hpp b/include/medium/material.hpp index cc41078a..f2ae95b2 100644 --- a/include/medium/material.hpp +++ b/include/medium/material.hpp @@ -1,11 +1,10 @@ #pragma once -#include "dim2/acoustic/isotropic/properties.hpp" -#include "dim2/elastic/anisotropic/properties.hpp" -#include "dim2/elastic/isotropic/properties.hpp" +#include "dim2/acoustic/isotropic/material.hpp" +#include "dim2/elastic/anisotropic/material.hpp" +#include "dim2/elastic/isotropic/material.hpp" #include "enumerations/specfem_enums.hpp" #include "point/properties.hpp" -#include "properties.hpp" #include "specfem_setup.hpp" #include #include @@ -21,7 +20,7 @@ namespace medium { */ template -class material : public specfem::medium::properties { +class material : public specfem::medium::impl_material { public: constexpr static auto medium_tag = MediumTag; ///< Medium tag constexpr static auto property_tag = PropertyTag; ///< Property tag @@ -45,7 +44,7 @@ class material : public specfem::medium::properties { */ template material(Args &&...args) - : specfem::medium::properties( + : specfem::medium::impl_material( std::forward(args)...) {} ///@} From 185f6db20344eac59df046c702e70a961ba57019 Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Thu, 27 Feb 2025 10:17:06 -0500 Subject: [PATCH 07/21] Move impl/material.hpp --- include/medium/impl/material.hpp | 17 ----------------- include/medium/material.hpp | 19 +++++++++++++++---- 2 files changed, 15 insertions(+), 21 deletions(-) delete mode 100644 include/medium/impl/material.hpp diff --git a/include/medium/impl/material.hpp b/include/medium/impl/material.hpp deleted file mode 100644 index ea483e29..00000000 --- a/include/medium/impl/material.hpp +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include "enumerations/medium.hpp" - -namespace specfem { -namespace medium { -/** - * @brief Material properties for a given medium and property - * - * @tparam MediumTag Medium tag for the material - * @tparam PropertyTag Property tag for the material - */ -template -class impl_material; -} // namespace medium -} // namespace specfem diff --git a/include/medium/material.hpp b/include/medium/material.hpp index f2ae95b2..302cfa1e 100644 --- a/include/medium/material.hpp +++ b/include/medium/material.hpp @@ -1,8 +1,5 @@ #pragma once -#include "dim2/acoustic/isotropic/material.hpp" -#include "dim2/elastic/anisotropic/material.hpp" -#include "dim2/elastic/isotropic/material.hpp" #include "enumerations/specfem_enums.hpp" #include "point/properties.hpp" #include "specfem_setup.hpp" @@ -20,7 +17,17 @@ namespace medium { */ template -class material : public specfem::medium::impl_material { +class impl_material; + +/** + * @brief Material properties for a given medium and property + * + * @tparam MediumTag Medium tag for the material + * @tparam PropertyTag Property tag for the material + */ +template +class material : public impl_material { public: constexpr static auto medium_tag = MediumTag; ///< Medium tag constexpr static auto property_tag = PropertyTag; ///< Property tag @@ -62,3 +69,7 @@ class material : public specfem::medium::impl_material { } // namespace medium } // namespace specfem + +#include "dim2/acoustic/isotropic/material.hpp" +#include "dim2/elastic/anisotropic/material.hpp" +#include "dim2/elastic/isotropic/material.hpp" From 3a1c36bb0c5145621e34447e537929c3e2a7ca1d Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Thu, 27 Feb 2025 10:23:34 -0500 Subject: [PATCH 08/21] Merge material_properties and properties_container --- include/compute/properties/interface.hpp | 1 - include/compute/properties/properties.hpp | 7 +-- .../dim2/acoustic/isotropic/material.hpp | 1 - .../dim2/elastic/anisotropic/material.hpp | 1 - .../dim2/elastic/isotropic/material.hpp | 1 - include/medium/material_properties.hpp | 62 ------------------- include/medium/properties_container.hpp | 37 +++++++++++ include/mesh/materials/materials.hpp | 2 +- include/mesh/materials/materials.tpp | 2 +- src/IO/mesh.cpp | 6 +- src/compute/compute_properties.cpp | 6 +- src/mesh/materials/materials.cpp | 6 +- 12 files changed, 51 insertions(+), 81 deletions(-) delete mode 100644 include/medium/material_properties.hpp diff --git a/include/compute/properties/interface.hpp b/include/compute/properties/interface.hpp index d7ae8a21..5129b34a 100644 --- a/include/compute/properties/interface.hpp +++ b/include/compute/properties/interface.hpp @@ -1,5 +1,4 @@ #pragma once -#include "medium/material_properties.hpp" #include "medium/properties_container.hpp" #include "properties.hpp" diff --git a/include/compute/properties/properties.hpp b/include/compute/properties/properties.hpp index 2addf56a..85f97ccd 100644 --- a/include/compute/properties/properties.hpp +++ b/include/compute/properties/properties.hpp @@ -6,7 +6,6 @@ #include "kokkos_abstractions.h" #include "macros.hpp" #include "medium/material.hpp" -#include "medium/material_properties.hpp" #include "medium/properties_container.hpp" #include "point/coordinates.hpp" #include "specfem_setup.hpp" @@ -23,7 +22,7 @@ namespace compute { * */ struct properties - : public impl::value_containers { + : public impl::value_containers { /** * @name Constructors */ @@ -59,12 +58,12 @@ struct properties */ void copy_to_host() { impl::value_containers< - specfem::medium::material_properties>::copy_to_host(); + specfem::medium::properties_container>::copy_to_host(); } void copy_to_device() { impl::value_containers< - specfem::medium::material_properties>::copy_to_device(); + specfem::medium::properties_container>::copy_to_device(); } }; diff --git a/include/medium/dim2/acoustic/isotropic/material.hpp b/include/medium/dim2/acoustic/isotropic/material.hpp index 9d98a760..4249d721 100644 --- a/include/medium/dim2/acoustic/isotropic/material.hpp +++ b/include/medium/dim2/acoustic/isotropic/material.hpp @@ -1,7 +1,6 @@ #pragma once #include "enumerations/medium.hpp" -#include "medium/impl/material.hpp" #include "point/properties.hpp" #include "specfem_setup.hpp" #include diff --git a/include/medium/dim2/elastic/anisotropic/material.hpp b/include/medium/dim2/elastic/anisotropic/material.hpp index 9a850635..1472157e 100644 --- a/include/medium/dim2/elastic/anisotropic/material.hpp +++ b/include/medium/dim2/elastic/anisotropic/material.hpp @@ -1,7 +1,6 @@ #pragma once #include "enumerations/specfem_enums.hpp" -#include "medium/impl/material.hpp" #include "point/properties.hpp" #include "specfem_setup.hpp" #include diff --git a/include/medium/dim2/elastic/isotropic/material.hpp b/include/medium/dim2/elastic/isotropic/material.hpp index b420d518..c7f02dda 100644 --- a/include/medium/dim2/elastic/isotropic/material.hpp +++ b/include/medium/dim2/elastic/isotropic/material.hpp @@ -1,7 +1,6 @@ #pragma once #include "enumerations/medium.hpp" -#include "medium/impl/material.hpp" #include "point/properties.hpp" #include "specfem_setup.hpp" #include diff --git a/include/medium/material_properties.hpp b/include/medium/material_properties.hpp deleted file mode 100644 index 5c146cb1..00000000 --- a/include/medium/material_properties.hpp +++ /dev/null @@ -1,62 +0,0 @@ -#pragma once - -#include "compute/compute_mesh.hpp" -#include "enumerations/interface.hpp" -#include "medium/properties_container.hpp" -#include "mesh/materials/materials.hpp" -#include "mesh/tags/tags.hpp" - -namespace specfem { -namespace medium { - -template -struct material_properties - : public specfem::medium::properties_container { - constexpr static auto value_type = type; - constexpr static auto property_type = property; - constexpr static auto dimension = specfem::dimension::type::dim2; - - material_properties() = default; - - material_properties( - const Kokkos::View elements, - const int ngllz, const int ngllx, - const specfem::mesh::materials &materials, const bool has_gll_model, - const specfem::kokkos::HostView1d property_index_mapping) - : specfem::medium::properties_container( - elements.extent(0), ngllz, ngllx) { - - const int nelement = elements.extent(0); - int count = 0; - for (int i = 0; i < nelement; ++i) { - const int ispec = elements(i); - property_index_mapping(ispec) = count; - if (!has_gll_model) { - for (int iz = 0; iz < ngllz; ++iz) { - for (int ix = 0; ix < ngllx; ++ix) { - // Get the material at index from mesh::materials - auto material = - std::get >( - materials[ispec]); - - // Assign the material property to the property container - auto point_property = material.get_properties(); - this->assign(specfem::point::index(count, iz, ix), - point_property); - } - } - } - count++; - } - - if (!has_gll_model) { - this->copy_to_device(); - } - - return; - } -}; - -} // namespace medium -} // namespace specfem diff --git a/include/medium/properties_container.hpp b/include/medium/properties_container.hpp index 94ae23aa..f7e2e333 100644 --- a/include/medium/properties_container.hpp +++ b/include/medium/properties_container.hpp @@ -41,6 +41,43 @@ struct impl_properties_container { data("specfem::benchmarks::properties::data", nspec, ngllz, ngllx, N), h_data(Kokkos::create_mirror_view(data)) {} + impl_properties_container( + const Kokkos::View elements, + const int ngllz, const int ngllx, + const specfem::mesh::materials &materials, const bool has_gll_model, + const specfem::kokkos::HostView1d property_index_mapping) + : impl_properties_container(elements.extent(0), ngllz, ngllx) { + + const int nelement = elements.extent(0); + int count = 0; + for (int i = 0; i < nelement; ++i) { + const int ispec = elements(i); + property_index_mapping(ispec) = count; + if (!has_gll_model) { + for (int iz = 0; iz < ngllz; ++iz) { + for (int ix = 0; ix < ngllx; ++ix) { + // Get the material at index from mesh::materials + auto material = + std::get >( + materials[ispec]); + + // Assign the material property to the property container + auto point_property = material.get_properties(); + this->assign(specfem::point::index(count, iz, ix), + point_property); + } + } + } + count++; + } + + if (!has_gll_model) { + this->copy_to_device(); + } + + return; + } + private: template KOKKOS_FORCEINLINE_FUNCTION void diff --git a/include/mesh/materials/materials.hpp b/include/mesh/materials/materials.hpp index 06f447f1..fa500572 100644 --- a/include/mesh/materials/materials.hpp +++ b/include/mesh/materials/materials.hpp @@ -44,7 +44,7 @@ struct materials { struct material { int n_materials; ///< Number of elements std::vector > - material_properties; ///< Material properties + element_materials; ///< Material properties material() = default; diff --git a/include/mesh/materials/materials.tpp b/include/mesh/materials/materials.tpp index 867ed038..66259400 100644 --- a/include/mesh/materials/materials.tpp +++ b/include/mesh/materials/materials.tpp @@ -14,6 +14,6 @@ specfem::mesh::materials::material::material( const std::vector > &l_materials) : n_materials(n_materials), - material_properties(l_materials) {} + element_materials(l_materials) {} #endif /* end of include guard: _MESH_MATERIALS_TPP */ diff --git a/src/IO/mesh.cpp b/src/IO/mesh.cpp index 29021be5..8a10399c 100644 --- a/src/IO/mesh.cpp +++ b/src/IO/mesh.cpp @@ -194,12 +194,12 @@ specfem::IO::read_mesh(const std::string filename, std::to_string(mesh.materials.n_materials) + "\n\n"); const auto l_elastic_isotropic = - mesh.materials.elastic_isotropic.material_properties; + mesh.materials.elastic_isotropic.element_materials; const auto l_acoustic_isotropic = - mesh.materials.acoustic_isotropic.material_properties; + mesh.materials.acoustic_isotropic.element_materials; const auto l_elastic_anisotropic = - mesh.materials.elastic_anisotropic.material_properties; + mesh.materials.elastic_anisotropic.element_materials; for (const auto material : l_elastic_isotropic) { mpi->cout(material.print()); diff --git a/src/compute/compute_properties.cpp b/src/compute/compute_properties.cpp index f21b2c0f..51b1eeb9 100644 --- a/src/compute/compute_properties.cpp +++ b/src/compute/compute_properties.cpp @@ -31,19 +31,19 @@ specfem::compute::properties::properties( h_property_index_mapping(ispec) = -1; } - acoustic_isotropic = specfem::medium::material_properties< + acoustic_isotropic = specfem::medium::properties_container< specfem::element::medium_tag::acoustic, specfem::element::property_tag::isotropic>( acoustic_elements, ngllz, ngllx, materials, has_gll_model, h_property_index_mapping); - elastic_isotropic = specfem::medium::material_properties< + elastic_isotropic = specfem::medium::properties_container< specfem::element::medium_tag::elastic, specfem::element::property_tag::isotropic>( elastic_isotropic_elements, ngllz, ngllx, materials, has_gll_model, h_property_index_mapping); - elastic_anisotropic = specfem::medium::material_properties< + elastic_anisotropic = specfem::medium::properties_container< specfem::element::medium_tag::elastic, specfem::element::property_tag::anisotropic>( elastic_anisotropic_elements, ngllz, ngllx, materials, has_gll_model, diff --git a/src/mesh/materials/materials.cpp b/src/mesh/materials/materials.cpp index d2fb9ba0..6eeac712 100644 --- a/src/mesh/materials/materials.cpp +++ b/src/mesh/materials/materials.cpp @@ -22,7 +22,7 @@ specfem::mesh::materials::operator[](const int index) const { material_specification.property == specfem::element::property_tag::isotropic) { return this->elastic_isotropic - .material_properties[material_specification.index]; + .element_materials[material_specification.index]; // Return elastic anisotropic } else if (material_specification.type == @@ -30,7 +30,7 @@ specfem::mesh::materials::operator[](const int index) const { material_specification.property == specfem::element::property_tag::anisotropic) { return this->elastic_anisotropic - .material_properties[material_specification.index]; + .element_materials[material_specification.index]; // Return acoustic isotropic } else if (material_specification.type == @@ -38,7 +38,7 @@ specfem::mesh::materials::operator[](const int index) const { material_specification.property == specfem::element::property_tag::isotropic) { return this->acoustic_isotropic - .material_properties[material_specification.index]; + .element_materials[material_specification.index]; // Throw an error if the material type is not supported } else { throw std::runtime_error("Material type not supported"); From 222ad0ceac78cc4dd3f9da8be0b4d5a5fdc0f2c2 Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Thu, 27 Feb 2025 14:00:13 -0500 Subject: [PATCH 09/21] Update view_type for properties_container. --- include/medium/properties_container.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/medium/properties_container.hpp b/include/medium/properties_container.hpp index 9a762252..89ac1135 100644 --- a/include/medium/properties_container.hpp +++ b/include/medium/properties_container.hpp @@ -20,7 +20,7 @@ namespace impl { template struct impl_properties_container { - using view_type = typename Kokkos::View; constexpr static auto nprops = N; constexpr static auto dimension = specfem::dimension::type::dim2; @@ -32,7 +32,7 @@ struct impl_properties_container { int ngllx; ///< number of quadrature points in x dimension view_type data; - view_type::HostMirror h_data; + typename view_type::HostMirror h_data; impl_properties_container() = default; From 63e426ed2a89dca1e3f4139cf06240a926ab98a5 Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Thu, 27 Feb 2025 14:39:24 -0500 Subject: [PATCH 10/21] Move impl_properties to impl/values --- include/point/impl/values.hpp | 74 +++++++++++++++++++ include/point/properties.hpp | 130 +++++++++------------------------- 2 files changed, 108 insertions(+), 96 deletions(-) create mode 100644 include/point/impl/values.hpp diff --git a/include/point/impl/values.hpp b/include/point/impl/values.hpp new file mode 100644 index 00000000..4b43e98a --- /dev/null +++ b/include/point/impl/values.hpp @@ -0,0 +1,74 @@ +#pragma once + +#include "datatypes/simd.hpp" +#include "enumerations/medium.hpp" + +#define DEFINE_POINT_VALUE(prop) \ + constexpr static int i_##prop = __COUNTER__ - _counter - 1; \ + KOKKOS_INLINE_FUNCTION constexpr value_type prop() const { \ + return base_type::data[i_##prop]; \ + } \ + KOKKOS_INLINE_FUNCTION constexpr void prop(value_type &val) { \ + base_type::data[i_##prop] = val; \ + } + +namespace specfem { +namespace point { + +namespace impl { +template struct values { + using simd = specfem::datatype::simd; ///< SIMD type + using value_type = typename simd::datatype; + constexpr static auto nprops = N; + + value_type data[N]; + + KOKKOS_FUNCTION + values() = default; + + /** + * @brief array constructor + * + */ + KOKKOS_FUNCTION + values(const value_type *value) { + for (int i = 0; i < N; ++i) { + data[i] = value[i]; + } + } + + /** + * @brief value constructor + * + */ + template = 0> + values(Args... args) : data{ args... } {} + + /** + * @brief Equality operator + * + */ + KOKKOS_FUNCTION + bool operator==(const values &rhs) const { + for (int i = 0; i < N; ++i) { + if (data[i] != rhs.data[i]) { + return false; + } + } + return true; + } + + /** + * @brief Inequality operator + * + */ + KOKKOS_FUNCTION + bool operator!=(const values &rhs) const { + return !(*this == rhs); + } +}; +} // namespace impl + +} // namespace point +} // namespace specfem diff --git a/include/point/properties.hpp b/include/point/properties.hpp index 34befd8b..f88f58cd 100644 --- a/include/point/properties.hpp +++ b/include/point/properties.hpp @@ -1,76 +1,10 @@ #pragma once -#include "datatypes/simd.hpp" -#include "enumerations/medium.hpp" - -#define DEFINE_PROP(prop) \ - constexpr static int i_##prop = __COUNTER__ - _counter - 1; \ - KOKKOS_INLINE_FUNCTION constexpr value_type prop() const { \ - return base_type::data[i_##prop]; \ - } \ - KOKKOS_INLINE_FUNCTION constexpr void prop(value_type &val) { \ - base_type::data[i_##prop] = val; \ - } +#include "impl/values.hpp" namespace specfem { namespace point { -namespace impl { -template struct impl_properties { - using simd = specfem::datatype::simd; ///< SIMD type - using value_type = typename simd::datatype; - constexpr static bool is_point_properties = true; - constexpr static auto nprops = N; - - value_type data[N]; - - KOKKOS_FUNCTION - impl_properties() = default; - - /** - * @brief array constructor - * - */ - KOKKOS_FUNCTION - impl_properties(const value_type *value) { - for (int i = 0; i < N; ++i) { - data[i] = value[i]; - } - } - - /** - * @brief value constructor - * - */ - template = 0> - impl_properties(Args... args) : data{ args... } {} - - /** - * @brief Equality operator - * - */ - KOKKOS_FUNCTION - bool operator==(const impl_properties &rhs) const { - for (int i = 0; i < N; ++i) { - if (data[i] != rhs.data[i]) { - return false; - } - } - return true; - } - - /** - * @brief Inequality operator - * - */ - KOKKOS_FUNCTION - bool operator!=(const impl_properties &rhs) const { - return !(*this == rhs); - } -}; -} // namespace impl - /** * @brief Store properties of the medium at a quadrature point * @@ -93,28 +27,30 @@ template struct properties - : public impl::impl_properties<3, UseSIMD> { + : public impl::values<3, UseSIMD> { /** * @name Typedefs * */ ///@{ + using base_type = impl::values<3, UseSIMD>; + using value_type = typename base_type::value_type; + constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::elastic; constexpr static auto property_tag = specfem::element::property_tag::isotropic; - using simd = specfem::datatype::simd; - using value_type = typename simd::datatype; + constexpr static bool is_point_properties = true; constexpr static int _counter = __COUNTER__; ///@} - using base_type = impl::impl_properties<3, UseSIMD>; + using base_type::base_type; - DEFINE_PROP(lambdaplus2mu) ///< Lame's parameter @f$ \lambda + 2\mu @f$ - DEFINE_PROP(mu) ///< shear modulus @f$ \mu @f$ - DEFINE_PROP(rho) ///< density @f$ \rho @f$ + DEFINE_POINT_VALUE(lambdaplus2mu) ///< Lame's parameter @f$ \lambda + 2\mu @f$ + DEFINE_POINT_VALUE(mu) ///< shear modulus @f$ \mu @f$ + DEFINE_POINT_VALUE(rho) ///< density @f$ \rho @f$ KOKKOS_INLINE_FUNCTION constexpr value_type rho_vp() const { return Kokkos::sqrt(rho() * lambdaplus2mu()); ///< P-wave velocity @f$ \rho @@ -135,23 +71,25 @@ template struct properties - : public impl::impl_properties<10, UseSIMD> { + : public impl::values<10, UseSIMD> { /** * @name Typedefs * */ ///@{ + using base_type = impl::values<10, UseSIMD>; + using value_type = typename base_type::value_type; + constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::elastic; constexpr static auto property_tag = specfem::element::property_tag::anisotropic; - using simd = specfem::datatype::simd; - using value_type = typename simd::datatype; + constexpr static bool is_point_properties = true; constexpr static int _counter = __COUNTER__; ///@} - using base_type = impl::impl_properties<10, UseSIMD>; + using base_type::base_type; /** @@ -159,16 +97,16 @@ struct properties struct properties - : public impl::impl_properties<2, UseSIMD> { + : public impl::values<2, UseSIMD> { /** * @name Typedefs * */ ///@{ + using base_type = impl::values<2, UseSIMD>; + using value_type = typename base_type::value_type; + constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::acoustic; constexpr static auto property_tag = specfem::element::property_tag::isotropic; - using simd = specfem::datatype::simd; - using value_type = typename simd::datatype; + constexpr static bool is_point_properties = true; constexpr static int _counter = __COUNTER__; ///@} - using base_type = impl::impl_properties<2, UseSIMD>; + using base_type::base_type; - DEFINE_PROP(rho_inverse) ///< @f$ \frac{1}{\rho} @f$ - DEFINE_PROP(kappa) ///< Bulk modulus @f$ \kappa @f$ + DEFINE_POINT_VALUE(rho_inverse) ///< @f$ \frac{1}{\rho} @f$ + DEFINE_POINT_VALUE(kappa) ///< Bulk modulus @f$ \kappa @f$ KOKKOS_INLINE_FUNCTION constexpr value_type kappa_inverse() const { return (static_cast(1.0)) / @@ -223,5 +163,3 @@ struct properties Date: Thu, 27 Feb 2025 16:18:53 -0500 Subject: [PATCH 11/21] Create shared parent class for containers. --- include/compute/properties/properties.hpp | 4 +- .../isotropic/properties_container.hpp | 11 +- .../anisotropic/properties_container.hpp | 29 +-- .../isotropic/properties_container.hpp | 13 +- include/medium/impl/containers.hpp | 156 ++++++++++++++++ include/medium/properties_container.hpp | 169 ++---------------- 6 files changed, 194 insertions(+), 188 deletions(-) create mode 100644 include/medium/impl/containers.hpp diff --git a/include/compute/properties/properties.hpp b/include/compute/properties/properties.hpp index 85f97ccd..3093befd 100644 --- a/include/compute/properties/properties.hpp +++ b/include/compute/properties/properties.hpp @@ -108,7 +108,7 @@ load_on_device(const IndexType &lcoord, static_assert(DimensionType == specfem::dimension::type::dim2, "Only 2D properties are supported"); - properties.get_container().load_device_properties( + properties.get_container().load_device_values( l_index, point_properties); } @@ -148,7 +148,7 @@ void load_on_host(const IndexType &lcoord, static_assert(DimensionType == specfem::dimension::type::dim2, "Only 2D properties are supported"); - properties.get_container().load_host_properties( + properties.get_container().load_host_values( l_index, point_properties); } diff --git a/include/medium/dim2/acoustic/isotropic/properties_container.hpp b/include/medium/dim2/acoustic/isotropic/properties_container.hpp index 6a788138..95a1ac08 100644 --- a/include/medium/dim2/acoustic/isotropic/properties_container.hpp +++ b/include/medium/dim2/acoustic/isotropic/properties_container.hpp @@ -10,18 +10,17 @@ namespace medium { template <> struct properties_container - : public impl::impl_properties_container< + : public impl_properties_container< specfem::element::medium_tag::acoustic, specfem::element::property_tag::isotropic, 2> { using base_type = - impl::impl_properties_container; + impl_properties_container; using base_type::base_type; constexpr static int _counter = __COUNTER__; - DEFINE_CONTAINER(rho_inverse) - DEFINE_CONTAINER(kappa) + DEFINE_MEDIUM_VIEW(rho_inverse) + DEFINE_MEDIUM_VIEW(kappa) }; } // namespace medium diff --git a/include/medium/dim2/elastic/anisotropic/properties_container.hpp b/include/medium/dim2/elastic/anisotropic/properties_container.hpp index b78debd9..2cf27a5f 100644 --- a/include/medium/dim2/elastic/anisotropic/properties_container.hpp +++ b/include/medium/dim2/elastic/anisotropic/properties_container.hpp @@ -10,25 +10,26 @@ namespace medium { template <> struct properties_container - : public impl::impl_properties_container< + : public impl_properties_container< specfem::element::medium_tag::elastic, specfem::element::property_tag::anisotropic, 10> { - using base_type = impl::impl_properties_container< - specfem::element::medium_tag::elastic, - specfem::element::property_tag::anisotropic, 10>; + using base_type = + impl_properties_container; using base_type::base_type; constexpr static int _counter = __COUNTER__; - DEFINE_CONTAINER(c11) - DEFINE_CONTAINER(c13) - DEFINE_CONTAINER(c15) - DEFINE_CONTAINER(c33) - DEFINE_CONTAINER(c35) - DEFINE_CONTAINER(c55) - DEFINE_CONTAINER(c12) - DEFINE_CONTAINER(c23) - DEFINE_CONTAINER(c25) - DEFINE_CONTAINER(rho) + DEFINE_MEDIUM_VIEW(c11) + DEFINE_MEDIUM_VIEW(c13) + DEFINE_MEDIUM_VIEW(c15) + DEFINE_MEDIUM_VIEW(c33) + DEFINE_MEDIUM_VIEW(c35) + DEFINE_MEDIUM_VIEW(c55) + DEFINE_MEDIUM_VIEW(c12) + DEFINE_MEDIUM_VIEW(c23) + DEFINE_MEDIUM_VIEW(c25) + DEFINE_MEDIUM_VIEW(rho) }; } // namespace medium diff --git a/include/medium/dim2/elastic/isotropic/properties_container.hpp b/include/medium/dim2/elastic/isotropic/properties_container.hpp index 998d8877..217c5b23 100644 --- a/include/medium/dim2/elastic/isotropic/properties_container.hpp +++ b/include/medium/dim2/elastic/isotropic/properties_container.hpp @@ -10,19 +10,18 @@ namespace medium { template <> struct properties_container - : public impl::impl_properties_container< + : public impl_properties_container< specfem::element::medium_tag::elastic, specfem::element::property_tag::isotropic, 3> { using base_type = - impl::impl_properties_container; + impl_properties_container; using base_type::base_type; constexpr static int _counter = __COUNTER__; - DEFINE_CONTAINER(lambdaplus2mu) - DEFINE_CONTAINER(mu) - DEFINE_CONTAINER(rho) + DEFINE_MEDIUM_VIEW(lambdaplus2mu) + DEFINE_MEDIUM_VIEW(mu) + DEFINE_MEDIUM_VIEW(rho) }; } // namespace medium diff --git a/include/medium/impl/containers.hpp b/include/medium/impl/containers.hpp new file mode 100644 index 00000000..af56d2cb --- /dev/null +++ b/include/medium/impl/containers.hpp @@ -0,0 +1,156 @@ +#pragma once + +#include "enumerations/medium.hpp" + +#define DEFINE_MEDIUM_VIEW(prop) \ + constexpr static int i_##prop = __COUNTER__ - _counter - 1; \ + KOKKOS_INLINE_FUNCTION type_real &prop(const int &ispec, const int &iz, \ + const int &ix) const { \ + return base_type::data(ispec, iz, ix, i_##prop); \ + } \ + KOKKOS_INLINE_FUNCTION type_real &h_##prop(const int &ispec, const int &iz, \ + const int &ix) const { \ + return base_type::h_data(ispec, iz, ix, i_##prop); \ + } + +namespace specfem { +namespace medium { + +namespace impl { +template +struct medium_container { + using view_type = typename Kokkos::View; + constexpr static auto nprops = N; + constexpr static auto dimension = specfem::dimension::type::dim2; + constexpr static auto medium_tag = MediumTag; + constexpr static auto property_tag = PropertyTag; + + int nspec; ///< total number of acoustic spectral elements + int ngllz; ///< number of quadrature points in z dimension + int ngllx; ///< number of quadrature points in x dimension + + view_type data; + typename view_type::HostMirror h_data; + + medium_container() = default; + + medium_container(const int nspec, const int ngllz, const int ngllx) + : nspec(nspec), ngllz(ngllz), ngllx(ngllx), + data("specfem::medium::impl::container::data", nspec, ngllz, ngllx, N), + h_data(Kokkos::create_mirror_view(data)) {} + +private: + template + KOKKOS_FORCEINLINE_FUNCTION void + load_values(const specfem::point::index &index, + PointValues &values) const { + + static_assert(PointValues::dimension == dimension, "Dimension mismatch"); + static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); + static_assert(PointValues::property_tag == property_tag, + "Property tag mismatch"); + + const int ispec = index.ispec; + const int iz = index.iz; + const int ix = index.ix; + + for (int i = 0; i < nprops; i++) { + values.data[i] = + on_device ? data(ispec, iz, ix, i) : h_data(ispec, iz, ix, i); + } + } + + template + KOKKOS_FORCEINLINE_FUNCTION void + load_values(const specfem::point::simd_index &index, + PointValues &values) const { + + static_assert(PointValues::dimension == dimension, "Dimension mismatch"); + static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); + static_assert(PointValues::property_tag == property_tag, + "Property tag mismatch"); + + using simd = typename PointValues::simd; + using mask_type = typename simd::mask_type; + using tag_type = typename simd::tag_type; + + const int ispec = index.ispec; + const int iz = index.iz; + const int ix = index.ix; + + mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); + + for (int i = 0; i < nprops; i++) { + Kokkos::Experimental::where(mask, values.data[i]) + .copy_from(on_device ? &data(ispec, iz, ix, i) + : &h_data(ispec, iz, ix, i), + tag_type()); + } + } + +public: + template + KOKKOS_FORCEINLINE_FUNCTION void + load_device_values(const IndexType &index, PointValues &values) const { + load_values(index, values); + } + + template + KOKKOS_FORCEINLINE_FUNCTION void load_host_values(const IndexType &index, + PointValues &values) const { + load_values(index, values); + } + + template + inline void assign(const specfem::point::index &index, + const PointValues &values) const { + + static_assert(PointValues::dimension == dimension, "Dimension mismatch"); + static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); + static_assert(PointValues::property_tag == property_tag, + "Property tag mismatch"); + + const int ispec = index.ispec; + const int iz = index.iz; + const int ix = index.ix; + + for (int i = 0; i < nprops; i++) { + h_data(ispec, iz, ix, i) = values.data[i]; + } + } + + template + inline void assign(const specfem::point::simd_index &index, + const PointValues &values) const { + + static_assert(PointValues::dimension == dimension, "Dimension mismatch"); + static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); + static_assert(PointValues::property_tag == property_tag, + "Property tag mismatch"); + + using simd = typename PointValues::simd; + using mask_type = typename simd::mask_type; + using tag_type = typename simd::tag_type; + + const int ispec = index.ispec; + const int iz = index.iz; + const int ix = index.ix; + + mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); + + for (int i = 0; i < nprops; i++) { + Kokkos::Experimental::where(mask, values.data[i]) + .copy_to(&h_data(ispec, iz, ix, i), tag_type()); + } + } + + void copy_to_device() { Kokkos::deep_copy(data, h_data); } + + void copy_to_host() { Kokkos::deep_copy(h_data, data); } +}; +} // namespace impl + +} // namespace medium +} // namespace specfem diff --git a/include/medium/properties_container.hpp b/include/medium/properties_container.hpp index 058b7389..8eb2f9d8 100644 --- a/include/medium/properties_container.hpp +++ b/include/medium/properties_container.hpp @@ -1,45 +1,16 @@ #pragma once -#include "enumerations/medium.hpp" - -#define DEFINE_CONTAINER(prop) \ - constexpr static int i_##prop = __COUNTER__ - _counter - 1; \ - KOKKOS_INLINE_FUNCTION type_real &prop(const int &ispec, const int &iz, \ - const int &ix) const { \ - return base_type::data(ispec, iz, ix, i_##prop); \ - } \ - KOKKOS_INLINE_FUNCTION type_real &h_##prop(const int &ispec, const int &iz, \ - const int &ix) const { \ - return base_type::h_data(ispec, iz, ix, i_##prop); \ - } +#include "impl/containers.hpp" namespace specfem { namespace medium { -namespace impl { template -struct impl_properties_container { - using view_type = typename Kokkos::View; - constexpr static auto nprops = N; - constexpr static auto dimension = specfem::dimension::type::dim2; - constexpr static auto medium_tag = MediumTag; - constexpr static auto property_tag = PropertyTag; - - int nspec; ///< total number of acoustic spectral elements - int ngllz; ///< number of quadrature points in z dimension - int ngllx; ///< number of quadrature points in x dimension - - view_type data; - typename view_type::HostMirror h_data; - - impl_properties_container() = default; - - impl_properties_container(const int nspec, const int ngllz, const int ngllx) - : nspec(nspec), ngllz(ngllz), ngllx(ngllx), - data("specfem::benchmarks::properties::data", nspec, ngllz, ngllx, N), - h_data(Kokkos::create_mirror_view(data)) {} +struct impl_properties_container + : public impl::medium_container { + using base_type = impl::medium_container; + using base_type::base_type; impl_properties_container( const Kokkos::View elements, @@ -58,13 +29,15 @@ struct impl_properties_container { for (int ix = 0; ix < ngllx; ++ix) { // Get the material at index from mesh::materials auto material = - std::get >( + std::get >( materials[ispec]); // Assign the material property to the property container auto point_property = material.get_properties(); - this->assign(specfem::point::index(count, iz, ix), - point_property); + this->assign( + specfem::point::index(count, iz, ix), + point_property); } } } @@ -77,127 +50,7 @@ struct impl_properties_container { return; } - -private: - template - KOKKOS_FORCEINLINE_FUNCTION void - load_properties(const specfem::point::index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == medium_tag, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_tag, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - for (int i = 0; i < nprops; i++) { - property.data[i] = - on_device ? data(ispec, iz, ix, i) : h_data(ispec, iz, ix, i); - } - } - - template - KOKKOS_FORCEINLINE_FUNCTION void - load_properties(const specfem::point::simd_index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == medium_tag, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_tag, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); - - for (int i = 0; i < nprops; i++) { - Kokkos::Experimental::where(mask, property.data[i]) - .copy_from(on_device ? &data(ispec, iz, ix, i) - : &h_data(ispec, iz, ix, i), - tag_type()); - } - } - -public: - template - KOKKOS_FORCEINLINE_FUNCTION void - load_device_properties(const IndexType &index, - PointProperties &property) const { - load_properties(index, property); - } - - template - KOKKOS_FORCEINLINE_FUNCTION void - load_host_properties(const IndexType &index, - PointProperties &property) const { - load_properties(index, property); - } - - template - inline void assign(const specfem::point::index &index, - const PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == medium_tag, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_tag, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - for (int i = 0; i < nprops; i++) { - h_data(ispec, iz, ix, i) = property.data[i]; - } - } - - template - inline void assign(const specfem::point::simd_index &index, - const PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == medium_tag, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_tag, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); - - for (int i = 0; i < nprops; i++) { - Kokkos::Experimental::where(mask, property.data[i]) - .copy_to(&h_data(ispec, iz, ix, i), tag_type()); - } - } - - void copy_to_device() { Kokkos::deep_copy(data, h_data); } - - void copy_to_host() { Kokkos::deep_copy(h_data, data); } }; -} // namespace impl template @@ -213,5 +66,3 @@ struct properties_container { #include "dim2/acoustic/isotropic/properties_container.hpp" #include "dim2/elastic/anisotropic/properties_container.hpp" #include "dim2/elastic/isotropic/properties_container.hpp" - -#undef DEFINE_CONTAINER From 974a0ff52ae0a784128c3489e00dc71b5fe4d751 Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Thu, 27 Feb 2025 16:23:38 -0500 Subject: [PATCH 12/21] Refactor kernels_container --- include/medium/kernels_container.hpp | 53 +++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/include/medium/kernels_container.hpp b/include/medium/kernels_container.hpp index 938be54f..3792d1fb 100644 --- a/include/medium/kernels_container.hpp +++ b/include/medium/kernels_container.hpp @@ -1,14 +1,57 @@ #pragma once -#include "enumerations/medium.hpp" -#include "kokkos_abstractions.h" -#include "point/coordinates.hpp" -#include "point/kernels.hpp" -#include +#include "impl/containers.hpp" namespace specfem { namespace medium { +template +struct impl_properties_container + : public impl::medium_container { + using base_type = impl::medium_container; + using base_type::base_type; + + impl_kernels_container( + const Kokkos::View elements, + const int ngllz, const int ngllx, + const specfem::mesh::materials &materials, const bool has_gll_model, + const specfem::kokkos::HostView1d property_index_mapping) + : impl_kernels_container(elements.extent(0), ngllz, ngllx) { + + const int nelement = elements.extent(0); + int count = 0; + for (int i = 0; i < nelement; ++i) { + const int ispec = elements(i); + property_index_mapping(ispec) = count; + if (!has_gll_model) { + for (int iz = 0; iz < ngllz; ++iz) { + for (int ix = 0; ix < ngllx; ++ix) { + // Get the material at index from mesh::materials + auto material = + std::get >( + materials[ispec]); + + // Assign the material property to the property container + auto point_property = material.get_properties(); + this->assign( + specfem::point::index(count, iz, ix), + point_property); + } + } + } + count++; + } + + if (!has_gll_model) { + this->copy_to_device(); + } + + return; + } +}; + template class kernels_container; From 1e14192fc780c824ba7b1eeac429f3a81b16e7ac Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Fri, 28 Feb 2025 11:26:28 -0500 Subject: [PATCH 13/21] Refactor kernels. --- include/IO/kernel/writer.tpp | 105 ++-- include/compute/kernels/kernels.hpp | 12 +- include/compute/properties/properties.hpp | 4 +- .../acoustic/isotropic/kernels_container.hpp | 420 +------------- .../elastic/anisotropic/kernels_container.hpp | 523 +----------------- .../elastic/isotropic/kernels_container.hpp | 489 +--------------- include/medium/impl/containers.hpp | 121 +++- include/medium/kernels_container.hpp | 26 +- include/medium/properties_container.hpp | 10 +- include/point/kernels.hpp | 301 ++-------- tests/unit-tests/assembly/kernels/kernels.cpp | 157 +++--- 11 files changed, 361 insertions(+), 1807 deletions(-) diff --git a/include/IO/kernel/writer.tpp b/include/IO/kernel/writer.tpp index ec458b17..ff1457ef 100644 --- a/include/IO/kernel/writer.tpp +++ b/include/IO/kernel/writer.tpp @@ -21,6 +21,9 @@ void specfem::IO::kernel_writer::write(specfem::compute::assembly using DomainView = Kokkos::View; + using DomainKernelView = + Kokkos::View; + kernels.copy_to_host(); typename OutputLibrary::File file(output_folder + "/Kernels"); @@ -44,12 +47,13 @@ void specfem::IO::kernel_writer::write(specfem::compute::assembly DomainView x("xcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, ngllx); DomainView z("zcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, ngllx); - DomainView rho("rho", n_elastic_isotropic, ngllz, ngllx); - DomainView mu("mu", n_elastic_isotropic, ngllz, ngllx); - DomainView kappa("kappa", n_elastic_isotropic, ngllz, ngllx); - DomainView rhop("rhop", n_elastic_isotropic, ngllz, ngllx); - DomainView alpha("alpha", n_elastic_isotropic, ngllz, ngllx); - DomainView beta("beta", n_elastic_isotropic, ngllz, ngllx); + using PointKernelType = typename specfem::point::kernels; + constexpr int nprops = PointKernelType::nprops; + + DomainKernelView data("data", n_elastic_isotropic, ngllz, ngllx, nprops); for (int i = 0; i < n_elastic_isotropic; i++) { const int ispec = element_indices(i); @@ -59,32 +63,20 @@ void specfem::IO::kernel_writer::write(specfem::compute::assembly z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); const specfem::point::index index( ispec, iz, ix); - specfem::point::kernels - point_kernels; + PointKernelType point_kernels; specfem::compute::load_on_host(index, kernels, point_kernels); - rho(i, iz, ix) = point_kernels.rho; - mu(i, iz, ix) = point_kernels.mu; - kappa(i, iz, ix) = point_kernels.kappa; - rhop(i, iz, ix) = point_kernels.rhop; - alpha(i, iz, ix) = point_kernels.alpha; - beta(i, iz, ix) = point_kernels.beta; + for (int l = 0; l < nprops; l++) { + data(i, iz, ix, l) = point_kernels.data[l]; + } } } } elastic.createDataset("X", x).write(); elastic.createDataset("Z", z).write(); - elastic.createDataset("rho", rho).write(); - elastic.createDataset("mu", mu).write(); - elastic.createDataset("kappa", kappa).write(); - elastic.createDataset("rhop", rhop).write(); - elastic.createDataset("alpha", alpha).write(); - elastic.createDataset("beta", beta).write(); + elastic.createDataset("data", data).write(); } { @@ -98,13 +90,13 @@ void specfem::IO::kernel_writer::write(specfem::compute::assembly DomainView x("xcoordinates_elastic_anisotropic", n_elastic_anisotropic, ngllz, ngllx); DomainView z("zcoordinates_elastic_anisotropic", n_elastic_anisotropic, ngllz, ngllx); - DomainView rho("rho", n_elastic_anisotropic, ngllz, ngllx); - DomainView c11("c11", n_elastic_anisotropic, ngllz, ngllx); - DomainView c13("c13", n_elastic_anisotropic, ngllz, ngllx); - DomainView c15("c15", n_elastic_anisotropic, ngllz, ngllx); - DomainView c33("c33", n_elastic_anisotropic, ngllz, ngllx); - DomainView c35("c35", n_elastic_anisotropic, ngllz, ngllx); - DomainView c55("c55", n_elastic_anisotropic, ngllz, ngllx); + using PointKernelType = typename specfem::point::kernels; + constexpr int nprops = PointKernelType::nprops; + + DomainKernelView data("data", n_elastic_anisotropic, ngllz, ngllx, nprops); for (int i = 0; i < n_elastic_anisotropic; i++) { const int ispec = element_indices(i); @@ -114,34 +106,20 @@ void specfem::IO::kernel_writer::write(specfem::compute::assembly z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); const specfem::point::index index( ispec, iz, ix); - specfem::point::kernels - point_kernels; + PointKernelType point_kernels; specfem::compute::load_on_host(index, kernels, point_kernels); - rho(i, iz, ix) = point_kernels.rho; - c11(i, iz, ix) = point_kernels.c11; - c13(i, iz, ix) = point_kernels.c13; - c15(i, iz, ix) = point_kernels.c15; - c33(i, iz, ix) = point_kernels.c33; - c35(i, iz, ix) = point_kernels.c35; - c55(i, iz, ix) = point_kernels.c55; + for (int l = 0; l < nprops; l++) { + data(i, iz, ix, l) = point_kernels.data[l]; + } } } } elastic.createDataset("X", x).write(); elastic.createDataset("Z", z).write(); - elastic.createDataset("rho", rho).write(); - elastic.createDataset("c11", c11).write(); - elastic.createDataset("c13", c13).write(); - elastic.createDataset("c15", c15).write(); - elastic.createDataset("c33", c33).write(); - elastic.createDataset("c35", c35).write(); - elastic.createDataset("c55", c55).write(); + elastic.createDataset("data", data).write(); } { @@ -153,10 +131,13 @@ void specfem::IO::kernel_writer::write(specfem::compute::assembly DomainView x("xcoordinates_acoustic", n_acoustic, ngllz, ngllx); DomainView z("zcoordinates_acoustic", n_acoustic, ngllz, ngllx); - DomainView rho("rho", n_acoustic, ngllz, ngllx); - DomainView kappa("kappa", n_acoustic, ngllz, ngllx); - DomainView rho_prime("rho_prime", n_acoustic, ngllz, ngllx); - DomainView alpha("alpha", n_acoustic, ngllz, ngllx); + using PointKernelType = typename specfem::point::kernels; + constexpr int nprops = PointKernelType::nprops; + + DomainKernelView data("data", n_acoustic, ngllz, ngllx, nprops); for (int i = 0; i < n_acoustic; i++) { const int ispec = element_indices(i); @@ -166,28 +147,20 @@ void specfem::IO::kernel_writer::write(specfem::compute::assembly z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); const specfem::point::index index( ispec, iz, ix); - specfem::point::kernels - point_kernels; + PointKernelType point_kernels; specfem::compute::load_on_host(index, kernels, point_kernels); - rho(i, iz, ix) = point_kernels.rho; - kappa(i, iz, ix) = point_kernels.kappa; - rho_prime(i, iz, ix) = point_kernels.rhop; - alpha(i, iz, ix) = point_kernels.alpha; + for (int l = 0; l < nprops; l++) { + data(i, iz, ix, l) = point_kernels.data[l]; + } } } } acoustic.createDataset("X", x).write(); acoustic.createDataset("Z", z).write(); - acoustic.createDataset("rho", rho).write(); - acoustic.createDataset("kappa", kappa).write(); - acoustic.createDataset("rho_prime", rho_prime).write(); - acoustic.createDataset("alpha", alpha).write(); + acoustic.createDataset("data", data).write(); } assert(n_elastic_isotropic + n_elastic_anisotropic + n_acoustic == nspec); diff --git a/include/compute/kernels/kernels.hpp b/include/compute/kernels/kernels.hpp index bd97d676..fe09199d 100644 --- a/include/compute/kernels/kernels.hpp +++ b/include/compute/kernels/kernels.hpp @@ -89,7 +89,7 @@ KOKKOS_FUNCTION void load_on_device(const IndexType &index, IndexType l_index = index; l_index.ispec = ispec; - kernels.get_container().load_device_kernels( + kernels.get_container().load_device_values( l_index, point_kernels); return; @@ -122,7 +122,7 @@ void load_on_host(const IndexType &index, const kernels &kernels, IndexType l_index = index; l_index.ispec = ispec; - kernels.get_container().load_host_kernels( + kernels.get_container().load_host_values( l_index, point_kernels); return; @@ -155,7 +155,7 @@ void store_on_host(const IndexType &index, const PointKernelType &point_kernels, IndexType l_index = index; l_index.ispec = ispec; - kernels.get_container().update_kernels_on_host( + kernels.get_container().store_host_values( l_index, point_kernels); return; @@ -189,7 +189,7 @@ KOKKOS_FUNCTION void store_on_device(const IndexType &index, IndexType l_index = index; l_index.ispec = ispec; - kernels.get_container().update_kernels_on_device( + kernels.get_container().store_device_values( l_index, point_kernels); return; @@ -225,7 +225,7 @@ KOKKOS_FUNCTION void add_on_device(const IndexType &index, IndexType l_index = index; l_index.ispec = ispec; - kernels.get_container().add_kernels_on_device( + kernels.get_container().add_device_values( l_index, point_kernels); return; @@ -259,7 +259,7 @@ void add_on_host(const IndexType &index, const PointKernelType &point_kernels, IndexType l_index = index; l_index.ispec = ispec; - kernels.get_container().add_kernels_on_host( + kernels.get_container().add_host_values( l_index, point_kernels); return; diff --git a/include/compute/properties/properties.hpp b/include/compute/properties/properties.hpp index 3093befd..2fc86b56 100644 --- a/include/compute/properties/properties.hpp +++ b/include/compute/properties/properties.hpp @@ -187,8 +187,8 @@ void store_on_host(const IndexType &lcoord, static_assert(DimensionType == specfem::dimension::type::dim2, "Only 2D properties are supported"); - properties.get_container().assign(l_index, - point_properties); + properties.get_container().store_host_values( + l_index, point_properties); } } // namespace compute } // namespace specfem diff --git a/include/medium/dim2/acoustic/isotropic/kernels_container.hpp b/include/medium/dim2/acoustic/isotropic/kernels_container.hpp index e8b64e67..a0aee70b 100644 --- a/include/medium/dim2/acoustic/isotropic/kernels_container.hpp +++ b/include/medium/dim2/acoustic/isotropic/kernels_container.hpp @@ -1,412 +1,28 @@ #pragma once -#include "enumerations/medium.hpp" -#include "kokkos_abstractions.h" -#include "point/coordinates.hpp" -#include "point/kernels.hpp" -#include +#include "medium/properties_container.hpp" +#include "point/interface.hpp" +#include namespace specfem { namespace medium { template <> -class kernels_container { -public: - constexpr static auto value_type = specfem::element::medium_tag::acoustic; - constexpr static auto property_type = - specfem::element::property_tag::isotropic; - int nspec; - int ngllz; - int ngllx; - - using ViewType = Kokkos::View; - ViewType rho; - ViewType::HostMirror h_rho; - ViewType kappa; - ViewType::HostMirror h_kappa; - ViewType rho_prime; - ViewType::HostMirror h_rho_prime; - ViewType alpha; - ViewType::HostMirror h_alpha; - - kernels_container() = default; - - kernels_container(const int nspec, const int ngllz, const int ngllx) - : nspec(nspec), ngllz(ngllz), ngllx(ngllx), - rho("specfem::medium::acoustic::rho", nspec, ngllz, ngllx), - kappa("specfem::medium::acoustic::kappa", nspec, ngllz, ngllx), - rho_prime("specfem::medium::acoustic::rho_prime", nspec, ngllz, ngllx), - alpha("specfem::medium::acoustic::alpha", nspec, ngllz, ngllx), - h_rho(Kokkos::create_mirror_view(rho)), - h_kappa(Kokkos::create_mirror_view(kappa)), - h_rho_prime(Kokkos::create_mirror_view(rho_prime)), - h_alpha(Kokkos::create_mirror_view(alpha)) { - - initialize(); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void load_device_kernels( - const specfem::point::index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - kernels.rho = rho(ispec, iz, ix); - kernels.kappa = kappa(ispec, iz, ix); - kernels.rhop = rho_prime(ispec, iz, ix); - kernels.alpha = alpha(ispec, iz, ix); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void load_device_kernels( - const specfem::point::simd_index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_from(&rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.kappa) - .copy_from(&kappa(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.rhop) - .copy_from(&rho_prime(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.alpha) - .copy_from(&alpha(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void load_host_kernels( - const specfem::point::index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - kernels.rho = h_rho(ispec, iz, ix); - kernels.kappa = h_kappa(ispec, iz, ix); - kernels.rhop = h_rho_prime(ispec, iz, ix); - kernels.alpha = h_alpha(ispec, iz, ix); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void load_host_kernels( - const specfem::point::simd_index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_from(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.kappa) - .copy_from(&h_kappa(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.rhop) - .copy_from(&h_rho_prime(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.alpha) - .copy_from(&h_alpha(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void update_kernels_on_device( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - rho(ispec, iz, ix) = kernels.rho; - kappa(ispec, iz, ix) = kernels.kappa; - rho_prime(ispec, iz, ix) = kernels.rhop; - alpha(ispec, iz, ix) = kernels.alpha; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void update_kernels_on_device( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_to(&rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.kappa) - .copy_to(&kappa(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.rhop) - .copy_to(&rho_prime(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.alpha) - .copy_to(&alpha(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void update_kernels_on_host( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho(ispec, iz, ix) = kernels.rho; - h_kappa(ispec, iz, ix) = kernels.kappa; - h_rho_prime(ispec, iz, ix) = kernels.rhop; - h_alpha(ispec, iz, ix) = kernels.alpha; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void update_kernels_on_host( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_to(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.kappa) - .copy_to(&h_kappa(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.rhop) - .copy_to(&h_rho_prime(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.alpha) - .copy_to(&h_alpha(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void add_kernels_on_device( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - rho(ispec, iz, ix) += kernels.rho; - kappa(ispec, iz, ix) += kernels.kappa; - rho_prime(ispec, iz, ix) += kernels.rhop; - alpha(ispec, iz, ix) += kernels.alpha; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void add_kernels_on_device( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - simd_type lhs; - - Kokkos::Experimental::where(mask, lhs).copy_from(&rho(ispec, iz, ix), - tag_type()); - lhs += kernels.rho; - Kokkos::Experimental::where(mask, lhs).copy_to(&rho(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&kappa(ispec, iz, ix), - tag_type()); - lhs += kernels.kappa; - Kokkos::Experimental::where(mask, lhs).copy_to(&kappa(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&rho_prime(ispec, iz, ix), - tag_type()); - lhs += kernels.rhop; - Kokkos::Experimental::where(mask, lhs).copy_to(&rho_prime(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&alpha(ispec, iz, ix), - tag_type()); - lhs += kernels.alpha; - Kokkos::Experimental::where(mask, lhs).copy_to(&alpha(ispec, iz, ix), - tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void add_kernels_on_host( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho(ispec, iz, ix) += kernels.rho; - h_kappa(ispec, iz, ix) += kernels.kappa; - h_rho_prime(ispec, iz, ix) += kernels.rhop; - h_alpha(ispec, iz, ix) += kernels.alpha; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void add_kernels_on_host( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - simd_type lhs; - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_rho(ispec, iz, ix), - tag_type()); - lhs += kernels.rho; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_rho(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_kappa(ispec, iz, ix), - tag_type()); - lhs += kernels.kappa; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_kappa(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from( - &h_rho_prime(ispec, iz, ix), tag_type()); - lhs += kernels.rhop; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_rho_prime(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_alpha(ispec, iz, ix), - tag_type()); - lhs += kernels.alpha; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_alpha(ispec, iz, ix), - tag_type()); - } - - void copy_to_host() { - Kokkos::deep_copy(h_rho, rho); - Kokkos::deep_copy(h_kappa, kappa); - Kokkos::deep_copy(h_rho_prime, rho_prime); - Kokkos::deep_copy(h_alpha, alpha); - } - - void copy_to_device() { - Kokkos::deep_copy(rho, h_rho); - Kokkos::deep_copy(kappa, h_kappa); - Kokkos::deep_copy(rho_prime, h_rho_prime); - Kokkos::deep_copy(alpha, h_alpha); - } - - void initialize() { - Kokkos::parallel_for( - "specfem::medium::acoustic::initialize", - Kokkos::MDRangePolicy >({ 0, 0, 0 }, - { nspec, ngllz, ngllx }), - KOKKOS_CLASS_LAMBDA(const int ispec, const int iz, const int ix) { - this->rho(ispec, iz, ix) = 0.0; - this->kappa(ispec, iz, ix) = 0.0; - this->rho_prime(ispec, iz, ix) = 0.0; - this->alpha(ispec, iz, ix) = 0.0; - }); - } +struct kernels_container + : public impl_kernels_container { + using base_type = + impl_kernels_container; + using base_type::base_type; + constexpr static int _counter = __COUNTER__; + + DEFINE_MEDIUM_VIEW(rho) + DEFINE_MEDIUM_VIEW(kappa) + DEFINE_MEDIUM_VIEW(rhop) + DEFINE_MEDIUM_VIEW(alpha) }; } // namespace medium diff --git a/include/medium/dim2/elastic/anisotropic/kernels_container.hpp b/include/medium/dim2/elastic/anisotropic/kernels_container.hpp index b03e170c..90c86d75 100644 --- a/include/medium/dim2/elastic/anisotropic/kernels_container.hpp +++ b/include/medium/dim2/elastic/anisotropic/kernels_container.hpp @@ -1,510 +1,31 @@ #pragma once -#include "enumerations/medium.hpp" -#include "kokkos_abstractions.h" -#include "point/coordinates.hpp" -#include "point/kernels.hpp" -#include + +#include "medium/properties_container.hpp" +#include "point/interface.hpp" +#include namespace specfem { namespace medium { template <> -class kernels_container { -public: - constexpr static auto value_type = specfem::element::medium_tag::elastic; - constexpr static auto property_type = - specfem::element::property_tag::anisotropic; - int nspec; - int ngllz; - int ngllx; - - using ViewType = Kokkos::View; - - ViewType rho; - ViewType::HostMirror h_rho; - ViewType c11; - ViewType::HostMirror h_c11; - ViewType c13; - ViewType::HostMirror h_c13; - ViewType c15; - ViewType::HostMirror h_c15; - ViewType c33; - ViewType::HostMirror h_c33; - ViewType c35; - ViewType::HostMirror h_c35; - ViewType c55; - ViewType::HostMirror h_c55; - - kernels_container() = default; - - kernels_container(const int nspec, const int ngllz, const int ngllx) - : nspec(nspec), ngllz(ngllz), ngllx(ngllx), - rho("specfem::compute::properties::rho", nspec, ngllz, ngllx), - h_rho(Kokkos::create_mirror_view(rho)), - c11("specfem::compute::properties::c11", nspec, ngllz, ngllx), - h_c11(Kokkos::create_mirror_view(c11)), - c13("specfem::compute::properties::c13", nspec, ngllz, ngllx), - h_c13(Kokkos::create_mirror_view(c13)), - c15("specfem::compute::properties::c15", nspec, ngllz, ngllx), - h_c15(Kokkos::create_mirror_view(c15)), - c33("specfem::compute::properties::c33", nspec, ngllz, ngllx), - h_c33(Kokkos::create_mirror_view(c33)), - c35("specfem::compute::properties::c35", nspec, ngllz, ngllx), - h_c35(Kokkos::create_mirror_view(c35)), - c55("specfem::compute::properties::c55", nspec, ngllz, ngllx), - h_c55(Kokkos::create_mirror_view(c55)) { - - initialize(); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void load_device_kernels( - const specfem::point::index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - kernels.rho = rho(ispec, iz, ix); - kernels.c11 = c11(ispec, iz, ix); - kernels.c13 = c13(ispec, iz, ix); - kernels.c15 = c15(ispec, iz, ix); - kernels.c33 = c33(ispec, iz, ix); - kernels.c35 = c35(ispec, iz, ix); - kernels.c55 = c55(ispec, iz, ix); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void load_device_kernels( - const specfem::point::simd_index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_from(&rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c11) - .copy_from(&c11(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c13) - .copy_from(&c13(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c15) - .copy_from(&c15(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c33) - .copy_from(&c33(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c35) - .copy_from(&c35(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c55) - .copy_from(&c55(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void - load_host_kernels(specfem::point::index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - kernels.rho = h_rho(ispec, iz, ix); - kernels.c11 = h_c11(ispec, iz, ix); - kernels.c13 = h_c13(ispec, iz, ix); - kernels.c15 = h_c15(ispec, iz, ix); - kernels.c33 = h_c33(ispec, iz, ix); - kernels.c35 = h_c35(ispec, iz, ix); - kernels.c55 = h_c55(ispec, iz, ix); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void load_host_kernels( - specfem::point::simd_index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_from(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c11) - .copy_from(&h_c11(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c13) - .copy_from(&h_c13(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c15) - .copy_from(&h_c15(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c33) - .copy_from(&h_c33(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c35) - .copy_from(&h_c35(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c55) - .copy_from(&h_c55(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void update_kernels_on_device( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - rho(ispec, iz, ix) = kernels.rho; - c11(ispec, iz, ix) = kernels.c11; - c13(ispec, iz, ix) = kernels.c13; - c15(ispec, iz, ix) = kernels.c15; - c33(ispec, iz, ix) = kernels.c33; - c35(ispec, iz, ix) = kernels.c35; - c55(ispec, iz, ix) = kernels.c55; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void update_kernels_on_device( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_to(&rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c11) - .copy_to(&c11(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c13) - .copy_to(&c13(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c15) - .copy_to(&c15(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c33) - .copy_to(&c33(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c35) - .copy_to(&c35(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c55) - .copy_to(&c55(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void update_kernels_on_host( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho(ispec, iz, ix) = kernels.rho; - h_c11(ispec, iz, ix) = kernels.c11; - h_c13(ispec, iz, ix) = kernels.c13; - h_c15(ispec, iz, ix) = kernels.c15; - h_c33(ispec, iz, ix) = kernels.c33; - h_c35(ispec, iz, ix) = kernels.c35; - h_c55(ispec, iz, ix) = kernels.c55; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void update_kernels_on_host( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - Kokkos::Experimental::where(mask, kernels.rho) - .copy_to(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c11) - .copy_to(&h_c11(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c13) - .copy_to(&h_c13(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c15) - .copy_to(&h_c15(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c33) - .copy_to(&h_c33(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c35) - .copy_to(&h_c35(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c55) - .copy_to(&h_c55(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void add_kernels_on_device( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - rho(ispec, iz, ix) += kernels.rho; - c11(ispec, iz, ix) += kernels.c11; - c13(ispec, iz, ix) += kernels.c13; - c15(ispec, iz, ix) += kernels.c15; - c33(ispec, iz, ix) += kernels.c33; - c35(ispec, iz, ix) += kernels.c35; - c55(ispec, iz, ix) += kernels.c55; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void add_kernels_on_device( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - simd_type lhs; - - Kokkos::Experimental::where(mask, lhs).copy_from(&rho(ispec, iz, ix), - tag_type()); - lhs += kernels.rho; - Kokkos::Experimental::where(mask, lhs).copy_to(&rho(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&c11(ispec, iz, ix), - tag_type()); - lhs += kernels.c11; - Kokkos::Experimental::where(mask, lhs).copy_to(&c11(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&c13(ispec, iz, ix), - tag_type()); - lhs += kernels.c13; - Kokkos::Experimental::where(mask, lhs).copy_to(&c13(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&c15(ispec, iz, ix), - tag_type()); - lhs += kernels.c15; - Kokkos::Experimental::where(mask, lhs).copy_to(&c15(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&c33(ispec, iz, ix), - tag_type()); - lhs += kernels.c33; - Kokkos::Experimental::where(mask, lhs).copy_to(&c33(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&c35(ispec, iz, ix), - tag_type()); - lhs += kernels.c35; - Kokkos::Experimental::where(mask, lhs).copy_to(&c35(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&c55(ispec, iz, ix), - tag_type()); - lhs += kernels.c55; - Kokkos::Experimental::where(mask, lhs).copy_to(&c55(ispec, iz, ix), - tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void add_kernels_on_host( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho(ispec, iz, ix) += kernels.rho; - h_c11(ispec, iz, ix) += kernels.c11; - h_c13(ispec, iz, ix) += kernels.c13; - h_c15(ispec, iz, ix) += kernels.c15; - h_c33(ispec, iz, ix) += kernels.c33; - h_c35(ispec, iz, ix) += kernels.c35; - h_c55(ispec, iz, ix) += kernels.c55; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void add_kernels_on_host( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - simd_type lhs; - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_rho(ispec, iz, ix), - tag_type()); - lhs += kernels.rho; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_rho(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_c11(ispec, iz, ix), - tag_type()); - lhs += kernels.c11; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_c11(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_c13(ispec, iz, ix), - tag_type()); - lhs += kernels.c13; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_c13(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_c15(ispec, iz, ix), - tag_type()); - lhs += kernels.c15; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_c15(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_c33(ispec, iz, ix), - tag_type()); - lhs += kernels.c33; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_c33(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_c35(ispec, iz, ix), - tag_type()); - lhs += kernels.c35; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_c35(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_c55(ispec, iz, ix), - tag_type()); - lhs += kernels.c55; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_c55(ispec, iz, ix), - tag_type()); - } - - void copy_to_host() { - Kokkos::deep_copy(h_rho, rho); - Kokkos::deep_copy(h_c11, c11); - Kokkos::deep_copy(h_c13, c13); - Kokkos::deep_copy(h_c15, c15); - Kokkos::deep_copy(h_c33, c33); - Kokkos::deep_copy(h_c35, c35); - Kokkos::deep_copy(h_c55, c55); - } - - void copy_to_device() { - Kokkos::deep_copy(rho, h_rho); - Kokkos::deep_copy(c11, h_c11); - Kokkos::deep_copy(c13, h_c13); - Kokkos::deep_copy(c15, h_c15); - Kokkos::deep_copy(c33, h_c33); - Kokkos::deep_copy(c35, h_c35); - Kokkos::deep_copy(c55, h_c55); - } - - void initialize() { - Kokkos::parallel_for( - "specfem::medium::elastic::anisotropic::initialize", - Kokkos::MDRangePolicy >({ 0, 0, 0 }, - { nspec, ngllz, ngllx }), - KOKKOS_CLASS_LAMBDA(const int ispec, const int iz, const int ix) { - this->rho(ispec, iz, ix) = 0.0; - this->c11(ispec, iz, ix) = 0.0; - this->c13(ispec, iz, ix) = 0.0; - this->c15(ispec, iz, ix) = 0.0; - this->c33(ispec, iz, ix) = 0.0; - this->c35(ispec, iz, ix) = 0.0; - this->c55(ispec, iz, ix) = 0.0; - }); - } +struct kernels_container + : public impl_kernels_container { + using base_type = + impl_kernels_container; + using base_type::base_type; + constexpr static int _counter = __COUNTER__; + + DEFINE_MEDIUM_VIEW(rho) + DEFINE_MEDIUM_VIEW(c11) + DEFINE_MEDIUM_VIEW(c13) + DEFINE_MEDIUM_VIEW(c15) + DEFINE_MEDIUM_VIEW(c33) + DEFINE_MEDIUM_VIEW(c35) + DEFINE_MEDIUM_VIEW(c55) }; } // namespace medium diff --git a/include/medium/dim2/elastic/isotropic/kernels_container.hpp b/include/medium/dim2/elastic/isotropic/kernels_container.hpp index 65073f82..888def47 100644 --- a/include/medium/dim2/elastic/isotropic/kernels_container.hpp +++ b/include/medium/dim2/elastic/isotropic/kernels_container.hpp @@ -1,479 +1,30 @@ #pragma once -#include "enumerations/medium.hpp" -#include "kokkos_abstractions.h" -#include "point/coordinates.hpp" -#include "point/kernels.hpp" -#include +#include "medium/properties_container.hpp" +#include "point/interface.hpp" +#include namespace specfem { namespace medium { template <> -class kernels_container { -public: - constexpr static auto value_type = specfem::element::medium_tag::elastic; - constexpr static auto property_type = - specfem::element::property_tag::isotropic; - int nspec; - int ngllz; - int ngllx; - - using ViewType = Kokkos::View; - - ViewType rho; - ViewType::HostMirror h_rho; - ViewType mu; - ViewType::HostMirror h_mu; - ViewType kappa; - ViewType::HostMirror h_kappa; - ViewType rhop; - ViewType::HostMirror h_rhop; - ViewType alpha; - ViewType::HostMirror h_alpha; - ViewType beta; - ViewType::HostMirror h_beta; - - kernels_container() = default; - - kernels_container(const int nspec, const int ngllz, const int ngllx) - : nspec(nspec), ngllz(ngllz), ngllx(ngllx), - rho("specfem::medium::elastic::rho", nspec, ngllz, ngllx), - mu("specfem::medium::elastic::mu", nspec, ngllz, ngllx), - kappa("specfem::medium::elastic::kappa", nspec, ngllz, ngllx), - rhop("specfem::medium::elastic::rhop", nspec, ngllz, ngllx), - alpha("specfem::medium::elastic::alpha", nspec, ngllz, ngllx), - beta("specfem::medium::elastic::beta", nspec, ngllz, ngllx), - h_rho(Kokkos::create_mirror_view(rho)), - h_mu(Kokkos::create_mirror_view(mu)), - h_kappa(Kokkos::create_mirror_view(kappa)), - h_rhop(Kokkos::create_mirror_view(rhop)), - h_alpha(Kokkos::create_mirror_view(alpha)), - h_beta(Kokkos::create_mirror_view(beta)) { - - initialize(); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void load_device_kernels( - const specfem::point::index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - kernels.rho = rho(ispec, iz, ix); - kernels.mu = mu(ispec, iz, ix); - kernels.kappa = kappa(ispec, iz, ix); - kernels.rhop = rhop(ispec, iz, ix); - kernels.alpha = alpha(ispec, iz, ix); - kernels.beta = beta(ispec, iz, ix); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void load_device_kernels( - const specfem::point::simd_index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_from(&rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.mu) - .copy_from(&mu(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.kappa) - .copy_from(&kappa(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.rhop) - .copy_from(&rhop(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.alpha) - .copy_from(&alpha(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.beta) - .copy_from(&beta(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void - load_host_kernels(specfem::point::index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - kernels.rho = h_rho(ispec, iz, ix); - kernels.mu = h_mu(ispec, iz, ix); - kernels.kappa = h_kappa(ispec, iz, ix); - kernels.rhop = h_rhop(ispec, iz, ix); - kernels.alpha = h_alpha(ispec, iz, ix); - kernels.beta = h_beta(ispec, iz, ix); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void load_host_kernels( - specfem::point::simd_index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_from(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.mu) - .copy_from(&h_mu(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.kappa) - .copy_from(&h_kappa(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.rhop) - .copy_from(&h_rhop(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.alpha) - .copy_from(&h_alpha(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.beta) - .copy_from(&h_beta(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void update_kernels_on_device( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - rho(ispec, iz, ix) = kernels.rho; - mu(ispec, iz, ix) = kernels.mu; - kappa(ispec, iz, ix) = kernels.kappa; - rhop(ispec, iz, ix) = kernels.rhop; - alpha(ispec, iz, ix) = kernels.alpha; - beta(ispec, iz, ix) = kernels.beta; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void update_kernels_on_device( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_to(&rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.mu) - .copy_to(&mu(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.kappa) - .copy_to(&kappa(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.rhop) - .copy_to(&rhop(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.alpha) - .copy_to(&alpha(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.beta) - .copy_to(&beta(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void update_kernels_on_host( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho(ispec, iz, ix) = kernels.rho; - h_mu(ispec, iz, ix) = kernels.mu; - h_kappa(ispec, iz, ix) = kernels.kappa; - h_rhop(ispec, iz, ix) = kernels.rhop; - h_alpha(ispec, iz, ix) = kernels.alpha; - h_beta(ispec, iz, ix) = kernels.beta; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void update_kernels_on_host( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_to(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.mu) - .copy_to(&h_mu(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.kappa) - .copy_to(&h_kappa(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.rhop) - .copy_to(&h_rhop(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.alpha) - .copy_to(&h_alpha(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.beta) - .copy_to(&h_beta(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void add_kernels_on_device( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - rho(ispec, iz, ix) += kernels.rho; - mu(ispec, iz, ix) += kernels.mu; - kappa(ispec, iz, ix) += kernels.kappa; - rhop(ispec, iz, ix) += kernels.rhop; - alpha(ispec, iz, ix) += kernels.alpha; - beta(ispec, iz, ix) += kernels.beta; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void add_kernels_on_device( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - simd_type lhs; - - Kokkos::Experimental::where(mask, lhs).copy_from(&rho(ispec, iz, ix), - tag_type()); - lhs += kernels.rho; - Kokkos::Experimental::where(mask, lhs).copy_to(&rho(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&mu(ispec, iz, ix), - tag_type()); - lhs += kernels.mu; - Kokkos::Experimental::where(mask, lhs).copy_to(&mu(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&kappa(ispec, iz, ix), - tag_type()); - lhs += kernels.kappa; - Kokkos::Experimental::where(mask, lhs).copy_to(&kappa(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&rhop(ispec, iz, ix), - tag_type()); - lhs += kernels.rhop; - Kokkos::Experimental::where(mask, lhs).copy_to(&rhop(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&alpha(ispec, iz, ix), - tag_type()); - lhs += kernels.alpha; - Kokkos::Experimental::where(mask, lhs).copy_to(&alpha(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&beta(ispec, iz, ix), - tag_type()); - lhs += kernels.beta; - Kokkos::Experimental::where(mask, lhs).copy_to(&beta(ispec, iz, ix), - tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void add_kernels_on_host( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho(ispec, iz, ix) += kernels.rho; - h_mu(ispec, iz, ix) += kernels.mu; - h_kappa(ispec, iz, ix) += kernels.kappa; - h_rhop(ispec, iz, ix) += kernels.rhop; - h_alpha(ispec, iz, ix) += kernels.alpha; - h_beta(ispec, iz, ix) += kernels.beta; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void add_kernels_on_host( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - simd_type lhs; - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_rho(ispec, iz, ix), - tag_type()); - lhs += kernels.rho; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_rho(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_mu(ispec, iz, ix), - tag_type()); - lhs += kernels.mu; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_mu(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_kappa(ispec, iz, ix), - tag_type()); - lhs += kernels.kappa; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_kappa(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_rhop(ispec, iz, ix), - tag_type()); - lhs += kernels.rhop; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_rhop(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_alpha(ispec, iz, ix), - tag_type()); - lhs += kernels.alpha; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_alpha(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_beta(ispec, iz, ix), - tag_type()); - lhs += kernels.beta; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_beta(ispec, iz, ix), - tag_type()); - } - - void copy_to_host() { - Kokkos::deep_copy(h_rho, rho); - Kokkos::deep_copy(h_mu, mu); - Kokkos::deep_copy(h_kappa, kappa); - Kokkos::deep_copy(h_rhop, rhop); - Kokkos::deep_copy(h_alpha, alpha); - Kokkos::deep_copy(h_beta, beta); - } - - void copy_to_device() { - Kokkos::deep_copy(rho, h_rho); - Kokkos::deep_copy(mu, h_mu); - Kokkos::deep_copy(kappa, h_kappa); - Kokkos::deep_copy(rhop, h_rhop); - Kokkos::deep_copy(alpha, h_alpha); - Kokkos::deep_copy(beta, h_beta); - } - - void initialize() { - Kokkos::parallel_for( - "specfem::medium::elastic::isotropic::initialize", - Kokkos::MDRangePolicy >({ 0, 0, 0 }, - { nspec, ngllz, ngllx }), - KOKKOS_CLASS_LAMBDA(const int ispec, const int iz, const int ix) { - this->rho(ispec, iz, ix) = 0.0; - this->mu(ispec, iz, ix) = 0.0; - this->kappa(ispec, iz, ix) = 0.0; - this->rhop(ispec, iz, ix) = 0.0; - this->alpha(ispec, iz, ix) = 0.0; - this->beta(ispec, iz, ix) = 0.0; - }); - } +struct kernels_container + : public impl_kernels_container { + using base_type = + impl_kernels_container; + using base_type::base_type; + constexpr static int _counter = __COUNTER__; + + DEFINE_MEDIUM_VIEW(rho) + DEFINE_MEDIUM_VIEW(mu) + DEFINE_MEDIUM_VIEW(kappa) + DEFINE_MEDIUM_VIEW(rhop) + DEFINE_MEDIUM_VIEW(alpha) + DEFINE_MEDIUM_VIEW(beta) }; } // namespace medium diff --git a/include/medium/impl/containers.hpp b/include/medium/impl/containers.hpp index af56d2cb..6a2f0b0d 100644 --- a/include/medium/impl/containers.hpp +++ b/include/medium/impl/containers.hpp @@ -90,22 +90,58 @@ struct medium_container { } } -public: - template - KOKKOS_FORCEINLINE_FUNCTION void - load_device_values(const IndexType &index, PointValues &values) const { - load_values(index, values); + template + inline void store_values(const specfem::point::index &index, + const PointValues &values) const { + + static_assert(PointValues::dimension == dimension, "Dimension mismatch"); + static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); + static_assert(PointValues::property_tag == property_tag, + "Property tag mismatch"); + + const int ispec = index.ispec; + const int iz = index.iz; + const int ix = index.ix; + + for (int i = 0; i < nprops; i++) { + if constexpr (on_device) { + data(ispec, iz, ix, i) = values.data[i]; + } else { + h_data(ispec, iz, ix, i) = values.data[i]; + } + } } - template - KOKKOS_FORCEINLINE_FUNCTION void load_host_values(const IndexType &index, - PointValues &values) const { - load_values(index, values); + template + inline void store_values(const specfem::point::simd_index &index, + const PointValues &values) const { + + static_assert(PointValues::dimension == dimension, "Dimension mismatch"); + static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); + static_assert(PointValues::property_tag == property_tag, + "Property tag mismatch"); + + using simd = typename PointValues::simd; + using mask_type = typename simd::mask_type; + using tag_type = typename simd::tag_type; + + const int ispec = index.ispec; + const int iz = index.iz; + const int ix = index.ix; + + mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); + + for (int i = 0; i < nprops; i++) { + Kokkos::Experimental::where(mask, values.data[i]) + .copy_to(on_device ? &data(ispec, iz, ix, i) + : &h_data(ispec, iz, ix, i), + tag_type()); + } } - template - inline void assign(const specfem::point::index &index, - const PointValues &values) const { + template + inline void add_values(const specfem::point::index &index, + const PointValues &values) const { static_assert(PointValues::dimension == dimension, "Dimension mismatch"); static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); @@ -117,13 +153,17 @@ struct medium_container { const int ix = index.ix; for (int i = 0; i < nprops; i++) { - h_data(ispec, iz, ix, i) = values.data[i]; + if constexpr (on_device) { + data(ispec, iz, ix, i) += values.data[i]; + } else { + h_data(ispec, iz, ix, i) += values.data[i]; + } } } - template - inline void assign(const specfem::point::simd_index &index, - const PointValues &values) const { + template + inline void add_values(const specfem::point::simd_index &index, + const PointValues &values) const { static_assert(PointValues::dimension == dimension, "Dimension mismatch"); static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); @@ -131,6 +171,7 @@ struct medium_container { "Property tag mismatch"); using simd = typename PointValues::simd; + using simd_type = typename simd::datatype; using mask_type = typename simd::mask_type; using tag_type = typename simd::tag_type; @@ -138,14 +179,58 @@ struct medium_container { const int iz = index.iz; const int ix = index.ix; + simd_type lhs; + mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); for (int i = 0; i < nprops; i++) { - Kokkos::Experimental::where(mask, values.data[i]) - .copy_to(&h_data(ispec, iz, ix, i), tag_type()); + Kokkos::Experimental::where(mask, lhs).copy_from( + on_device ? &data(ispec, iz, ix, i) : &h_data(ispec, iz, ix, i), + tag_type()); + lhs += values.data[i]; + Kokkos::Experimental::where(mask, lhs).copy_to( + on_device ? &data(ispec, iz, ix, i) : &h_data(ispec, iz, ix, i), + tag_type()); } } +public: + template + KOKKOS_FORCEINLINE_FUNCTION void + load_device_values(const IndexType &index, PointValues &values) const { + load_values(index, values); + } + + template + KOKKOS_FORCEINLINE_FUNCTION void load_host_values(const IndexType &index, + PointValues &values) const { + load_values(index, values); + } + + template + KOKKOS_FORCEINLINE_FUNCTION void + store_device_values(const IndexType &index, PointValues &values) const { + store_values(index, values); + } + + template + KOKKOS_FORCEINLINE_FUNCTION void + store_host_values(const IndexType &index, PointValues &values) const { + store_values(index, values); + } + + template + KOKKOS_FORCEINLINE_FUNCTION void + add_device_values(const IndexType &index, PointValues &values) const { + add_values(index, values); + } + + template + KOKKOS_FORCEINLINE_FUNCTION void add_host_values(const IndexType &index, + PointValues &values) const { + add_values(index, values); + } + void copy_to_device() { Kokkos::deep_copy(data, h_data); } void copy_to_host() { Kokkos::deep_copy(h_data, data); } diff --git a/include/medium/kernels_container.hpp b/include/medium/kernels_container.hpp index 3792d1fb..f79b28e2 100644 --- a/include/medium/kernels_container.hpp +++ b/include/medium/kernels_container.hpp @@ -7,7 +7,7 @@ namespace medium { template -struct impl_properties_container +struct impl_kernels_container : public impl::medium_container { using base_type = impl::medium_container; using base_type::base_type; @@ -15,7 +15,6 @@ struct impl_properties_container impl_kernels_container( const Kokkos::View elements, const int ngllz, const int ngllx, - const specfem::mesh::materials &materials, const bool has_gll_model, const specfem::kokkos::HostView1d property_index_mapping) : impl_kernels_container(elements.extent(0), ngllz, ngllx) { @@ -24,31 +23,8 @@ struct impl_properties_container for (int i = 0; i < nelement; ++i) { const int ispec = elements(i); property_index_mapping(ispec) = count; - if (!has_gll_model) { - for (int iz = 0; iz < ngllz; ++iz) { - for (int ix = 0; ix < ngllx; ++ix) { - // Get the material at index from mesh::materials - auto material = - std::get >( - materials[ispec]); - - // Assign the material property to the property container - auto point_property = material.get_properties(); - this->assign( - specfem::point::index(count, iz, ix), - point_property); - } - } - } count++; } - - if (!has_gll_model) { - this->copy_to_device(); - } - - return; } }; diff --git a/include/medium/properties_container.hpp b/include/medium/properties_container.hpp index 8eb2f9d8..809a581f 100644 --- a/include/medium/properties_container.hpp +++ b/include/medium/properties_container.hpp @@ -35,7 +35,7 @@ struct impl_properties_container // Assign the material property to the property container auto point_property = material.get_properties(); - this->assign( + this->store_host_values( specfem::point::index(count, iz, ix), point_property); } @@ -50,6 +50,14 @@ struct impl_properties_container return; } + + template + KOKKOS_FORCEINLINE_FUNCTION void + add_device_values(const IndexType &index, PointValues &values) const = delete; + + template + KOKKOS_FORCEINLINE_FUNCTION void + add_host_values(const IndexType &index, PointValues &values) const = delete; }; template +#include "impl/values.hpp" namespace specfem { namespace point { @@ -31,107 +28,40 @@ struct kernels; template struct kernels { -public: + specfem::element::property_tag::isotropic, UseSIMD> + : public impl::values<6, UseSIMD> { + /** * @name Typedefs * */ ///@{ - using simd = - typename specfem::datatype::simd; ///< SIMD type - using value_type = - typename simd::datatype; ///< Underlying data type to store the kernels - ///@} + using base_type = impl::values<6, UseSIMD>; + using value_type = typename base_type::value_type; -public: - /** - * @name Compile time constants - * - */ - ///@{ + constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::elastic; constexpr static auto property_tag = specfem::element::property_tag::isotropic; - constexpr static auto dimension = specfem::dimension::type::dim2; - ///@} - /** - * @name Misfit Kernels - * - */ - ///@{ - value_type rho; ///< \f$ K_{\rho} \f$ - value_type mu; ///< \f$ K_{\mu} \f$ - value_type kappa; ///< \f$ K_{\kappa} \f$ - value_type alpha; ///< \f$ K_{\alpha} \f$ - value_type beta; ///< \f$ K_{\beta} \f$ - value_type rhop; ///< \f$ K_{\rho'} \f$ + constexpr static bool is_point_properties = true; + constexpr static int _counter = __COUNTER__; ///@} - /** - * @name Constructors - * - */ - ///@{ - /** - * @brief Default constructor - * - */ - KOKKOS_FUNCTION - kernels() = default; - - /** - * @brief single value constructor - * - */ - KOKKOS_FUNCTION - kernels(const value_type value) - : rho(value), mu(value), kappa(value), alpha(value), beta(value), - rhop(value) {} + using base_type::base_type; /** - * @brief Constructor + * @name Misfit Kernels * - * @param rho \f$ K_{\rho} \f$ - * @param mu \f$ K_{\mu} \f$ - * @param kappa \f$ K_{\kappa} \f$ - * @param rhop \f$ K_{\rho'} \f$ - * @param alpha \f$ K_{\alpha} \f$ - * @param beta \f$ K_{\beta} \f$ */ - KOKKOS_FUNCTION - kernels(const value_type rho, const value_type mu, const value_type kappa, - const value_type rhop, const value_type alpha, const value_type beta) - : rho(rho), mu(mu), kappa(kappa), rhop(rhop), alpha(alpha), beta(beta) {} - + ///@{ + DEFINE_POINT_VALUE(rho) ///< \f$ K_{\rho} \f$ + DEFINE_POINT_VALUE(mu) ///< \f$ K_{\mu} \f$ + DEFINE_POINT_VALUE(kappa) ///< \f$ K_{\kappa} \f$ + DEFINE_POINT_VALUE(rhop) ///< \f$ K_{\rho'} \f$ + DEFINE_POINT_VALUE(alpha) ///< \f$ K_{\alpha} \f$ + DEFINE_POINT_VALUE(beta) ///< \f$ K_{\beta} \f$ ///@} - - /** - * @brief Equality operator - * - */ - KOKKOS_FUNCTION - bool operator==(const kernels &rhs) const { - return rho == rhs.rho && mu == rhs.mu && kappa == rhs.kappa && - rhop == rhs.rhop && alpha == rhs.alpha && beta == rhs.beta; - } - - /** - * @brief Inequality operator - * - */ - KOKKOS_FUNCTION - bool operator!=(const kernels &rhs) const { return !(*this == rhs); } - - KOKKOS_FUNCTION - bool operator==(const value_type value) { - return rho == value && mu == value && kappa == value && rhop == value && - alpha == value && beta == value; - } - - KOKKOS_FUNCTION - bool operator!=(const value_type value) { return !(*this == value); } }; // end elastic isotropic @@ -144,109 +74,41 @@ struct kernels struct kernels { -public: + specfem::element::property_tag::anisotropic, UseSIMD> + : public impl::values<7, UseSIMD> { + /** * @name Typedefs * */ ///@{ - using simd = - typename specfem::datatype::simd; ///< SIMD type - using value_type = - typename simd::datatype; ///< Underlying data type to store the kernels - ///@} + using base_type = impl::values<7, UseSIMD>; + using value_type = typename base_type::value_type; -public: - /** - * @name Compile time constants - * - */ - ///@{ + constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::elastic; constexpr static auto property_tag = - specfem::element::property_tag::anisotropic; - constexpr static auto dimension = specfem::dimension::type::dim2; - ///@} + specfem::element::property_tag::isotropic; - /** - * @name Misfit Kernels - * - */ - ///@{ - value_type rho; ///< \f$ K_{\rho} \f$ - value_type c11; ///< \f$ K_{c_{11}} \f$ - value_type c13; ///< \f$ K_{c_{13}} \f$ - value_type c15; ///< \f$ K_{c_{15}} \f$ - value_type c33; ///< \f$ K_{c_{33}} \f$ - value_type c35; ///< \f$ K_{c_{35}} \f$ - value_type c55; ///< \f$ K_{c_{55}} \f$ + constexpr static bool is_point_properties = true; + constexpr static int _counter = __COUNTER__; ///@} - /** - * @name Constructors - * - */ - ///@{ - /** - * @brief Default constructor - * - */ - KOKKOS_FUNCTION - kernels() = default; + using base_type::base_type; /** - * @brief single value constructor + * @name Misfit Kernels * */ - KOKKOS_FUNCTION - kernels(const value_type value) - : rho(value), c11(value), c13(value), c15(value), c33(value), c35(value), - c55(value) {} - - /** - * @brief Constructor - * - * @param rho \f$ K_{\rho} \f$ - * @param c11 \f$ K_{c_{11}} \f$ - * @param c13 \f$ K_{c_{13}} \f$ - * @param c15 \f$ K_{c_{15}} \f$ - * @param c33 \f$ K_{c_{33}} \f$ - * @param c35 \f$ K_{c_{35}} \f$ - * @param c55 \f$ K_{c_{55}} \f$ - */ - KOKKOS_FUNCTION - kernels(const value_type rho, const value_type c11, const value_type c13, - const value_type c15, const value_type c33, const value_type c35, - const value_type c55) - : rho(rho), c11(c11), c13(c13), c15(c15), c33(c33), c35(c35), c55(c55) {} + ///@{ + DEFINE_POINT_VALUE(rho) ///< \f$ K_{\rho} \f$ + DEFINE_POINT_VALUE(c11) ///< \f$ K_{c_{11}} \f$ + DEFINE_POINT_VALUE(c13) ///< \f$ K_{c_{13}} \f$ + DEFINE_POINT_VALUE(c15) ///< \f$ K_{c_{15}} \f$ + DEFINE_POINT_VALUE(c33) ///< \f$ K_{c_{33}} \f$ + DEFINE_POINT_VALUE(c35) ///< \f$ K_{c_{35}} \f$ + DEFINE_POINT_VALUE(c55) ///< \f$ K_{c_{55}} \f$ ///@} - - /** - * @brief Equality operator - * - */ - KOKKOS_FUNCTION - bool operator==(const kernels &rhs) const { - return rho == rhs.rho && c11 == rhs.c11 && c13 == rhs.c13 && - c15 == rhs.c15 && c33 == rhs.c33 && c35 == rhs.c35 && c55 == rhs.c55; - } - - /** - * @brief Inequality operator - * - */ - KOKKOS_FUNCTION - bool operator!=(const kernels &rhs) const { return !(*this == rhs); } - - KOKKOS_FUNCTION - bool operator==(const value_type value) { - return rho == value && c11 == value && c13 == value && c15 == value && - c33 == value && c35 == value && c55 == value; - } - - KOKKOS_FUNCTION - bool operator!=(const value_type value) { return !(*this == value); } }; // end elastic anisotropic @@ -259,62 +121,27 @@ struct kernels struct kernels { -public: + specfem::element::property_tag::isotropic, UseSIMD> + : public impl::values<4, UseSIMD> { + /** * @name Typedefs * */ ///@{ - using simd = typename specfem::datatype::simd; ///< SIMD - ///< type - using value_type = typename simd::datatype; ///< Underlying data type to store - ///< the kernels - ///@} + using base_type = impl::values<4, UseSIMD>; + using value_type = typename base_type::value_type; -public: - /** - * @name Compile time constants - * - */ - ///@{ - constexpr static auto medium_tag = specfem::element::medium_tag::acoustic; + constexpr static auto dimension = specfem::dimension::type::dim2; + constexpr static auto medium_tag = specfem::element::medium_tag::elastic; constexpr static auto property_tag = specfem::element::property_tag::isotropic; - constexpr static auto dimension = specfem::dimension::type::dim2; - ///@} -public: - /** - * @name Misfit Kernels - * - */ - ///@{ - value_type rho; ///< \f$ K_{\rho} \f$ - value_type kappa; ///< \f$ K_{\kappa} \f$ - value_type rhop; ///< \f$ K_{\rho'} \f$ - value_type alpha; ///< \f$ K_{\alpha} \f$ + constexpr static bool is_point_properties = true; + constexpr static int _counter = __COUNTER__; ///@} - /** - * @name Constructors - * - */ - ///@{ - /** - * @brief Default constructor - * - */ - KOKKOS_FUNCTION - kernels() = default; - - /** - * @brief single value constructor - * - */ - KOKKOS_FUNCTION - kernels(const value_type value) - : rho(value), kappa(value), rhop(value), alpha(value) {} + using base_type::base_type; /** * @brief Constructor @@ -324,36 +151,18 @@ struct kernels(2.0) * kappa; - } - ///@} - - /** - * @brief Equality operator - * - */ - KOKKOS_FUNCTION - bool operator==(const kernels &rhs) const { - return rho == rhs.rho && kappa == rhs.kappa && rhop == rhs.rhop && - alpha == rhs.alpha; - } + : kernels(rho, kappa, rho * kappa, static_cast(2.0) * kappa) {} /** - * @brief Inequality operator + * @name Misfit Kernels * */ - KOKKOS_FUNCTION - bool operator!=(const kernels &rhs) const { return !(*this == rhs); } - - KOKKOS_FUNCTION - bool operator==(const value_type value) { - return rho == value && kappa == value && rhop == value && alpha == value; - } - - KOKKOS_FUNCTION - bool operator!=(const value_type value) { return !(*this == value); } + ///@{ + DEFINE_POINT_VALUE(rho) ///< \f$ K_{\rho} \f$ + DEFINE_POINT_VALUE(kappa) ///< \f$ K_{\kappa} \f$ + DEFINE_POINT_VALUE(rhop) ///< \f$ K_{\rho'} \f$ + DEFINE_POINT_VALUE(alpha) ///< \f$ K_{\alpha} \f$ + ///@} }; } // namespace point diff --git a/tests/unit-tests/assembly/kernels/kernels.cpp b/tests/unit-tests/assembly/kernels/kernels.cpp index f3aca0af..4eb8ba30 100644 --- a/tests/unit-tests/assembly/kernels/kernels.cpp +++ b/tests/unit-tests/assembly/kernels/kernels.cpp @@ -24,12 +24,12 @@ std::string get_error_message( message << "\n\t Expected: " << value; message << "\n\t Got: \n"; - message << "\t\trho = " << point_kernel.rho << "\n"; - message << "\t\tmu = " << point_kernel.mu << "\n"; - message << "\t\tkappa = " << point_kernel.kappa << "\n"; - message << "\t\trhop = " << point_kernel.rhop << "\n"; - message << "\t\talpha = " << point_kernel.alpha << "\n"; - message << "\t\tbeta = " << point_kernel.beta << "\n"; + message << "\t\trho = " << point_kernel.rho() << "\n"; + message << "\t\tmu = " << point_kernel.mu() << "\n"; + message << "\t\tkappa = " << point_kernel.kappa() << "\n"; + message << "\t\trhop = " << point_kernel.rhop() << "\n"; + message << "\t\talpha = " << point_kernel.alpha() << "\n"; + message << "\t\tbeta = " << point_kernel.beta() << "\n"; return message.str(); } @@ -44,13 +44,13 @@ std::string get_error_message( message << "\n\t Expected: " << value; message << "\n\t Got: \n"; - message << "\t\trho = " << point_kernel.rho << "\n"; - message << "\t\tc11 = " << point_kernel.c11 << "\n"; - message << "\t\tc13 = " << point_kernel.c13 << "\n"; - message << "\t\tc15 = " << point_kernel.c15 << "\n"; - message << "\t\tc33 = " << point_kernel.c33 << "\n"; - message << "\t\tc35 = " << point_kernel.c35 << "\n"; - message << "\t\tc55 = " << point_kernel.c55 << "\n"; + message << "\t\trho = " << point_kernel.rho() << "\n"; + message << "\t\tc11 = " << point_kernel.c11() << "\n"; + message << "\t\tc13 = " << point_kernel.c13() << "\n"; + message << "\t\tc15 = " << point_kernel.c15() << "\n"; + message << "\t\tc33 = " << point_kernel.c33() << "\n"; + message << "\t\tc35 = " << point_kernel.c35() << "\n"; + message << "\t\tc55 = " << point_kernel.c55() << "\n"; return message.str(); } @@ -65,10 +65,10 @@ std::string get_error_message( message << "\n\t Expected: " << value; message << "\n\t Got: \n"; - message << "\t\trho = " << point_kernel.rho << "\n"; - message << "\t\tkappa = " << point_kernel.kappa << "\n"; - message << "\t\trhop = " << point_kernel.rhop << "\n"; - message << "\t\talpha = " << point_kernel.alpha << "\n"; + message << "\t\trho = " << point_kernel.rho() << "\n"; + message << "\t\tkappa = " << point_kernel.kappa() << "\n"; + message << "\t\trhop = " << point_kernel.rhop() << "\n"; + message << "\t\talpha = " << point_kernel.alpha() << "\n"; return message.str(); } @@ -105,12 +105,12 @@ get_point_kernel(const int ispec, const int iz, const int ix, specfem::element::property_tag::isotropic, false> point_kernel; - point_kernel.rho = elastic_isotropic.h_rho(ispec_l, iz, ix); - point_kernel.mu = elastic_isotropic.h_mu(ispec_l, iz, ix); - point_kernel.kappa = elastic_isotropic.h_kappa(ispec_l, iz, ix); - point_kernel.rhop = elastic_isotropic.h_rhop(ispec_l, iz, ix); - point_kernel.alpha = elastic_isotropic.h_alpha(ispec_l, iz, ix); - point_kernel.beta = elastic_isotropic.h_beta(ispec_l, iz, ix); + point_kernel.rho(elastic_isotropic.h_rho(ispec_l, iz, ix)); + point_kernel.mu(elastic_isotropic.h_mu(ispec_l, iz, ix)); + point_kernel.kappa(elastic_isotropic.h_kappa(ispec_l, iz, ix)); + point_kernel.rhop(elastic_isotropic.h_rhop(ispec_l, iz, ix)); + point_kernel.alpha(elastic_isotropic.h_alpha(ispec_l, iz, ix)); + point_kernel.beta(elastic_isotropic.h_beta(ispec_l, iz, ix)); return point_kernel; } @@ -129,12 +129,12 @@ get_point_kernel( specfem::element::property_tag::isotropic, false> point_kernel_l; - point_kernel_l.rho = point_kernel.rho[lane]; - point_kernel_l.mu = point_kernel.mu[lane]; - point_kernel_l.kappa = point_kernel.kappa[lane]; - point_kernel_l.rhop = point_kernel.rhop[lane]; - point_kernel_l.alpha = point_kernel.alpha[lane]; - point_kernel_l.beta = point_kernel.beta[lane]; + point_kernel_l.rho(point_kernel.rho()[lane]); + point_kernel_l.mu(point_kernel.mu()[lane]); + point_kernel_l.kappa(point_kernel.kappa()[lane]); + point_kernel_l.rhop(point_kernel.rhop()[lane]); + point_kernel_l.alpha(point_kernel.alpha()[lane]); + point_kernel_l.beta(point_kernel.beta()[lane]); return point_kernel_l; } @@ -155,13 +155,13 @@ get_point_kernel(const int ispec, const int iz, const int ix, specfem::element::property_tag::anisotropic, false> point_kernel; - point_kernel.rho = elastic_anisotropic.h_rho(ispec_l, iz, ix); - point_kernel.c11 = elastic_anisotropic.h_c11(ispec_l, iz, ix); - point_kernel.c13 = elastic_anisotropic.h_c13(ispec_l, iz, ix); - point_kernel.c15 = elastic_anisotropic.h_c15(ispec_l, iz, ix); - point_kernel.c33 = elastic_anisotropic.h_c33(ispec_l, iz, ix); - point_kernel.c35 = elastic_anisotropic.h_c35(ispec_l, iz, ix); - point_kernel.c55 = elastic_anisotropic.h_c55(ispec_l, iz, ix); + point_kernel.rho(elastic_anisotropic.h_rho(ispec_l, iz, ix)); + point_kernel.c11(elastic_anisotropic.h_c11(ispec_l, iz, ix)); + point_kernel.c13(elastic_anisotropic.h_c13(ispec_l, iz, ix)); + point_kernel.c15(elastic_anisotropic.h_c15(ispec_l, iz, ix)); + point_kernel.c33(elastic_anisotropic.h_c33(ispec_l, iz, ix)); + point_kernel.c35(elastic_anisotropic.h_c35(ispec_l, iz, ix)); + point_kernel.c55(elastic_anisotropic.h_c55(ispec_l, iz, ix)); return point_kernel; } @@ -180,13 +180,13 @@ get_point_kernel( specfem::element::property_tag::anisotropic, false> point_kernel_l; - point_kernel_l.rho = point_kernel.rho[lane]; - point_kernel_l.c11 = point_kernel.c11[lane]; - point_kernel_l.c13 = point_kernel.c13[lane]; - point_kernel_l.c15 = point_kernel.c15[lane]; - point_kernel_l.c33 = point_kernel.c33[lane]; - point_kernel_l.c35 = point_kernel.c35[lane]; - point_kernel_l.c55 = point_kernel.c55[lane]; + point_kernel_l.rho(point_kernel.rho()[lane]); + point_kernel_l.c11(point_kernel.c11()[lane]); + point_kernel_l.c13(point_kernel.c13()[lane]); + point_kernel_l.c15(point_kernel.c15()[lane]); + point_kernel_l.c33(point_kernel.c33()[lane]); + point_kernel_l.c35(point_kernel.c35()[lane]); + point_kernel_l.c55(point_kernel.c55()[lane]); return point_kernel_l; } @@ -207,10 +207,10 @@ get_point_kernel(const int ispec, const int iz, const int ix, specfem::element::property_tag::isotropic, false> point_kernel; - point_kernel.rho = acoustic_isotropic.h_rho(ispec_l, iz, ix); - point_kernel.kappa = acoustic_isotropic.h_kappa(ispec_l, iz, ix); - point_kernel.alpha = acoustic_isotropic.h_alpha(ispec_l, iz, ix); - point_kernel.rhop = acoustic_isotropic.h_rho_prime(ispec_l, iz, ix); + point_kernel.rho(acoustic_isotropic.h_rho(ispec_l, iz, ix)); + point_kernel.kappa(acoustic_isotropic.h_kappa(ispec_l, iz, ix)); + point_kernel.alpha(acoustic_isotropic.h_alpha(ispec_l, iz, ix)); + point_kernel.rhop(acoustic_isotropic.h_rhop(ispec_l, iz, ix)); return point_kernel; } @@ -229,10 +229,10 @@ get_point_kernel( specfem::element::property_tag::isotropic, false> point_kernel_l; - point_kernel_l.rho = point_kernel.rho[lane]; - point_kernel_l.kappa = point_kernel.kappa[lane]; - point_kernel_l.alpha = point_kernel.alpha[lane]; - point_kernel_l.rhop = point_kernel.rhop[lane]; + point_kernel_l.rho(point_kernel.rho()[lane]); + point_kernel_l.kappa(point_kernel.kappa()[lane]); + point_kernel_l.alpha(point_kernel.alpha()[lane]); + point_kernel_l.rhop(point_kernel.rhop()[lane]); return point_kernel_l; } @@ -254,6 +254,10 @@ void check_to_value(const specfem::compute::element_types &element_types, constexpr int simd_size = specfem::datatype::simd::size(); + using PointType = specfem::point::kernels; + constexpr int nprops = PointType::nprops; + for (int i = 0; i < ispecs.extent(0); ++i) { for (int iz = 0; iz < ngllz; iz++) { for (int ix = 0; ix < ngllx; ix++) { @@ -265,14 +269,16 @@ void check_to_value(const specfem::compute::element_types &element_types, const auto point_kernel = get_point_kernel( ielement + j, iz, ix, kernels); const type_real value = values_to_store(i); - if (point_kernel != value) { - std::ostringstream message; + for (int l = 0; l < nprops; l++) { + if (point_kernel.data[l] != value) { + std::ostringstream message; - message << "\n \t Error at ispec = " << ielement + j - << ", iz = " << iz << ", ix = " << ix; - message << get_error_message(point_kernel, value); + message << "\n \t Error at ispec = " << ielement + j + << ", iz = " << iz << ", ix = " << ix; + message << get_error_message(point_kernel, value); - throw std::runtime_error(message.str()); + throw std::runtime_error(message.str()); + } } } } @@ -313,7 +319,10 @@ void execute_store_or_add(specfem::compute::kernels &kernels, const auto index = get_index(ielement, n_simd_elements, iz, ix); const type_real value = values_to_store(i); - PointType point(value); + PointType point; + for (int l = 0; l < PointType::nprops; l++) { + point.data[l] = value; + } if constexpr (Store) { specfem::compute::store_on_device(index, point, kernels_l); } else if constexpr (Add) { @@ -411,6 +420,7 @@ void check_load_on_device( const int element_size = elements.extent(0); const int step = element_size / N; + constexpr int nprops = PointType::nprops; for (int i = 0; i < N; i++) { ispecs_h(i) = elements(i * step); @@ -459,29 +469,34 @@ void check_load_on_device( if constexpr (using_simd) { for (int lane = 0; lane < n_simd_elements; lane++) { const auto point_kernel_l = get_point_kernel(lane, point_kernel); - if (point_kernel_l != value_l) { + for (int l = 0; l < nprops; l++) { + if (point_kernel_l.data[l] != value_l) { + std::ostringstream message; + + message << "\n \t Error in function load_on_device"; + + message << "\n \t Error at ispec = " << ielement + << ", iz = " << 0 << ", ix = " << 0; + message << get_error_message(point_kernel_l, value_l); + + throw std::runtime_error(message.str()); + } + } + } + } else if constexpr (!using_simd) { + for (int l = 0; l < nprops; l++) { + if (point_kernel.data[l] != value_l) { std::ostringstream message; message << "\n \t Error in function load_on_device"; message << "\n \t Error at ispec = " << ielement << ", iz = " << 0 << ", ix = " << 0; - message << get_error_message(point_kernel_l, value_l); + message << get_error_message(point_kernel, value_l); throw std::runtime_error(message.str()); } } - } else if constexpr (!using_simd) { - if (point_kernel != value_l) { - std::ostringstream message; - message << "\n \t Error in function load_on_device"; - - message << "\n \t Error at ispec = " << ielement << ", iz = " << 0 - << ", ix = " << 0; - message << get_error_message(point_kernel, value_l); - - throw std::runtime_error(message.str()); - } } } } From 4df6e08d950c8e2250e8381d7a784ec78ee092f7 Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Mon, 3 Mar 2025 13:04:20 -0500 Subject: [PATCH 14/21] Make on_device always constexr. --- include/compute/compute_mesh.hpp | 22 ++- .../compute/compute_partial_derivatives.hpp | 75 ++++---- include/compute/fields/data_access.tpp | 180 ++++++------------ include/compute/fields/simulation_field.hpp | 17 ++ 4 files changed, 133 insertions(+), 161 deletions(-) diff --git a/include/compute/compute_mesh.hpp b/include/compute/compute_mesh.hpp index e2aea2bf..1bf4d4f1 100644 --- a/include/compute/compute_mesh.hpp +++ b/include/compute/compute_mesh.hpp @@ -260,16 +260,22 @@ impl_load(const MemberType &team, int ix, iz; sub2ind(xz, NGLL, iz, ix); if constexpr (store_hprime_gll) { - element_quadrature.hprime_gll(iz, ix) = - on_device ? quadrature.gll.hprime(iz, ix) - : quadrature.gll.h_hprime(iz, ix); + if constexpr (on_device) { + element_quadrature.hprime_gll(iz, ix) = + quadrature.gll.hprime(iz, ix); + } else { + element_quadrature.hprime_gll(iz, ix) = + quadrature.gll.h_hprime(iz, ix); + } } if constexpr (store_weight_times_hprime_gll) { - element_quadrature.hprime_wgll(ix, iz) = - on_device - ? quadrature.gll.hprime(iz, ix) * quadrature.gll.weights(iz) - : quadrature.gll.h_hprime(iz, ix) * - quadrature.gll.h_weights(iz); + if constexpr (on_device) { + element_quadrature.hprime_wgll(ix, iz) = + quadrature.gll.hprime(iz, ix) * quadrature.gll.weights(iz); + } else { + element_quadrature.hprime_wgll(ix, iz) = + quadrature.gll.h_hprime(iz, ix) * quadrature.gll.h_weights(iz); + } } }); } diff --git a/include/compute/compute_partial_derivatives.hpp b/include/compute/compute_partial_derivatives.hpp index 0244fc7d..4a144084 100644 --- a/include/compute/compute_partial_derivatives.hpp +++ b/include/compute/compute_partial_derivatives.hpp @@ -94,27 +94,32 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_load( mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - Kokkos::Experimental::where(mask, partial_derivatives.xix) - .copy_from(on_device ? &derivatives.xix(ispec, iz, ix) - : &derivatives.h_xix(ispec, iz, ix), - tag_type()); - Kokkos::Experimental::where(mask, partial_derivatives.gammax) - .copy_from(on_device ? &derivatives.gammax(ispec, iz, ix) - : &derivatives.h_gammax(ispec, iz, ix), - tag_type()); - Kokkos::Experimental::where(mask, partial_derivatives.xiz) - .copy_from(on_device ? &derivatives.xiz(ispec, iz, ix) - : &derivatives.h_xiz(ispec, iz, ix), - tag_type()); - Kokkos::Experimental::where(mask, partial_derivatives.gammaz) - .copy_from(on_device ? &derivatives.gammaz(ispec, iz, ix) - : &derivatives.h_gammaz(ispec, iz, ix), - tag_type()); - if constexpr (StoreJacobian) { - Kokkos::Experimental::where(mask, partial_derivatives.jacobian) - .copy_from(on_device ? &derivatives.jacobian(ispec, iz, ix) - : &derivatives.h_jacobian(ispec, iz, ix), - tag_type()); + if constexpr (on_device) { + Kokkos::Experimental::where(mask, partial_derivatives.xix) + .copy_from(&derivatives.xix(ispec, iz, ix), tag_type()); + Kokkos::Experimental::where(mask, partial_derivatives.gammax) + .copy_from(&derivatives.gammax(ispec, iz, ix), tag_type()); + Kokkos::Experimental::where(mask, partial_derivatives.xiz) + .copy_from(&derivatives.xiz(ispec, iz, ix), tag_type()); + Kokkos::Experimental::where(mask, partial_derivatives.gammaz) + .copy_from(&derivatives.gammaz(ispec, iz, ix), tag_type()); + if constexpr (StoreJacobian) { + Kokkos::Experimental::where(mask, partial_derivatives.jacobian) + .copy_from(&derivatives.jacobian(ispec, iz, ix), tag_type()); + } + } else { + Kokkos::Experimental::where(mask, partial_derivatives.xix) + .copy_from(&derivatives.h_xix(ispec, iz, ix), tag_type()); + Kokkos::Experimental::where(mask, partial_derivatives.gammax) + .copy_from(&derivatives.h_gammax(ispec, iz, ix), tag_type()); + Kokkos::Experimental::where(mask, partial_derivatives.xiz) + .copy_from(&derivatives.h_xiz(ispec, iz, ix), tag_type()); + Kokkos::Experimental::where(mask, partial_derivatives.gammaz) + .copy_from(&derivatives.h_gammaz(ispec, iz, ix), tag_type()); + if constexpr (StoreJacobian) { + Kokkos::Experimental::where(mask, partial_derivatives.jacobian) + .copy_from(&derivatives.h_jacobian(ispec, iz, ix), tag_type()); + } } } @@ -133,18 +138,22 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_load( constexpr static bool StoreJacobian = PointPartialDerivativesType::store_jacobian; - partial_derivatives.xix = on_device ? derivatives.xix(ispec, iz, ix) - : derivatives.h_xix(ispec, iz, ix); - partial_derivatives.gammax = on_device ? derivatives.gammax(ispec, iz, ix) - : derivatives.h_gammax(ispec, iz, ix); - partial_derivatives.xiz = on_device ? derivatives.xiz(ispec, iz, ix) - : derivatives.h_xiz(ispec, iz, ix); - partial_derivatives.gammaz = on_device ? derivatives.gammaz(ispec, iz, ix) - : derivatives.h_gammaz(ispec, iz, ix); - if constexpr (StoreJacobian) { - partial_derivatives.jacobian = on_device - ? derivatives.jacobian(ispec, iz, ix) - : derivatives.h_jacobian(ispec, iz, ix); + if constexpr (on_device) { + partial_derivatives.xix = derivatives.xix(ispec, iz, ix); + partial_derivatives.gammax = derivatives.gammax(ispec, iz, ix); + partial_derivatives.xiz = derivatives.xiz(ispec, iz, ix); + partial_derivatives.gammaz = derivatives.gammaz(ispec, iz, ix); + if constexpr (StoreJacobian) { + partial_derivatives.jacobian = derivatives.jacobian(ispec, iz, ix); + } + } else { + partial_derivatives.xix = derivatives.h_xix(ispec, iz, ix); + partial_derivatives.gammax = derivatives.h_gammax(ispec, iz, ix); + partial_derivatives.xiz = derivatives.h_xiz(ispec, iz, ix); + partial_derivatives.gammaz = derivatives.h_gammaz(ispec, iz, ix); + if constexpr (StoreJacobian) { + partial_derivatives.jacobian = derivatives.h_jacobian(ispec, iz, ix); + } } } diff --git a/include/compute/fields/data_access.tpp b/include/compute/fields/data_access.tpp index f8aea0ec..aff301a4 100644 --- a/include/compute/fields/data_access.tpp +++ b/include/compute/fields/data_access.tpp @@ -18,9 +18,9 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_load(const int iglob, constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < ViewType::components; ++icomp) { @@ -66,10 +66,10 @@ impl_load(const typename ViewType::simd::mask_type &mask, constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); for (int lane = 0; lane < ViewType::simd::size(); ++lane) { if (!mask[lane]) { @@ -122,7 +122,7 @@ impl_load(const specfem::point::simd_assembly_index &index, constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; const int iglob = index.iglob; @@ -132,7 +132,7 @@ impl_load(const specfem::point::simd_assembly_index &index, mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { @@ -172,11 +172,8 @@ template &index, const WavefieldType &field, ViewType &point_field) { - constexpr static auto MediumType = ViewType::medium_tag; - const int iglob_l = on_device ? field.index_mapping(index.ispec, index.iz, index.ix) : field.h_index_mapping(index.ispec, index.iz, index.ix); - const int iglob = on_device ? - field.assembly_index_mapping(iglob_l, static_cast(MediumType)) : field.h_assembly_index_mapping(iglob_l, static_cast(MediumType)); - impl_load(iglob, field, point_field); + constexpr static auto MediumTag = ViewType::medium_tag; + impl_load(field.template get_iglob(index.ispec, index.iz, index.ix, MediumTag), field, point_field); } template &index, const WavefieldType &field, ViewType &point_field) { - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; const int iglob = index.iglob; impl_load(iglob, field, point_field); } @@ -198,7 +195,7 @@ template &index, const WavefieldType &field, ViewType &point_field) { - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; int iglob[ViewType::simd::size()]; using mask_type = typename ViewType::simd::mask_type; @@ -206,17 +203,10 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_load( mask_type mask([&](std::size_t lane) { return index.mask(lane); }); for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = on_device ? - ((index.mask(std::size_t(lane))) - ? field.assembly_index_mapping( - field.index_mapping(index.ispec + lane, index.iz, index.ix), - static_cast(MediumType)) - : field.nglob + 1) : - ((index.mask(std::size_t(lane))) - ? field.h_assembly_index_mapping( - field.h_index_mapping(index.ispec + lane, index.iz, index.ix), - static_cast(MediumType)) - : field.nglob + 1); + iglob[lane] = index.mask(lane) ? + field.template get_iglob( + index.ispec + lane, index.iz, index.ix, MediumTag) : + field.nglob + 1; } impl_load(mask, iglob, field, point_field); @@ -234,9 +224,9 @@ impl_store(const int iglob, const ViewType &point_field, constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); for (int icomp = 0; icomp < ViewType::components; ++icomp) { if constexpr (StoreDisplacement) { @@ -269,10 +259,10 @@ impl_store(const typename ViewType::simd::mask_type &mask, constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); for (int lane = 0; lane < ViewType::simd::size(); ++lane) { if (!mask[lane]) { continue; @@ -324,7 +314,7 @@ impl_store(const specfem::point::simd_assembly_index &index, constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; using mask_type = typename ViewType::simd::mask_type; @@ -334,7 +324,7 @@ impl_store(const specfem::point::simd_assembly_index &index, mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.displacement(icomp)) @@ -374,15 +364,9 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_store(const specfem::point::index &index, const ViewType &point_field, const WavefieldType &field) { - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = on_device ? field.assembly_index_mapping( - field.index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)) : field.h_assembly_index_mapping( - field.h_index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); + constexpr static auto MediumTag = ViewType::medium_tag; - impl_store(iglob, point_field, field); + impl_store(field.template get_iglob(index.ispec, index.iz, index.ix, MediumTag), point_field, field); } template &index, const ViewType &point_field, const WavefieldType &field) { - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; const int iglob = index.iglob; @@ -408,7 +392,7 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_store( const specfem::point::simd_index &index, const ViewType &point_field, const WavefieldType &field) { - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; int iglob[ViewType::simd::size()]; using mask_type = typename ViewType::simd::mask_type; @@ -416,17 +400,10 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_store( mask_type mask([&](std::size_t lane) { return index.mask(lane); }); for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = on_device ? - ((index.mask(std::size_t(lane))) - ? field.assembly_index_mapping( - field.index_mapping(index.ispec + lane, index.iz, index.ix), - static_cast(MediumType)) - : field.nglob + 1) : - ((index.mask(std::size_t(lane))) - ? field.assembly_index_mapping( - field.index_mapping(index.ispec + lane, index.iz, index.ix), - static_cast(MediumType)) - : field.nglob + 1); + iglob[lane] = index.mask(lane) ? + field.template get_iglob( + index.ispec + lane, index.iz, index.ix, MediumTag) : + field.nglob + 1; } impl_store(mask, iglob, point_field, field); @@ -444,9 +421,9 @@ impl_add(const int iglob, const ViewType &point_field, constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); for (int icomp = 0; icomp < ViewType::components; ++icomp) { if constexpr (StoreDisplacement) { curr_field.template get_field(iglob, icomp) += point_field.displacement(icomp); @@ -478,10 +455,10 @@ impl_add(const typename ViewType::simd::mask_type &mask, constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); for (int lane = 0; lane < ViewType::simd::size(); ++lane) { if (!mask[lane]) { continue; @@ -533,7 +510,7 @@ impl_add(const specfem::point::simd_assembly_index &index, constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; using mask_type = typename ViewType::simd::mask_type; @@ -543,7 +520,7 @@ impl_add(const specfem::point::simd_assembly_index &index, mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { typename ViewType::simd::datatype lhs; @@ -603,15 +580,8 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_add(const specfem::point::index &index, const ViewType &point_field, const WavefieldType &field) { - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = on_device ? field.assembly_index_mapping( - field.index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)) : field.h_assembly_index_mapping( - field.h_index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); - - impl_add(iglob, point_field, field); + constexpr static auto MediumTag = ViewType::medium_tag; + impl_add(field.template get_iglob(index.ispec, index.iz, index.ix, MediumTag), point_field, field); } template &index, const ViewType &point_field, const WavefieldType &field) { - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; const int iglob = index.iglob; @@ -637,7 +607,7 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_add(const specfem::point::simd_index &index, const ViewType &point_field, const WavefieldType &field) { - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; int iglob[ViewType::simd::size()]; using mask_type = typename ViewType::simd::mask_type; @@ -645,17 +615,9 @@ impl_add(const specfem::point::simd_index &index, mask_type mask([&](std::size_t lane) { return index.mask(lane); }); for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = on_device ? - ((index.mask(std::size_t(lane))) - ? field.assembly_index_mapping( - field.index_mapping(index.ispec + lane, index.iz, index.ix), - static_cast(MediumType)) - : field.nglob + 1) : - ((index.mask(std::size_t(lane))) - ? field.h_assembly_index_mapping( - field.h_index_mapping(index.ispec + lane, index.iz, index.ix), - static_cast(MediumType)) - : field.nglob + 1); + iglob[lane] = index.mask(lane) ? + field.template get_iglob(index.ispec + lane, index.iz, index.ix, MediumTag) : + field.nglob + 1; } impl_add(mask, iglob, point_field, field); @@ -673,9 +635,9 @@ impl_atomic_add(const int iglob, const ViewType &point_field, constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); for (int icomp = 0; icomp < ViewType::components; ++icomp) { if constexpr (StoreDisplacement) { Kokkos::atomic_add(&(curr_field.template get_field(iglob, icomp)), @@ -711,10 +673,10 @@ impl_atomic_add(const typename ViewType::simd::mask_type &mask, constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); for (int lane = 0; lane < ViewType::simd::size(); ++lane) { if (!mask[lane]) { continue; @@ -762,15 +724,9 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_atomic_add( const specfem::point::index &index, const ViewType &point_field, const WavefieldType &field) { - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = on_device ? field.assembly_index_mapping( - field.index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)) : field.h_assembly_index_mapping( - field.h_index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); + constexpr static auto MediumTag = ViewType::medium_tag; - impl_atomic_add(iglob, point_field, field); + impl_atomic_add(field.template get_iglob(index.ispec, index.iz, index.ix, MediumTag), point_field, field); } template &index, const ViewType &point_field, const WavefieldType &field) { - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; int iglob[ViewType::simd::size()]; using mask_type = typename ViewType::simd::mask_type; @@ -789,17 +745,9 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_atomic_add( mask_type mask([&](std::size_t lane) { return index.mask(lane); }); for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = on_device ? - ((index.mask(std::size_t(lane))) - ? field.assembly_index_mapping( - field.index_mapping(index.ispec + lane, index.iz, index.ix), - static_cast(MediumType)) - : field.nglob + 1): - ((index.mask(std::size_t(lane))) - ? field.assembly_index_mapping( - field.index_mapping(index.ispec + lane, index.iz, index.ix), - static_cast(MediumType)) - : field.nglob + 1); + iglob[lane] = index.mask(lane) ? + field.template get_iglob(index.ispec + lane, index.iz, index.ix, MediumTag) : + field.nglob + 1; } impl_atomic_add(mask, iglob, point_field, field); @@ -817,7 +765,7 @@ impl_load(const MemberType &team, const int ispec, constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; constexpr static int NGLL = ViewType::ngll; @@ -831,14 +779,12 @@ impl_load(const MemberType &team, const int ispec, "Calling team must have a device execution space"); } - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); Kokkos::parallel_for( Kokkos::TeamThreadRange(team, NGLL * NGLL), [&](const int &xz) { int iz, ix; sub2ind(xz, NGLL, iz, ix); - const int iglob = on_device ? field.assembly_index_mapping( - field.index_mapping(ispec, iz, ix), static_cast(MediumType)) : field.h_assembly_index_mapping( - field.h_index_mapping(ispec, iz, ix), static_cast(MediumType)); + const int iglob = field.template get_iglob(ispec, iz, ix, MediumTag); for (int icomp = 0; icomp < components; ++icomp) { if constexpr (StoreDisplacement) { @@ -876,7 +822,7 @@ impl_load(const MemberType &team, const IteratorType &iterator, constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; constexpr static int NGLL = ViewType::ngll; @@ -900,7 +846,7 @@ impl_load(const MemberType &team, const IteratorType &iterator, typename ViewType::memory_space>::accessible, "Calling team must have access to the memory space of the view"); - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); Kokkos::parallel_for( Kokkos::TeamThreadRange(team, iterator.chunk_size()), [&](const int &i) { const auto iterator_index = iterator(i); @@ -909,9 +855,7 @@ impl_load(const MemberType &team, const IteratorType &iterator, const int iz = iterator_index.index.iz; const int ix = iterator_index.index.ix; - const int iglob = on_device ? field.assembly_index_mapping( - field.index_mapping(ispec, iz, ix), static_cast(MediumType)) : field.h_assembly_index_mapping( - field.h_index_mapping(ispec, iz, ix), static_cast(MediumType)); + const int iglob = field.template get_iglob(ispec, iz, ix, MediumTag); for (int icomp = 0; icomp < components; ++icomp) { if constexpr (StoreDisplacement) { @@ -949,7 +893,7 @@ impl_load(const MemberType &team, const IteratorType &iterator, constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; constexpr static int NGLL = ViewType::ngll; @@ -972,7 +916,7 @@ impl_load(const MemberType &team, const IteratorType &iterator, "Calling team must have a device execution space"); } - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); Kokkos::parallel_for( Kokkos::TeamThreadRange(team, iterator.chunk_size()), [&](const int &i) { const auto iterator_index = iterator(i); @@ -986,11 +930,7 @@ impl_load(const MemberType &team, const IteratorType &iterator, continue; } - const int iglob = on_device ? field.assembly_index_mapping( - field.index_mapping(ispec + lane, iz, ix), - static_cast(MediumType)) : field.h_assembly_index_mapping( - field.h_index_mapping(ispec + lane, iz, ix), - static_cast(MediumType)); + const int iglob = field.template get_iglob(ispec + lane, iz, ix, MediumTag); for (int icomp = 0; icomp < components; ++icomp) { if constexpr (StoreDisplacement) { diff --git a/include/compute/fields/simulation_field.hpp b/include/compute/fields/simulation_field.hpp index bd72ee7e..ba37c829 100644 --- a/include/compute/fields/simulation_field.hpp +++ b/include/compute/fields/simulation_field.hpp @@ -111,6 +111,23 @@ struct simulation_field { } } + /** + * @brief Returns the assembled index given element index. + * + */ + template + KOKKOS_INLINE_FUNCTION constexpr int + get_iglob(const int &ispec, const int &iz, const int &ix, + const specfem::element::medium_tag MediumTag) const { + if constexpr (on_device) { + return assembly_index_mapping(index_mapping(ispec, iz, ix), + static_cast(MediumTag)); + } else { + return h_assembly_index_mapping(h_index_mapping(ispec, iz, ix), + static_cast(MediumTag)); + } + } + int nglob = 0; ///< Number of global degrees of freedom int nspec; ///< Number of spectral elements int ngllz; ///< Number of quadrature points in z direction From 49d3abfcc780240d092577b1b349843d7c04301c Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Tue, 4 Mar 2025 16:30:33 -0500 Subject: [PATCH 15/21] Bug fix. --- examples/Tromp_2005/plot.py | 18 ++++++--- include/IO/kernel/writer.tpp | 60 ++---------------------------- include/IO/property/writer.tpp | 3 -- include/medium/impl/containers.hpp | 31 +++++++++------ include/point/kernels.hpp | 4 +- 5 files changed, 36 insertions(+), 80 deletions(-) diff --git a/examples/Tromp_2005/plot.py b/examples/Tromp_2005/plot.py index e644c9b8..775d423d 100644 --- a/examples/Tromp_2005/plot.py +++ b/examples/Tromp_2005/plot.py @@ -7,12 +7,14 @@ def load_data(directory): X = np.loadtxt(directory + "/ElasticIsotropic/X.txt") Z = np.loadtxt(directory + "/ElasticIsotropic/Z.txt") - rho = np.loadtxt(directory + "/ElasticIsotropic/rho.txt") - kappa = np.loadtxt(directory + "/ElasticIsotropic/kappa.txt") - mu = np.loadtxt(directory + "/ElasticIsotropic/mu.txt") - rhop = np.loadtxt(directory + "/ElasticIsotropic/rhop.txt") - alpha = np.loadtxt(directory + "/ElasticIsotropic/alpha.txt") - beta = np.loadtxt(directory + "/ElasticIsotropic/beta.txt") + data = np.loadtxt(directory + "/ElasticIsotropic/data.txt").reshape(X.shape[0], 6) + + rho = data[:, 0] + mu = data[:, 1] + kappa = data[:, 2] + rhop = data[:, 3] + alpha = data[:, 4] + beta = data[:, 5] return X, Z, rho, kappa, mu, rhop, alpha, beta @@ -97,3 +99,7 @@ def plot_kernels(input_directory, output): plt.savefig(output, dpi=300) return + + +if __name__ == "__main__": + plot_kernels("OUTPUT_FILES/Kernels", "Kernels_out.png") diff --git a/include/IO/kernel/writer.tpp b/include/IO/kernel/writer.tpp index ff1457ef..2b326e13 100644 --- a/include/IO/kernel/writer.tpp +++ b/include/IO/kernel/writer.tpp @@ -21,9 +21,6 @@ void specfem::IO::kernel_writer::write(specfem::compute::assembly using DomainView = Kokkos::View; - using DomainKernelView = - Kokkos::View; - kernels.copy_to_host(); typename OutputLibrary::File file(output_folder + "/Kernels"); @@ -47,36 +44,19 @@ void specfem::IO::kernel_writer::write(specfem::compute::assembly DomainView x("xcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, ngllx); DomainView z("zcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, ngllx); - using PointKernelType = typename specfem::point::kernels; - constexpr int nprops = PointKernelType::nprops; - - DomainKernelView data("data", n_elastic_isotropic, ngllz, ngllx, nprops); - for (int i = 0; i < n_elastic_isotropic; i++) { const int ispec = element_indices(i); for (int iz = 0; iz < ngllz; iz++) { for (int ix = 0; ix < ngllx; ix++) { x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); - const specfem::point::index index( - ispec, iz, ix); - PointKernelType point_kernels; - - specfem::compute::load_on_host(index, kernels, point_kernels); - - for (int l = 0; l < nprops; l++) { - data(i, iz, ix, l) = point_kernels.data[l]; - } } } } elastic.createDataset("X", x).write(); elastic.createDataset("Z", z).write(); - elastic.createDataset("data", data).write(); + elastic.createDataset("data", kernels.elastic_isotropic.h_data).write(); } { @@ -90,36 +70,19 @@ void specfem::IO::kernel_writer::write(specfem::compute::assembly DomainView x("xcoordinates_elastic_anisotropic", n_elastic_anisotropic, ngllz, ngllx); DomainView z("zcoordinates_elastic_anisotropic", n_elastic_anisotropic, ngllz, ngllx); - using PointKernelType = typename specfem::point::kernels; - constexpr int nprops = PointKernelType::nprops; - - DomainKernelView data("data", n_elastic_anisotropic, ngllz, ngllx, nprops); - for (int i = 0; i < n_elastic_anisotropic; i++) { const int ispec = element_indices(i); for (int iz = 0; iz < ngllz; iz++) { for (int ix = 0; ix < ngllx; ix++) { x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); - const specfem::point::index index( - ispec, iz, ix); - PointKernelType point_kernels; - - specfem::compute::load_on_host(index, kernels, point_kernels); - - for (int l = 0; l < nprops; l++) { - data(i, iz, ix, l) = point_kernels.data[l]; - } } } } elastic.createDataset("X", x).write(); elastic.createDataset("Z", z).write(); - elastic.createDataset("data", data).write(); + elastic.createDataset("data", kernels.elastic_anisotropic.h_data).write(); } { @@ -131,36 +94,19 @@ void specfem::IO::kernel_writer::write(specfem::compute::assembly DomainView x("xcoordinates_acoustic", n_acoustic, ngllz, ngllx); DomainView z("zcoordinates_acoustic", n_acoustic, ngllz, ngllx); - using PointKernelType = typename specfem::point::kernels; - constexpr int nprops = PointKernelType::nprops; - - DomainKernelView data("data", n_acoustic, ngllz, ngllx, nprops); - for (int i = 0; i < n_acoustic; i++) { const int ispec = element_indices(i); for (int iz = 0; iz < ngllz; iz++) { for (int ix = 0; ix < ngllx; ix++) { x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); - const specfem::point::index index( - ispec, iz, ix); - PointKernelType point_kernels; - - specfem::compute::load_on_host(index, kernels, point_kernels); - - for (int l = 0; l < nprops; l++) { - data(i, iz, ix, l) = point_kernels.data[l]; - } } } } acoustic.createDataset("X", x).write(); acoustic.createDataset("Z", z).write(); - acoustic.createDataset("data", data).write(); + acoustic.createDataset("data", kernels.acoustic_isotropic.h_data).write(); } assert(n_elastic_isotropic + n_elastic_anisotropic + n_acoustic == nspec); diff --git a/include/IO/property/writer.tpp b/include/IO/property/writer.tpp index b0682f0d..5706c6da 100644 --- a/include/IO/property/writer.tpp +++ b/include/IO/property/writer.tpp @@ -56,7 +56,6 @@ void specfem::IO::property_writer::write(specfem::compute::assemb elastic.createDataset("X", x).write(); elastic.createDataset("Z", z).write(); - elastic.createDataset("data", properties.elastic_isotropic.h_data).write(); } @@ -83,7 +82,6 @@ void specfem::IO::property_writer::write(specfem::compute::assemb elastic.createDataset("X", x).write(); elastic.createDataset("Z", z).write(); - elastic.createDataset("data", properties.elastic_anisotropic.h_data).write(); } @@ -108,7 +106,6 @@ void specfem::IO::property_writer::write(specfem::compute::assemb acoustic.createDataset("X", x).write(); acoustic.createDataset("Z", z).write(); - acoustic.createDataset("data", properties.acoustic_isotropic.h_data).write(); } diff --git a/include/medium/impl/containers.hpp b/include/medium/impl/containers.hpp index 6a2f0b0d..a2183c48 100644 --- a/include/medium/impl/containers.hpp +++ b/include/medium/impl/containers.hpp @@ -91,8 +91,9 @@ struct medium_container { } template - inline void store_values(const specfem::point::index &index, - const PointValues &values) const { + KOKKOS_FORCEINLINE_FUNCTION void + store_values(const specfem::point::index &index, + const PointValues &values) const { static_assert(PointValues::dimension == dimension, "Dimension mismatch"); static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); @@ -113,8 +114,9 @@ struct medium_container { } template - inline void store_values(const specfem::point::simd_index &index, - const PointValues &values) const { + KOKKOS_FORCEINLINE_FUNCTION void + store_values(const specfem::point::simd_index &index, + const PointValues &values) const { static_assert(PointValues::dimension == dimension, "Dimension mismatch"); static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); @@ -132,16 +134,20 @@ struct medium_container { mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); for (int i = 0; i < nprops; i++) { - Kokkos::Experimental::where(mask, values.data[i]) - .copy_to(on_device ? &data(ispec, iz, ix, i) - : &h_data(ispec, iz, ix, i), - tag_type()); + if constexpr (on_device) { + Kokkos::Experimental::where(mask, values.data[i]) + .copy_to(&data(ispec, iz, ix, i), tag_type()); + } else { + Kokkos::Experimental::where(mask, values.data[i]) + .copy_to(&h_data(ispec, iz, ix, i), tag_type()); + } } } template - inline void add_values(const specfem::point::index &index, - const PointValues &values) const { + KOKKOS_FORCEINLINE_FUNCTION void + add_values(const specfem::point::index &index, + const PointValues &values) const { static_assert(PointValues::dimension == dimension, "Dimension mismatch"); static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); @@ -162,8 +168,9 @@ struct medium_container { } template - inline void add_values(const specfem::point::simd_index &index, - const PointValues &values) const { + KOKKOS_FORCEINLINE_FUNCTION void + add_values(const specfem::point::simd_index &index, + const PointValues &values) const { static_assert(PointValues::dimension == dimension, "Dimension mismatch"); static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); diff --git a/include/point/kernels.hpp b/include/point/kernels.hpp index b1a05775..6877dbd6 100644 --- a/include/point/kernels.hpp +++ b/include/point/kernels.hpp @@ -88,7 +88,7 @@ struct kernels Date: Tue, 4 Mar 2025 16:49:43 -0500 Subject: [PATCH 16/21] Remove material_kernels. --- include/compute/kernels/interface.hpp | 1 - include/compute/kernels/kernels.hpp | 9 +++-- .../{containers.hpp => medium_container.hpp} | 0 include/medium/kernels_container.hpp | 2 +- include/medium/material_kernels.hpp | 38 ------------------- include/medium/properties_container.hpp | 2 +- src/compute/compute_kernels.cpp | 6 +-- 7 files changed, 10 insertions(+), 48 deletions(-) rename include/medium/impl/{containers.hpp => medium_container.hpp} (100%) delete mode 100644 include/medium/material_kernels.hpp diff --git a/include/compute/kernels/interface.hpp b/include/compute/kernels/interface.hpp index ec0e4570..47df4ddd 100644 --- a/include/compute/kernels/interface.hpp +++ b/include/compute/kernels/interface.hpp @@ -2,4 +2,3 @@ #include "kernels.hpp" #include "medium/kernels_container.hpp" -#include "medium/material_kernels.hpp" diff --git a/include/compute/kernels/kernels.hpp b/include/compute/kernels/kernels.hpp index fe09199d..03674c55 100644 --- a/include/compute/kernels/kernels.hpp +++ b/include/compute/kernels/kernels.hpp @@ -3,7 +3,7 @@ #include "compute/element_types/element_types.hpp" #include "compute/impl/value_containers.hpp" #include "enumerations/medium.hpp" -#include "medium/material_kernels.hpp" +#include "medium/kernels_container.hpp" #include "mesh/materials/materials.hpp" #include "point/coordinates.hpp" #include "point/kernels.hpp" @@ -17,7 +17,7 @@ namespace compute { * */ struct kernels - : public impl::value_containers { + : public impl::value_containers { public: /** * @name Constructors @@ -49,11 +49,12 @@ struct kernels * */ void copy_to_host() { - impl::value_containers::copy_to_host(); + impl::value_containers::copy_to_host(); } void copy_to_device() { - impl::value_containers::copy_to_device(); + impl::value_containers< + specfem::medium::kernels_container>::copy_to_device(); } }; diff --git a/include/medium/impl/containers.hpp b/include/medium/impl/medium_container.hpp similarity index 100% rename from include/medium/impl/containers.hpp rename to include/medium/impl/medium_container.hpp diff --git a/include/medium/kernels_container.hpp b/include/medium/kernels_container.hpp index f79b28e2..8e472f25 100644 --- a/include/medium/kernels_container.hpp +++ b/include/medium/kernels_container.hpp @@ -1,6 +1,6 @@ #pragma once -#include "impl/containers.hpp" +#include "impl/medium_container.hpp" namespace specfem { namespace medium { diff --git a/include/medium/material_kernels.hpp b/include/medium/material_kernels.hpp deleted file mode 100644 index 3b3196c1..00000000 --- a/include/medium/material_kernels.hpp +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include "compute/properties/properties.hpp" -#include "enumerations/interface.hpp" -#include "kernels_container.hpp" -#include "kokkos_abstractions.h" -#include - -namespace specfem { -namespace medium { - -template -class material_kernels : public kernels_container { -public: - constexpr static auto value_type = type; - constexpr static auto property_type = property; - - material_kernels() = default; - - material_kernels( - const Kokkos::View elements, - const int ngllz, const int ngllx, - const specfem::kokkos::HostView1d property_index_mapping) - : specfem::medium::kernels_container( - elements.extent(0), ngllz, ngllx) { - - const int nelement = elements.extent(0); - int count = 0; - for (int i = 0; i < nelement; ++i) { - const int ispec = elements(i); - property_index_mapping(ispec) = count; - count++; - } - } -}; -} // namespace medium -} // namespace specfem diff --git a/include/medium/properties_container.hpp b/include/medium/properties_container.hpp index 809a581f..d711159a 100644 --- a/include/medium/properties_container.hpp +++ b/include/medium/properties_container.hpp @@ -1,6 +1,6 @@ #pragma once -#include "impl/containers.hpp" +#include "impl/medium_container.hpp" namespace specfem { namespace medium { diff --git a/src/compute/compute_kernels.cpp b/src/compute/compute_kernels.cpp index c0580fdc..c507dea0 100644 --- a/src/compute/compute_kernels.cpp +++ b/src/compute/compute_kernels.cpp @@ -31,17 +31,17 @@ specfem::compute::kernels::kernels( h_property_index_mapping(ispec) = -1; } - acoustic_isotropic = specfem::medium::material_kernels< + acoustic_isotropic = specfem::medium::kernels_container< specfem::element::medium_tag::acoustic, specfem::element::property_tag::isotropic>( acoustic_elements, ngllz, ngllx, h_property_index_mapping); - elastic_isotropic = specfem::medium::material_kernels< + elastic_isotropic = specfem::medium::kernels_container< specfem::element::medium_tag::elastic, specfem::element::property_tag::isotropic>( elastic_isotropic_elements, ngllz, ngllx, h_property_index_mapping); - elastic_anisotropic = specfem::medium::material_kernels< + elastic_anisotropic = specfem::medium::kernels_container< specfem::element::medium_tag::elastic, specfem::element::property_tag::anisotropic>( elastic_anisotropic_elements, ngllz, ngllx, h_property_index_mapping); From 81be972a1046ffb4d7bea39f9edd2ba7c672501c Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Tue, 4 Mar 2025 17:01:54 -0500 Subject: [PATCH 17/21] Update plot.py --- examples/Tromp_2005/plot.py | 16 ++++++++-------- include/point/impl/values.hpp | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/Tromp_2005/plot.py b/examples/Tromp_2005/plot.py index 775d423d..dcccd145 100644 --- a/examples/Tromp_2005/plot.py +++ b/examples/Tromp_2005/plot.py @@ -7,14 +7,14 @@ def load_data(directory): X = np.loadtxt(directory + "/ElasticIsotropic/X.txt") Z = np.loadtxt(directory + "/ElasticIsotropic/Z.txt") - data = np.loadtxt(directory + "/ElasticIsotropic/data.txt").reshape(X.shape[0], 6) - - rho = data[:, 0] - mu = data[:, 1] - kappa = data[:, 2] - rhop = data[:, 3] - alpha = data[:, 4] - beta = data[:, 5] + data = np.loadtxt(directory + "/ElasticIsotropic/data.txt").reshape(6, X.shape[0]) + + rho = data[0, :] + mu = data[1, :] + kappa = data[2, :] + rhop = data[3, :] + alpha = data[4, :] + beta = data[5, :] return X, Z, rho, kappa, mu, rhop, alpha, beta diff --git a/include/point/impl/values.hpp b/include/point/impl/values.hpp index 4b43e98a..b592074f 100644 --- a/include/point/impl/values.hpp +++ b/include/point/impl/values.hpp @@ -43,7 +43,7 @@ template struct values { */ template = 0> - values(Args... args) : data{ args... } {} + KOKKOS_FUNCTION values(Args... args) : data{ args... } {} /** * @brief Equality operator From da3119dc690e7cddf73396df9e5bec8ca4de3d0a Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Tue, 4 Mar 2025 17:18:31 -0500 Subject: [PATCH 18/21] Update on_device template. --- include/medium/impl/medium_container.hpp | 44 ++++++++++-------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/include/medium/impl/medium_container.hpp b/include/medium/impl/medium_container.hpp index a2183c48..ab8a6140 100644 --- a/include/medium/impl/medium_container.hpp +++ b/include/medium/impl/medium_container.hpp @@ -42,6 +42,16 @@ struct medium_container { h_data(Kokkos::create_mirror_view(data)) {} private: + template + KOKKOS_FORCEINLINE_FUNCTION constexpr type_real & + get_data(const int &ispec, const int &iz, const int &ix, const int &i) const { + if constexpr (on_device) { + return data(ispec, iz, ix, i); + } else { + return h_data(ispec, iz, ix, i); + } + } + template KOKKOS_FORCEINLINE_FUNCTION void load_values(const specfem::point::index &index, @@ -57,8 +67,7 @@ struct medium_container { const int ix = index.ix; for (int i = 0; i < nprops; i++) { - values.data[i] = - on_device ? data(ispec, iz, ix, i) : h_data(ispec, iz, ix, i); + values.data[i] = get_data(ispec, iz, ix, i); } } @@ -84,9 +93,7 @@ struct medium_container { for (int i = 0; i < nprops; i++) { Kokkos::Experimental::where(mask, values.data[i]) - .copy_from(on_device ? &data(ispec, iz, ix, i) - : &h_data(ispec, iz, ix, i), - tag_type()); + .copy_from(&get_data(ispec, iz, ix, i), tag_type()); } } @@ -105,11 +112,7 @@ struct medium_container { const int ix = index.ix; for (int i = 0; i < nprops; i++) { - if constexpr (on_device) { - data(ispec, iz, ix, i) = values.data[i]; - } else { - h_data(ispec, iz, ix, i) = values.data[i]; - } + get_data(ispec, iz, ix, i) = values.data[i]; } } @@ -134,13 +137,8 @@ struct medium_container { mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); for (int i = 0; i < nprops; i++) { - if constexpr (on_device) { - Kokkos::Experimental::where(mask, values.data[i]) - .copy_to(&data(ispec, iz, ix, i), tag_type()); - } else { - Kokkos::Experimental::where(mask, values.data[i]) - .copy_to(&h_data(ispec, iz, ix, i), tag_type()); - } + Kokkos::Experimental::where(mask, values.data[i]) + .copy_to(&get_data(ispec, iz, ix, i), tag_type()); } } @@ -159,11 +157,7 @@ struct medium_container { const int ix = index.ix; for (int i = 0; i < nprops; i++) { - if constexpr (on_device) { - data(ispec, iz, ix, i) += values.data[i]; - } else { - h_data(ispec, iz, ix, i) += values.data[i]; - } + get_data(ispec, iz, ix, i) += values.data[i]; } } @@ -192,12 +186,10 @@ struct medium_container { for (int i = 0; i < nprops; i++) { Kokkos::Experimental::where(mask, lhs).copy_from( - on_device ? &data(ispec, iz, ix, i) : &h_data(ispec, iz, ix, i), - tag_type()); + &get_data(ispec, iz, ix, i), tag_type()); lhs += values.data[i]; Kokkos::Experimental::where(mask, lhs).copy_to( - on_device ? &data(ispec, iz, ix, i) : &h_data(ispec, iz, ix, i), - tag_type()); + &get_data(ispec, iz, ix, i), tag_type()); } } From a8a86b989caed9f9034bbe0f7fc0cd204fe42352 Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Tue, 4 Mar 2025 17:42:46 -0500 Subject: [PATCH 19/21] Renamings in point/impl and medium/impl --- .../impl/{medium_container.hpp => medium_data.hpp} | 8 ++++---- include/medium/kernels_container.hpp | 6 +++--- include/medium/properties_container.hpp | 6 +++--- include/point/impl/{values.hpp => point_data.hpp} | 12 ++++++------ include/point/kernels.hpp | 14 +++++++------- include/point/properties.hpp | 14 +++++++------- 6 files changed, 30 insertions(+), 30 deletions(-) rename include/medium/impl/{medium_container.hpp => medium_data.hpp} (97%) rename include/point/impl/{values.hpp => point_data.hpp} (83%) diff --git a/include/medium/impl/medium_container.hpp b/include/medium/impl/medium_data.hpp similarity index 97% rename from include/medium/impl/medium_container.hpp rename to include/medium/impl/medium_data.hpp index ab8a6140..d1151257 100644 --- a/include/medium/impl/medium_container.hpp +++ b/include/medium/impl/medium_data.hpp @@ -19,7 +19,7 @@ namespace medium { namespace impl { template -struct medium_container { +struct medium_data { using view_type = typename Kokkos::View; constexpr static auto nprops = N; @@ -34,9 +34,9 @@ struct medium_container { view_type data; typename view_type::HostMirror h_data; - medium_container() = default; + medium_data() = default; - medium_container(const int nspec, const int ngllz, const int ngllx) + medium_data(const int nspec, const int ngllz, const int ngllx) : nspec(nspec), ngllz(ngllz), ngllx(ngllx), data("specfem::medium::impl::container::data", nspec, ngllz, ngllx, N), h_data(Kokkos::create_mirror_view(data)) {} @@ -93,7 +93,7 @@ struct medium_container { for (int i = 0; i < nprops; i++) { Kokkos::Experimental::where(mask, values.data[i]) - .copy_from(&get_data(ispec, iz, ix, i), tag_type()); + .copy_from(&get_data(ispec, iz, ix, i), tag_type()); } } diff --git a/include/medium/kernels_container.hpp b/include/medium/kernels_container.hpp index 8e472f25..9e56546b 100644 --- a/include/medium/kernels_container.hpp +++ b/include/medium/kernels_container.hpp @@ -1,6 +1,6 @@ #pragma once -#include "impl/medium_container.hpp" +#include "impl/medium_data.hpp" namespace specfem { namespace medium { @@ -8,8 +8,8 @@ namespace medium { template struct impl_kernels_container - : public impl::medium_container { - using base_type = impl::medium_container; + : public impl::medium_data { + using base_type = impl::medium_data; using base_type::base_type; impl_kernels_container( diff --git a/include/medium/properties_container.hpp b/include/medium/properties_container.hpp index d711159a..97802f89 100644 --- a/include/medium/properties_container.hpp +++ b/include/medium/properties_container.hpp @@ -1,6 +1,6 @@ #pragma once -#include "impl/medium_container.hpp" +#include "impl/medium_data.hpp" namespace specfem { namespace medium { @@ -8,8 +8,8 @@ namespace medium { template struct impl_properties_container - : public impl::medium_container { - using base_type = impl::medium_container; + : public impl::medium_data { + using base_type = impl::medium_data; using base_type::base_type; impl_properties_container( diff --git a/include/point/impl/values.hpp b/include/point/impl/point_data.hpp similarity index 83% rename from include/point/impl/values.hpp rename to include/point/impl/point_data.hpp index b592074f..43f58276 100644 --- a/include/point/impl/values.hpp +++ b/include/point/impl/point_data.hpp @@ -16,7 +16,7 @@ namespace specfem { namespace point { namespace impl { -template struct values { +template struct point_data { using simd = specfem::datatype::simd; ///< SIMD type using value_type = typename simd::datatype; constexpr static auto nprops = N; @@ -24,14 +24,14 @@ template struct values { value_type data[N]; KOKKOS_FUNCTION - values() = default; + point_data() = default; /** * @brief array constructor * */ KOKKOS_FUNCTION - values(const value_type *value) { + point_data(const value_type *value) { for (int i = 0; i < N; ++i) { data[i] = value[i]; } @@ -43,14 +43,14 @@ template struct values { */ template = 0> - KOKKOS_FUNCTION values(Args... args) : data{ args... } {} + KOKKOS_FUNCTION point_data(Args... args) : data{ args... } {} /** * @brief Equality operator * */ KOKKOS_FUNCTION - bool operator==(const values &rhs) const { + bool operator==(const point_data &rhs) const { for (int i = 0; i < N; ++i) { if (data[i] != rhs.data[i]) { return false; @@ -64,7 +64,7 @@ template struct values { * */ KOKKOS_FUNCTION - bool operator!=(const values &rhs) const { + bool operator!=(const point_data &rhs) const { return !(*this == rhs); } }; diff --git a/include/point/kernels.hpp b/include/point/kernels.hpp index 6877dbd6..26ee4db4 100644 --- a/include/point/kernels.hpp +++ b/include/point/kernels.hpp @@ -1,6 +1,6 @@ #pragma once -#include "impl/values.hpp" +#include "impl/point_data.hpp" namespace specfem { namespace point { @@ -29,14 +29,14 @@ template struct kernels - : public impl::values<6, UseSIMD> { + : public impl::point_data<6, UseSIMD> { /** * @name Typedefs * */ ///@{ - using base_type = impl::values<6, UseSIMD>; + using base_type = impl::point_data<6, UseSIMD>; using value_type = typename base_type::value_type; constexpr static auto dimension = specfem::dimension::type::dim2; @@ -75,14 +75,14 @@ template struct kernels - : public impl::values<7, UseSIMD> { + : public impl::point_data<7, UseSIMD> { /** * @name Typedefs * */ ///@{ - using base_type = impl::values<7, UseSIMD>; + using base_type = impl::point_data<7, UseSIMD>; using value_type = typename base_type::value_type; constexpr static auto dimension = specfem::dimension::type::dim2; @@ -122,14 +122,14 @@ template struct kernels - : public impl::values<4, UseSIMD> { + : public impl::point_data<4, UseSIMD> { /** * @name Typedefs * */ ///@{ - using base_type = impl::values<4, UseSIMD>; + using base_type = impl::point_data<4, UseSIMD>; using value_type = typename base_type::value_type; constexpr static auto dimension = specfem::dimension::type::dim2; diff --git a/include/point/properties.hpp b/include/point/properties.hpp index f88f58cd..b0fa45ce 100644 --- a/include/point/properties.hpp +++ b/include/point/properties.hpp @@ -1,6 +1,6 @@ #pragma once -#include "impl/values.hpp" +#include "impl/point_data.hpp" namespace specfem { namespace point { @@ -27,14 +27,14 @@ template struct properties - : public impl::values<3, UseSIMD> { + : public impl::point_data<3, UseSIMD> { /** * @name Typedefs * */ ///@{ - using base_type = impl::values<3, UseSIMD>; + using base_type = impl::point_data<3, UseSIMD>; using value_type = typename base_type::value_type; constexpr static auto dimension = specfem::dimension::type::dim2; @@ -71,14 +71,14 @@ template struct properties - : public impl::values<10, UseSIMD> { + : public impl::point_data<10, UseSIMD> { /** * @name Typedefs * */ ///@{ - using base_type = impl::values<10, UseSIMD>; + using base_type = impl::point_data<10, UseSIMD>; using value_type = typename base_type::value_type; constexpr static auto dimension = specfem::dimension::type::dim2; @@ -127,13 +127,13 @@ template struct properties - : public impl::values<2, UseSIMD> { + : public impl::point_data<2, UseSIMD> { /** * @name Typedefs * */ ///@{ - using base_type = impl::values<2, UseSIMD>; + using base_type = impl::point_data<2, UseSIMD>; using value_type = typename base_type::value_type; constexpr static auto dimension = specfem::dimension::type::dim2; From 2dc8e472fa7e6b4a05efbd40510948bb37b06724 Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Tue, 4 Mar 2025 18:09:11 -0500 Subject: [PATCH 20/21] Merge kernel and property writer. --- include/IO/impl/medium_writer.hpp | 13 ++++ include/IO/impl/medium_writer.tpp | 110 +++++++++++++++++++++++++++ include/IO/kernel/writer.tpp | 119 +++--------------------------- include/IO/property/writer.tpp | 117 +++-------------------------- src/IO/kernel/writer.cpp | 7 +- src/IO/property/writer.cpp | 1 + 6 files changed, 150 insertions(+), 217 deletions(-) create mode 100644 include/IO/impl/medium_writer.hpp create mode 100644 include/IO/impl/medium_writer.tpp diff --git a/include/IO/impl/medium_writer.hpp b/include/IO/impl/medium_writer.hpp new file mode 100644 index 00000000..b362b5bd --- /dev/null +++ b/include/IO/impl/medium_writer.hpp @@ -0,0 +1,13 @@ +#pragma once + +namespace specfem { +namespace IO { +namespace impl { +template +void medium_writer(const std::string &output_folder, + const std::string &output_namespace, + const specfem::compute::assembly &assembly, + ContainerType &container); +} // namespace impl +} // namespace IO +} // namespace specfem diff --git a/include/IO/impl/medium_writer.tpp b/include/IO/impl/medium_writer.tpp new file mode 100644 index 00000000..f7248d94 --- /dev/null +++ b/include/IO/impl/medium_writer.tpp @@ -0,0 +1,110 @@ +#pragma once + +#include "IO/impl/medium_writer.hpp" +#include "compute/assembly/assembly.hpp" +#include "enumerations/dimension.hpp" +#include "enumerations/medium.hpp" +#include "kokkos_abstractions.h" +#include + +template +void specfem::IO::impl::medium_writer(const std::string &output_folder, const std::string &output_namespace, const specfem::compute::assembly &assembly, ContainerType &container) { + const auto &mesh = assembly.mesh; + auto &element_types = assembly.element_types; + + using DomainView = + Kokkos::View; + + container.copy_to_host(); + + typename OutputLibrary::File file(output_folder + "/" + output_namespace); + + const int nspec = mesh.points.nspec; + const int ngllz = mesh.points.ngllz; + const int ngllx = mesh.points.ngllx; + + int n_elastic_isotropic; + int n_elastic_anisotropic; + int n_acoustic; + + { + typename OutputLibrary::Group elastic = file.createGroup("/ElasticIsotropic"); + + const auto element_indices = element_types.get_elements_on_host( + specfem::element::medium_tag::elastic, + specfem::element::property_tag::isotropic); + n_elastic_isotropic = element_indices.size(); + + DomainView x("xcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, ngllx); + DomainView z("zcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, ngllx); + + for (int i = 0; i < n_elastic_isotropic; i++) { + const int ispec = element_indices(i); + for (int iz = 0; iz < ngllz; iz++) { + for (int ix = 0; ix < ngllx; ix++) { + x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); + z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); + } + } + } + + elastic.createDataset("X", x).write(); + elastic.createDataset("Z", z).write(); + elastic.createDataset("data", container.elastic_isotropic.h_data).write(); + } + + { + typename OutputLibrary::Group elastic = file.createGroup("/ElasticAnisotropic"); + + const auto element_indices = element_types.get_elements_on_host( + specfem::element::medium_tag::elastic, + specfem::element::property_tag::anisotropic); + n_elastic_anisotropic = element_indices.size(); + + DomainView x("xcoordinates_elastic_anisotropic", n_elastic_anisotropic, ngllz, ngllx); + DomainView z("zcoordinates_elastic_anisotropic", n_elastic_anisotropic, ngllz, ngllx); + + for (int i = 0; i < n_elastic_anisotropic; i++) { + const int ispec = element_indices(i); + for (int iz = 0; iz < ngllz; iz++) { + for (int ix = 0; ix < ngllx; ix++) { + x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); + z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); + } + } + } + + elastic.createDataset("X", x).write(); + elastic.createDataset("Z", z).write(); + elastic.createDataset("data", container.elastic_anisotropic.h_data).write(); + } + + { + typename OutputLibrary::Group acoustic = file.createGroup("/Acoustic"); + + const auto element_indices = element_types.get_elements_on_host(specfem::element::medium_tag::acoustic); + n_acoustic = element_indices.size(); + + DomainView x("xcoordinates_acoustic", n_acoustic, ngllz, ngllx); + DomainView z("zcoordinates_acoustic", n_acoustic, ngllz, ngllx); + + for (int i = 0; i < n_acoustic; i++) { + const int ispec = element_indices(i); + for (int iz = 0; iz < ngllz; iz++) { + for (int ix = 0; ix < ngllx; ix++) { + x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); + z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); + } + } + } + + acoustic.createDataset("X", x).write(); + acoustic.createDataset("Z", z).write(); + acoustic.createDataset("data", container.acoustic_isotropic.h_data).write(); + } + + assert(n_elastic_isotropic + n_elastic_anisotropic + n_acoustic == nspec); + + std::cout << output_namespace << " written to " << output_folder << "/" << output_namespace + << std::endl; +} diff --git a/include/IO/kernel/writer.tpp b/include/IO/kernel/writer.tpp index 2b326e13..22e58f89 100644 --- a/include/IO/kernel/writer.tpp +++ b/include/IO/kernel/writer.tpp @@ -1,116 +1,19 @@ #pragma once -#include "compute/assembly/assembly.hpp" -#include "enumerations/dimension.hpp" -#include "enumerations/medium.hpp" -#include "kokkos_abstractions.h" -#include "point/kernels.hpp" -#include "IO/kernel/writer.hpp" -#include +#include "IO/property/writer.hpp" +#include "IO/impl/medium_writer.hpp" + +namespace specfem { +namespace IO { template -specfem::IO::kernel_writer::kernel_writer(const std::string output_folder) +kernel_writer::kernel_writer(const std::string output_folder) : output_folder(output_folder) {} template -void specfem::IO::kernel_writer::write(specfem::compute::assembly &assembly) { - const auto &mesh = assembly.mesh; - auto &element_types = assembly.element_types; - auto &kernels = assembly.kernels; - - using DomainView = - Kokkos::View; - - kernels.copy_to_host(); - - typename OutputLibrary::File file(output_folder + "/Kernels"); - - const int nspec = mesh.points.nspec; - const int ngllz = mesh.points.ngllz; - const int ngllx = mesh.points.ngllx; - - int n_elastic_isotropic; - int n_elastic_anisotropic; - int n_acoustic; - - { - typename OutputLibrary::Group elastic = file.createGroup("/ElasticIsotropic"); - - const auto element_indices = element_types.get_elements_on_host( - specfem::element::medium_tag::elastic, - specfem::element::property_tag::isotropic); - n_elastic_isotropic = element_indices.size(); - - DomainView x("xcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, ngllx); - DomainView z("zcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, ngllx); - - for (int i = 0; i < n_elastic_isotropic; i++) { - const int ispec = element_indices(i); - for (int iz = 0; iz < ngllz; iz++) { - for (int ix = 0; ix < ngllx; ix++) { - x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); - z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); - } - } - } - - elastic.createDataset("X", x).write(); - elastic.createDataset("Z", z).write(); - elastic.createDataset("data", kernels.elastic_isotropic.h_data).write(); - } - - { - typename OutputLibrary::Group elastic = file.createGroup("/ElasticAnisotropic"); - - const auto element_indices = element_types.get_elements_on_host( - specfem::element::medium_tag::elastic, - specfem::element::property_tag::anisotropic); - n_elastic_anisotropic = element_indices.size(); - - DomainView x("xcoordinates_elastic_anisotropic", n_elastic_anisotropic, ngllz, ngllx); - DomainView z("zcoordinates_elastic_anisotropic", n_elastic_anisotropic, ngllz, ngllx); - - for (int i = 0; i < n_elastic_anisotropic; i++) { - const int ispec = element_indices(i); - for (int iz = 0; iz < ngllz; iz++) { - for (int ix = 0; ix < ngllx; ix++) { - x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); - z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); - } - } - } - - elastic.createDataset("X", x).write(); - elastic.createDataset("Z", z).write(); - elastic.createDataset("data", kernels.elastic_anisotropic.h_data).write(); - } - - { - typename OutputLibrary::Group acoustic = file.createGroup("/Acoustic"); - - const auto element_indices = element_types.get_elements_on_host(specfem::element::medium_tag::acoustic); - n_acoustic = element_indices.size(); - - DomainView x("xcoordinates_acoustic", n_acoustic, ngllz, ngllx); - DomainView z("zcoordinates_acoustic", n_acoustic, ngllz, ngllx); - - for (int i = 0; i < n_acoustic; i++) { - const int ispec = element_indices(i); - for (int iz = 0; iz < ngllz; iz++) { - for (int ix = 0; ix < ngllx; ix++) { - x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); - z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); - } - } - } - - acoustic.createDataset("X", x).write(); - acoustic.createDataset("Z", z).write(); - acoustic.createDataset("data", kernels.acoustic_isotropic.h_data).write(); - } - - assert(n_elastic_isotropic + n_elastic_anisotropic + n_acoustic == nspec); - - std::cout << "Kernels written to " << output_folder << "/Kernels" - << std::endl; +void kernel_writer::write(specfem::compute::assembly &assembly) { + impl::medium_writer(output_folder, "Kernels", assembly, assembly.kernels); } + +} // namespace IO +} // namespace specfem diff --git a/include/IO/property/writer.tpp b/include/IO/property/writer.tpp index 5706c6da..61b192f0 100644 --- a/include/IO/property/writer.tpp +++ b/include/IO/property/writer.tpp @@ -1,116 +1,19 @@ #pragma once -#include "compute/assembly/assembly.hpp" -#include "enumerations/dimension.hpp" -#include "enumerations/medium.hpp" -#include "kokkos_abstractions.h" -#include "point/properties.hpp" #include "IO/property/writer.hpp" -#include +#include "IO/impl/medium_writer.hpp" + +namespace specfem { +namespace IO { template -specfem::IO::property_writer::property_writer(const std::string output_folder) +property_writer::property_writer(const std::string output_folder) : output_folder(output_folder) {} template -void specfem::IO::property_writer::write(specfem::compute::assembly &assembly) { - const auto &mesh = assembly.mesh; - auto &element_types = assembly.element_types; - auto &properties = assembly.properties; - - using DomainView = - Kokkos::View; - - properties.copy_to_host(); - - typename OutputLibrary::File file(output_folder + "/Properties"); - - const int nspec = mesh.points.nspec; - const int ngllz = mesh.points.ngllz; - const int ngllx = mesh.points.ngllx; - - int n_elastic_isotropic; - int n_elastic_anisotropic; - int n_acoustic; - - { - typename OutputLibrary::Group elastic = file.createGroup("/ElasticIsotropic"); - - const auto element_indices = element_types.get_elements_on_host( - specfem::element::medium_tag::elastic, - specfem::element::property_tag::isotropic); - n_elastic_isotropic = element_indices.size(); - - DomainView x("xcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, ngllx); - DomainView z("zcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, ngllx); - - for (int i = 0; i < n_elastic_isotropic; i++) { - const int ispec = element_indices(i); - for (int iz = 0; iz < ngllz; iz++) { - for (int ix = 0; ix < ngllx; ix++) { - x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); - z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); - } - } - } - - elastic.createDataset("X", x).write(); - elastic.createDataset("Z", z).write(); - elastic.createDataset("data", properties.elastic_isotropic.h_data).write(); - } - - { - typename OutputLibrary::Group elastic = file.createGroup("/ElasticAnisotropic"); - - const auto element_indices = element_types.get_elements_on_host( - specfem::element::medium_tag::elastic, - specfem::element::property_tag::anisotropic); - n_elastic_anisotropic = element_indices.size(); - - DomainView x("xcoordinates_elastic_anisotropic", n_elastic_anisotropic, ngllz, ngllx); - DomainView z("zcoordinates_elastic_anisotropic", n_elastic_anisotropic, ngllz, ngllx); - - for (int i = 0; i < n_elastic_anisotropic; i++) { - const int ispec = element_indices(i); - for (int iz = 0; iz < ngllz; iz++) { - for (int ix = 0; ix < ngllx; ix++) { - x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); - z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); - } - } - } - - elastic.createDataset("X", x).write(); - elastic.createDataset("Z", z).write(); - elastic.createDataset("data", properties.elastic_anisotropic.h_data).write(); - } - - { - typename OutputLibrary::Group acoustic = file.createGroup("/Acoustic"); - - const auto element_indices = element_types.get_elements_on_host(specfem::element::medium_tag::acoustic); - n_acoustic = element_indices.size(); - - DomainView x("xcoordinates_acoustic", n_acoustic, ngllz, ngllx); - DomainView z("zcoordinates_acoustic", n_acoustic, ngllz, ngllx); - - for (int i = 0; i < n_acoustic; i++) { - const int ispec = element_indices(i); - for (int iz = 0; iz < ngllz; iz++) { - for (int ix = 0; ix < ngllx; ix++) { - x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); - z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); - } - } - } - - acoustic.createDataset("X", x).write(); - acoustic.createDataset("Z", z).write(); - acoustic.createDataset("data", properties.acoustic_isotropic.h_data).write(); - } - - assert(n_elastic_isotropic + n_elastic_anisotropic + n_acoustic == nspec); - - std::cout << "Properties written to " << output_folder << "/Properties" - << std::endl; +void property_writer::write(specfem::compute::assembly &assembly) { + impl::medium_writer(output_folder, "Properties", assembly, assembly.properties); } + +} // namespace IO +} // namespace specfem diff --git a/src/IO/kernel/writer.cpp b/src/IO/kernel/writer.cpp index dd3181b1..e6cc64e0 100644 --- a/src/IO/kernel/writer.cpp +++ b/src/IO/kernel/writer.cpp @@ -1,10 +1,13 @@ #include "IO/kernel/writer.hpp" #include "IO/ASCII/ASCII.hpp" #include "IO/HDF5/HDF5.hpp" +#include "IO/impl/medium_writer.tpp" #include "IO/kernel/writer.tpp" // Explicit instantiation -template class specfem::IO::kernel_writer >; +template class specfem::IO::kernel_writer< + specfem::IO::HDF5 >; -template class specfem::IO::kernel_writer >; +template class specfem::IO::kernel_writer< + specfem::IO::ASCII >; diff --git a/src/IO/property/writer.cpp b/src/IO/property/writer.cpp index 0bd22db4..b34a5452 100644 --- a/src/IO/property/writer.cpp +++ b/src/IO/property/writer.cpp @@ -1,6 +1,7 @@ #include "IO/property/writer.hpp" #include "IO/ASCII/ASCII.hpp" #include "IO/HDF5/HDF5.hpp" +#include "IO/impl/medium_writer.tpp" #include "IO/property/writer.tpp" // Explicit instantiation From ae21b6f32a7107c042c0d82724337993f0548572 Mon Sep 17 00:00:00 2001 From: Congyue Cui Date: Wed, 5 Mar 2025 13:13:43 -0500 Subject: [PATCH 21/21] Rename medium_writer to write_contaner --- include/IO/impl/medium_writer.hpp | 9 ++++---- include/IO/impl/medium_writer.tpp | 34 +++++++++++++++++++------------ include/IO/kernel/writer.tpp | 2 +- include/IO/property/writer.tpp | 2 +- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/include/IO/impl/medium_writer.hpp b/include/IO/impl/medium_writer.hpp index b362b5bd..d8766229 100644 --- a/include/IO/impl/medium_writer.hpp +++ b/include/IO/impl/medium_writer.hpp @@ -4,10 +4,11 @@ namespace specfem { namespace IO { namespace impl { template -void medium_writer(const std::string &output_folder, - const std::string &output_namespace, - const specfem::compute::assembly &assembly, - ContainerType &container); +void write_container(const std::string &output_folder, + const std::string &output_namespace, + const specfem::compute::mesh &mesh, + const specfem::compute::element_types &element_types, + ContainerType &container); } // namespace impl } // namespace IO } // namespace specfem diff --git a/include/IO/impl/medium_writer.tpp b/include/IO/impl/medium_writer.tpp index f7248d94..114ea64c 100644 --- a/include/IO/impl/medium_writer.tpp +++ b/include/IO/impl/medium_writer.tpp @@ -8,10 +8,11 @@ #include template -void specfem::IO::impl::medium_writer(const std::string &output_folder, const std::string &output_namespace, const specfem::compute::assembly &assembly, ContainerType &container) { - const auto &mesh = assembly.mesh; - auto &element_types = assembly.element_types; - +void specfem::IO::impl::write_container( + const std::string &output_folder, const std::string &output_namespace, + const specfem::compute::mesh &mesh, + const specfem::compute::element_types &element_types, + ContainerType &container) { using DomainView = Kokkos::View; @@ -28,15 +29,18 @@ void specfem::IO::impl::medium_writer(const std::string &output_folder, const st int n_acoustic; { - typename OutputLibrary::Group elastic = file.createGroup("/ElasticIsotropic"); + typename OutputLibrary::Group elastic = + file.createGroup("/ElasticIsotropic"); const auto element_indices = element_types.get_elements_on_host( specfem::element::medium_tag::elastic, specfem::element::property_tag::isotropic); n_elastic_isotropic = element_indices.size(); - DomainView x("xcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, ngllx); - DomainView z("zcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, ngllx); + DomainView x("xcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, + ngllx); + DomainView z("zcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, + ngllx); for (int i = 0; i < n_elastic_isotropic; i++) { const int ispec = element_indices(i); @@ -54,15 +58,18 @@ void specfem::IO::impl::medium_writer(const std::string &output_folder, const st } { - typename OutputLibrary::Group elastic = file.createGroup("/ElasticAnisotropic"); + typename OutputLibrary::Group elastic = + file.createGroup("/ElasticAnisotropic"); const auto element_indices = element_types.get_elements_on_host( specfem::element::medium_tag::elastic, specfem::element::property_tag::anisotropic); n_elastic_anisotropic = element_indices.size(); - DomainView x("xcoordinates_elastic_anisotropic", n_elastic_anisotropic, ngllz, ngllx); - DomainView z("zcoordinates_elastic_anisotropic", n_elastic_anisotropic, ngllz, ngllx); + DomainView x("xcoordinates_elastic_anisotropic", n_elastic_anisotropic, + ngllz, ngllx); + DomainView z("zcoordinates_elastic_anisotropic", n_elastic_anisotropic, + ngllz, ngllx); for (int i = 0; i < n_elastic_anisotropic; i++) { const int ispec = element_indices(i); @@ -82,7 +89,8 @@ void specfem::IO::impl::medium_writer(const std::string &output_folder, const st { typename OutputLibrary::Group acoustic = file.createGroup("/Acoustic"); - const auto element_indices = element_types.get_elements_on_host(specfem::element::medium_tag::acoustic); + const auto element_indices = element_types.get_elements_on_host( + specfem::element::medium_tag::acoustic); n_acoustic = element_indices.size(); DomainView x("xcoordinates_acoustic", n_acoustic, ngllz, ngllx); @@ -105,6 +113,6 @@ void specfem::IO::impl::medium_writer(const std::string &output_folder, const st assert(n_elastic_isotropic + n_elastic_anisotropic + n_acoustic == nspec); - std::cout << output_namespace << " written to " << output_folder << "/" << output_namespace - << std::endl; + std::cout << output_namespace << " written to " << output_folder << "/" + << output_namespace << std::endl; } diff --git a/include/IO/kernel/writer.tpp b/include/IO/kernel/writer.tpp index 22e58f89..6d0a32fd 100644 --- a/include/IO/kernel/writer.tpp +++ b/include/IO/kernel/writer.tpp @@ -12,7 +12,7 @@ kernel_writer::kernel_writer(const std::string output_folder) template void kernel_writer::write(specfem::compute::assembly &assembly) { - impl::medium_writer(output_folder, "Kernels", assembly, assembly.kernels); + impl::write_container(output_folder, "Kernels", assembly.mesh, assembly.element_types, assembly.kernels); } } // namespace IO diff --git a/include/IO/property/writer.tpp b/include/IO/property/writer.tpp index 61b192f0..a894c320 100644 --- a/include/IO/property/writer.tpp +++ b/include/IO/property/writer.tpp @@ -12,7 +12,7 @@ property_writer::property_writer(const std::string output_folder) template void property_writer::write(specfem::compute::assembly &assembly) { - impl::medium_writer(output_folder, "Properties", assembly, assembly.properties); + impl::write_container(output_folder, "Properties", assembly.mesh, assembly.element_types, assembly.properties); } } // namespace IO