diff --git a/examples/Tromp_2005/plot.py b/examples/Tromp_2005/plot.py index e644c9b8..dcccd145 100644 --- a/examples/Tromp_2005/plot.py +++ b/examples/Tromp_2005/plot.py @@ -7,12 +7,14 @@ def load_data(directory): X = np.loadtxt(directory + "/ElasticIsotropic/X.txt") Z = np.loadtxt(directory + "/ElasticIsotropic/Z.txt") - rho = np.loadtxt(directory + "/ElasticIsotropic/rho.txt") - kappa = np.loadtxt(directory + "/ElasticIsotropic/kappa.txt") - mu = np.loadtxt(directory + "/ElasticIsotropic/mu.txt") - rhop = np.loadtxt(directory + "/ElasticIsotropic/rhop.txt") - alpha = np.loadtxt(directory + "/ElasticIsotropic/alpha.txt") - beta = np.loadtxt(directory + "/ElasticIsotropic/beta.txt") + data = np.loadtxt(directory + "/ElasticIsotropic/data.txt").reshape(6, X.shape[0]) + + rho = data[0, :] + mu = data[1, :] + kappa = data[2, :] + rhop = data[3, :] + alpha = data[4, :] + beta = data[5, :] return X, Z, rho, kappa, mu, rhop, alpha, beta @@ -97,3 +99,7 @@ def plot_kernels(input_directory, output): plt.savefig(output, dpi=300) return + + +if __name__ == "__main__": + plot_kernels("OUTPUT_FILES/Kernels", "Kernels_out.png") diff --git a/include/IO/impl/medium_writer.hpp b/include/IO/impl/medium_writer.hpp new file mode 100644 index 00000000..d8766229 --- /dev/null +++ b/include/IO/impl/medium_writer.hpp @@ -0,0 +1,14 @@ +#pragma once + +namespace specfem { +namespace IO { +namespace impl { +template +void write_container(const std::string &output_folder, + const std::string &output_namespace, + const specfem::compute::mesh &mesh, + const specfem::compute::element_types &element_types, + ContainerType &container); +} // namespace impl +} // namespace IO +} // namespace specfem diff --git a/include/IO/impl/medium_writer.tpp b/include/IO/impl/medium_writer.tpp new file mode 100644 index 00000000..114ea64c --- /dev/null +++ b/include/IO/impl/medium_writer.tpp @@ -0,0 +1,118 @@ +#pragma once + +#include "IO/impl/medium_writer.hpp" +#include "compute/assembly/assembly.hpp" +#include "enumerations/dimension.hpp" +#include "enumerations/medium.hpp" +#include "kokkos_abstractions.h" +#include + +template +void specfem::IO::impl::write_container( + const std::string &output_folder, const std::string &output_namespace, + const specfem::compute::mesh &mesh, + const specfem::compute::element_types &element_types, + ContainerType &container) { + using DomainView = + Kokkos::View; + + container.copy_to_host(); + + typename OutputLibrary::File file(output_folder + "/" + output_namespace); + + const int nspec = mesh.points.nspec; + const int ngllz = mesh.points.ngllz; + const int ngllx = mesh.points.ngllx; + + int n_elastic_isotropic; + int n_elastic_anisotropic; + int n_acoustic; + + { + typename OutputLibrary::Group elastic = + file.createGroup("/ElasticIsotropic"); + + const auto element_indices = element_types.get_elements_on_host( + specfem::element::medium_tag::elastic, + specfem::element::property_tag::isotropic); + n_elastic_isotropic = element_indices.size(); + + DomainView x("xcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, + ngllx); + DomainView z("zcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, + ngllx); + + for (int i = 0; i < n_elastic_isotropic; i++) { + const int ispec = element_indices(i); + for (int iz = 0; iz < ngllz; iz++) { + for (int ix = 0; ix < ngllx; ix++) { + x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); + z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); + } + } + } + + elastic.createDataset("X", x).write(); + elastic.createDataset("Z", z).write(); + elastic.createDataset("data", container.elastic_isotropic.h_data).write(); + } + + { + typename OutputLibrary::Group elastic = + file.createGroup("/ElasticAnisotropic"); + + const auto element_indices = element_types.get_elements_on_host( + specfem::element::medium_tag::elastic, + specfem::element::property_tag::anisotropic); + n_elastic_anisotropic = element_indices.size(); + + DomainView x("xcoordinates_elastic_anisotropic", n_elastic_anisotropic, + ngllz, ngllx); + DomainView z("zcoordinates_elastic_anisotropic", n_elastic_anisotropic, + ngllz, ngllx); + + for (int i = 0; i < n_elastic_anisotropic; i++) { + const int ispec = element_indices(i); + for (int iz = 0; iz < ngllz; iz++) { + for (int ix = 0; ix < ngllx; ix++) { + x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); + z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); + } + } + } + + elastic.createDataset("X", x).write(); + elastic.createDataset("Z", z).write(); + elastic.createDataset("data", container.elastic_anisotropic.h_data).write(); + } + + { + typename OutputLibrary::Group acoustic = file.createGroup("/Acoustic"); + + const auto element_indices = element_types.get_elements_on_host( + specfem::element::medium_tag::acoustic); + n_acoustic = element_indices.size(); + + DomainView x("xcoordinates_acoustic", n_acoustic, ngllz, ngllx); + DomainView z("zcoordinates_acoustic", n_acoustic, ngllz, ngllx); + + for (int i = 0; i < n_acoustic; i++) { + const int ispec = element_indices(i); + for (int iz = 0; iz < ngllz; iz++) { + for (int ix = 0; ix < ngllx; ix++) { + x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); + z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); + } + } + } + + acoustic.createDataset("X", x).write(); + acoustic.createDataset("Z", z).write(); + acoustic.createDataset("data", container.acoustic_isotropic.h_data).write(); + } + + assert(n_elastic_isotropic + n_elastic_anisotropic + n_acoustic == nspec); + + std::cout << output_namespace << " written to " << output_folder << "/" + << output_namespace << std::endl; +} diff --git a/include/IO/kernel/writer.tpp b/include/IO/kernel/writer.tpp index ec458b17..6d0a32fd 100644 --- a/include/IO/kernel/writer.tpp +++ b/include/IO/kernel/writer.tpp @@ -1,197 +1,19 @@ #pragma once -#include "compute/assembly/assembly.hpp" -#include "enumerations/dimension.hpp" -#include "enumerations/medium.hpp" -#include "kokkos_abstractions.h" -#include "point/kernels.hpp" -#include "IO/kernel/writer.hpp" -#include +#include "IO/property/writer.hpp" +#include "IO/impl/medium_writer.hpp" + +namespace specfem { +namespace IO { template -specfem::IO::kernel_writer::kernel_writer(const std::string output_folder) +kernel_writer::kernel_writer(const std::string output_folder) : output_folder(output_folder) {} template -void specfem::IO::kernel_writer::write(specfem::compute::assembly &assembly) { - const auto &mesh = assembly.mesh; - auto &element_types = assembly.element_types; - auto &kernels = assembly.kernels; - - using DomainView = - Kokkos::View; - - kernels.copy_to_host(); - - typename OutputLibrary::File file(output_folder + "/Kernels"); - - const int nspec = mesh.points.nspec; - const int ngllz = mesh.points.ngllz; - const int ngllx = mesh.points.ngllx; - - int n_elastic_isotropic; - int n_elastic_anisotropic; - int n_acoustic; - - { - typename OutputLibrary::Group elastic = file.createGroup("/ElasticIsotropic"); - - const auto element_indices = element_types.get_elements_on_host( - specfem::element::medium_tag::elastic, - specfem::element::property_tag::isotropic); - n_elastic_isotropic = element_indices.size(); - - DomainView x("xcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, ngllx); - DomainView z("zcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, ngllx); - - DomainView rho("rho", n_elastic_isotropic, ngllz, ngllx); - DomainView mu("mu", n_elastic_isotropic, ngllz, ngllx); - DomainView kappa("kappa", n_elastic_isotropic, ngllz, ngllx); - DomainView rhop("rhop", n_elastic_isotropic, ngllz, ngllx); - DomainView alpha("alpha", n_elastic_isotropic, ngllz, ngllx); - DomainView beta("beta", n_elastic_isotropic, ngllz, ngllx); - - for (int i = 0; i < n_elastic_isotropic; i++) { - const int ispec = element_indices(i); - for (int iz = 0; iz < ngllz; iz++) { - for (int ix = 0; ix < ngllx; ix++) { - x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); - z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); - const specfem::point::index index( - ispec, iz, ix); - specfem::point::kernels - point_kernels; - - specfem::compute::load_on_host(index, kernels, point_kernels); - - rho(i, iz, ix) = point_kernels.rho; - mu(i, iz, ix) = point_kernels.mu; - kappa(i, iz, ix) = point_kernels.kappa; - rhop(i, iz, ix) = point_kernels.rhop; - alpha(i, iz, ix) = point_kernels.alpha; - beta(i, iz, ix) = point_kernels.beta; - } - } - } - - elastic.createDataset("X", x).write(); - elastic.createDataset("Z", z).write(); - elastic.createDataset("rho", rho).write(); - elastic.createDataset("mu", mu).write(); - elastic.createDataset("kappa", kappa).write(); - elastic.createDataset("rhop", rhop).write(); - elastic.createDataset("alpha", alpha).write(); - elastic.createDataset("beta", beta).write(); - } - - { - typename OutputLibrary::Group elastic = file.createGroup("/ElasticAnisotropic"); - - const auto element_indices = element_types.get_elements_on_host( - specfem::element::medium_tag::elastic, - specfem::element::property_tag::anisotropic); - n_elastic_anisotropic = element_indices.size(); - - DomainView x("xcoordinates_elastic_anisotropic", n_elastic_anisotropic, ngllz, ngllx); - DomainView z("zcoordinates_elastic_anisotropic", n_elastic_anisotropic, ngllz, ngllx); - - DomainView rho("rho", n_elastic_anisotropic, ngllz, ngllx); - DomainView c11("c11", n_elastic_anisotropic, ngllz, ngllx); - DomainView c13("c13", n_elastic_anisotropic, ngllz, ngllx); - DomainView c15("c15", n_elastic_anisotropic, ngllz, ngllx); - DomainView c33("c33", n_elastic_anisotropic, ngllz, ngllx); - DomainView c35("c35", n_elastic_anisotropic, ngllz, ngllx); - DomainView c55("c55", n_elastic_anisotropic, ngllz, ngllx); - - for (int i = 0; i < n_elastic_anisotropic; i++) { - const int ispec = element_indices(i); - for (int iz = 0; iz < ngllz; iz++) { - for (int ix = 0; ix < ngllx; ix++) { - x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); - z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); - const specfem::point::index index( - ispec, iz, ix); - specfem::point::kernels - point_kernels; - - specfem::compute::load_on_host(index, kernels, point_kernels); - - rho(i, iz, ix) = point_kernels.rho; - c11(i, iz, ix) = point_kernels.c11; - c13(i, iz, ix) = point_kernels.c13; - c15(i, iz, ix) = point_kernels.c15; - c33(i, iz, ix) = point_kernels.c33; - c35(i, iz, ix) = point_kernels.c35; - c55(i, iz, ix) = point_kernels.c55; - } - } - } - - elastic.createDataset("X", x).write(); - elastic.createDataset("Z", z).write(); - elastic.createDataset("rho", rho).write(); - elastic.createDataset("c11", c11).write(); - elastic.createDataset("c13", c13).write(); - elastic.createDataset("c15", c15).write(); - elastic.createDataset("c33", c33).write(); - elastic.createDataset("c35", c35).write(); - elastic.createDataset("c55", c55).write(); - } - - { - typename OutputLibrary::Group acoustic = file.createGroup("/Acoustic"); - - const auto element_indices = element_types.get_elements_on_host(specfem::element::medium_tag::acoustic); - n_acoustic = element_indices.size(); - - DomainView x("xcoordinates_acoustic", n_acoustic, ngllz, ngllx); - DomainView z("zcoordinates_acoustic", n_acoustic, ngllz, ngllx); - - DomainView rho("rho", n_acoustic, ngllz, ngllx); - DomainView kappa("kappa", n_acoustic, ngllz, ngllx); - DomainView rho_prime("rho_prime", n_acoustic, ngllz, ngllx); - DomainView alpha("alpha", n_acoustic, ngllz, ngllx); - - for (int i = 0; i < n_acoustic; i++) { - const int ispec = element_indices(i); - for (int iz = 0; iz < ngllz; iz++) { - for (int ix = 0; ix < ngllx; ix++) { - x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); - z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); - const specfem::point::index index( - ispec, iz, ix); - specfem::point::kernels - point_kernels; - - specfem::compute::load_on_host(index, kernels, point_kernels); - - rho(i, iz, ix) = point_kernels.rho; - kappa(i, iz, ix) = point_kernels.kappa; - rho_prime(i, iz, ix) = point_kernels.rhop; - alpha(i, iz, ix) = point_kernels.alpha; - } - } - } - - acoustic.createDataset("X", x).write(); - acoustic.createDataset("Z", z).write(); - acoustic.createDataset("rho", rho).write(); - acoustic.createDataset("kappa", kappa).write(); - acoustic.createDataset("rho_prime", rho_prime).write(); - acoustic.createDataset("alpha", alpha).write(); - } - - assert(n_elastic_isotropic + n_elastic_anisotropic + n_acoustic == nspec); - - std::cout << "Kernels written to " << output_folder << "/Kernels" - << std::endl; +void kernel_writer::write(specfem::compute::assembly &assembly) { + impl::write_container(output_folder, "Kernels", assembly.mesh, assembly.element_types, assembly.kernels); } + +} // namespace IO +} // namespace specfem diff --git a/include/IO/property/reader.tpp b/include/IO/property/reader.tpp index 60253ebb..5adf7216 100644 --- a/include/IO/property/reader.tpp +++ b/include/IO/property/reader.tpp @@ -23,31 +23,19 @@ void specfem::IO::property_reader::read(specfem::compute::assembly { typename InputLibrary::Group elastic = file.openGroup("/ElasticIsotropic"); - elastic.openDataset("rho", properties.elastic_isotropic.h_rho).read(); - elastic.openDataset("mu", properties.elastic_isotropic.h_mu).read(); - elastic.openDataset("lambdaplus2mu", properties.elastic_isotropic.h_lambdaplus2mu).read(); + elastic.openDataset("data", properties.elastic_isotropic.h_data).read(); } { typename InputLibrary::Group elastic = file.openGroup("/ElasticAnisotropic"); - elastic.openDataset("rho", properties.elastic_anisotropic.h_rho).read(); - elastic.openDataset("c11", properties.elastic_anisotropic.h_c11).read(); - elastic.openDataset("c13", properties.elastic_anisotropic.h_c13).read(); - elastic.openDataset("c15", properties.elastic_anisotropic.h_c15).read(); - elastic.openDataset("c33", properties.elastic_anisotropic.h_c33).read(); - elastic.openDataset("c35", properties.elastic_anisotropic.h_c35).read(); - elastic.openDataset("c55", properties.elastic_anisotropic.h_c55).read(); - elastic.openDataset("c12", properties.elastic_anisotropic.h_c12).read(); - elastic.openDataset("c23", properties.elastic_anisotropic.h_c23).read(); - elastic.openDataset("c25", properties.elastic_anisotropic.h_c25).read(); + elastic.openDataset("data", properties.elastic_anisotropic.h_data).read(); } { typename InputLibrary::Group acoustic = file.openGroup("/Acoustic"); - acoustic.openDataset("rho_inverse", properties.acoustic_isotropic.h_rho_inverse).read(); - acoustic.openDataset("kappa", properties.acoustic_isotropic.h_kappa).read(); + acoustic.openDataset("data", properties.acoustic_isotropic.h_data).read(); } std::cout << "Properties read from " << input_folder << "/Properties" diff --git a/include/IO/property/writer.tpp b/include/IO/property/writer.tpp index 563a2242..a894c320 100644 --- a/include/IO/property/writer.tpp +++ b/include/IO/property/writer.tpp @@ -1,131 +1,19 @@ #pragma once -#include "compute/assembly/assembly.hpp" -#include "enumerations/dimension.hpp" -#include "enumerations/medium.hpp" -#include "kokkos_abstractions.h" -#include "point/properties.hpp" #include "IO/property/writer.hpp" -#include +#include "IO/impl/medium_writer.hpp" + +namespace specfem { +namespace IO { template -specfem::IO::property_writer::property_writer(const std::string output_folder) +property_writer::property_writer(const std::string output_folder) : output_folder(output_folder) {} template -void specfem::IO::property_writer::write(specfem::compute::assembly &assembly) { - const auto &mesh = assembly.mesh; - auto &element_types = assembly.element_types; - auto &properties = assembly.properties; - - using DomainView = - Kokkos::View; - - properties.copy_to_host(); - - typename OutputLibrary::File file(output_folder + "/Properties"); - - const int nspec = mesh.points.nspec; - const int ngllz = mesh.points.ngllz; - const int ngllx = mesh.points.ngllx; - - int n_elastic_isotropic; - int n_elastic_anisotropic; - int n_acoustic; - - { - typename OutputLibrary::Group elastic = file.createGroup("/ElasticIsotropic"); - - const auto element_indices = element_types.get_elements_on_host( - specfem::element::medium_tag::elastic, - specfem::element::property_tag::isotropic); - n_elastic_isotropic = element_indices.size(); - - DomainView x("xcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, ngllx); - DomainView z("zcoordinates_elastic_isotropic", n_elastic_isotropic, ngllz, ngllx); - - for (int i = 0; i < n_elastic_isotropic; i++) { - const int ispec = element_indices(i); - for (int iz = 0; iz < ngllz; iz++) { - for (int ix = 0; ix < ngllx; ix++) { - x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); - z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); - } - } - } - - elastic.createDataset("X", x).write(); - elastic.createDataset("Z", z).write(); - - elastic.createDataset("rho", properties.elastic_isotropic.h_rho).write(); - elastic.createDataset("mu", properties.elastic_isotropic.h_mu).write(); - elastic.createDataset("lambdaplus2mu", properties.elastic_isotropic.h_lambdaplus2mu).write(); - } - - { - typename OutputLibrary::Group elastic = file.createGroup("/ElasticAnisotropic"); - - const auto element_indices = element_types.get_elements_on_host( - specfem::element::medium_tag::elastic, - specfem::element::property_tag::anisotropic); - n_elastic_anisotropic = element_indices.size(); - - DomainView x("xcoordinates_elastic_anisotropic", n_elastic_anisotropic, ngllz, ngllx); - DomainView z("zcoordinates_elastic_anisotropic", n_elastic_anisotropic, ngllz, ngllx); - - for (int i = 0; i < n_elastic_anisotropic; i++) { - const int ispec = element_indices(i); - for (int iz = 0; iz < ngllz; iz++) { - for (int ix = 0; ix < ngllx; ix++) { - x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); - z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); - } - } - } - - elastic.createDataset("X", x).write(); - elastic.createDataset("Z", z).write(); - - elastic.createDataset("rho", properties.elastic_anisotropic.h_rho).write(); - elastic.createDataset("c11", properties.elastic_anisotropic.h_c11).write(); - elastic.createDataset("c13", properties.elastic_anisotropic.h_c13).write(); - elastic.createDataset("c15", properties.elastic_anisotropic.h_c15).write(); - elastic.createDataset("c33", properties.elastic_anisotropic.h_c33).write(); - elastic.createDataset("c35", properties.elastic_anisotropic.h_c35).write(); - elastic.createDataset("c55", properties.elastic_anisotropic.h_c55).write(); - elastic.createDataset("c12", properties.elastic_anisotropic.h_c12).write(); - elastic.createDataset("c23", properties.elastic_anisotropic.h_c23).write(); - elastic.createDataset("c25", properties.elastic_anisotropic.h_c25).write(); - } - - { - typename OutputLibrary::Group acoustic = file.createGroup("/Acoustic"); - - const auto element_indices = element_types.get_elements_on_host(specfem::element::medium_tag::acoustic); - n_acoustic = element_indices.size(); - - DomainView x("xcoordinates_acoustic", n_acoustic, ngllz, ngllx); - DomainView z("zcoordinates_acoustic", n_acoustic, ngllz, ngllx); - - for (int i = 0; i < n_acoustic; i++) { - const int ispec = element_indices(i); - for (int iz = 0; iz < ngllz; iz++) { - for (int ix = 0; ix < ngllx; ix++) { - x(i, iz, ix) = mesh.points.h_coord(0, ispec, iz, ix); - z(i, iz, ix) = mesh.points.h_coord(1, ispec, iz, ix); - } - } - } - - acoustic.createDataset("X", x).write(); - acoustic.createDataset("Z", z).write(); - - acoustic.createDataset("rho_inverse", properties.acoustic_isotropic.h_rho_inverse).write(); - acoustic.createDataset("kappa", properties.acoustic_isotropic.h_kappa).write(); - } - - assert(n_elastic_isotropic + n_elastic_anisotropic + n_acoustic == nspec); - - std::cout << "Properties written to " << output_folder << "/Properties" - << std::endl; +void property_writer::write(specfem::compute::assembly &assembly) { + impl::write_container(output_folder, "Properties", assembly.mesh, assembly.element_types, assembly.properties); } + +} // namespace IO +} // namespace specfem diff --git a/include/boundary_conditions/stacey/stacey.tpp b/include/boundary_conditions/stacey/stacey.tpp index 7f3010b0..b84b4a2e 100644 --- a/include/boundary_conditions/stacey/stacey.tpp +++ b/include/boundary_conditions/stacey/stacey.tpp @@ -57,7 +57,7 @@ impl_enforce_traction(const acoustic_type &, const isotropic_type &, // Apply Stacey boundary condition traction(0) += static_cast(-1.0) * factor * - property.rho_vpinverse * field.velocity(0); + property.rho_vpinverse() * field.velocity(0); // Do nothing return; @@ -101,7 +101,7 @@ impl_enforce_traction(const acoustic_type &, const isotropic_type &, // Apply Stacey boundary condition Kokkos::Experimental::where(mask, traction(0)) = traction(0) + static_cast(-1.0) * factor * - property.rho_vpinverse * field.velocity(0); + property.rho_vpinverse() * field.velocity(0); return; } @@ -146,8 +146,8 @@ impl_enforce_traction(const elastic_type &, const isotropic_type &, for (int icomp = 0; icomp < 2; ++icomp) { factor[icomp] = ((vn * dn(icomp) / (jacobian1d * jacobian1d)) * - (property.rho_vp - property.rho_vs)) + - field.velocity(icomp) * property.rho_vs; + (property.rho_vp() - property.rho_vs())) + + field.velocity(icomp) * property.rho_vs(); } traction(0) += static_cast(-1.0) * factor[0] * jacobian1d * @@ -203,8 +203,8 @@ impl_enforce_traction(const elastic_type &, const isotropic_type &, for (int icomp = 0; icomp < 2; ++icomp) { factor[icomp] = ((vn * dn(icomp) / (jacobian1d * jacobian1d)) * - (property.rho_vp - property.rho_vs)) + - field.velocity(icomp) * property.rho_vs; + (property.rho_vp() - property.rho_vs())) + + field.velocity(icomp) * property.rho_vs(); } Kokkos::Experimental::where(mask, traction(0)) = @@ -259,8 +259,8 @@ impl_enforce_traction(const elastic_type &, const anisotropic_type &, for (int icomp = 0; icomp < 2; ++icomp) { factor[icomp] = ((vn * dn(icomp) / (jacobian1d * jacobian1d)) * - (property.rho_vp - property.rho_vs)) + - field.velocity(icomp) * property.rho_vs; + (property.rho_vp() - property.rho_vs())) + + field.velocity(icomp) * property.rho_vs(); } traction(0) += static_cast(-1.0) * factor[0] * jacobian1d * @@ -316,8 +316,8 @@ impl_enforce_traction(const elastic_type &, const anisotropic_type &, for (int icomp = 0; icomp < 2; ++icomp) { factor[icomp] = ((vn * dn(icomp) / (jacobian1d * jacobian1d)) * - (property.rho_vp - property.rho_vs)) + - field.velocity(icomp) * property.rho_vs; + (property.rho_vp() - property.rho_vs())) + + field.velocity(icomp) * property.rho_vs(); } Kokkos::Experimental::where(mask, traction(0)) = diff --git a/include/compute/compute_mesh.hpp b/include/compute/compute_mesh.hpp index d87e236c..1bf4d4f1 100644 --- a/include/compute/compute_mesh.hpp +++ b/include/compute/compute_mesh.hpp @@ -229,7 +229,7 @@ struct mesh { */ /** - * @brief Load quadrature data for a spectral element on the device + * @brief Load quadrature data for a spectral element on host or device * * @ingroup QuadratureDataAccess * @@ -239,11 +239,11 @@ struct mesh { * @param quadrature Quadrature data * @param element_quadrature Quadrature data for the element (output) */ -template -KOKKOS_FUNCTION void -load_on_device(const MemberType &team, - const specfem::compute::quadrature &quadrature, - ViewType &element_quadrature) { +template +KOKKOS_INLINE_FUNCTION void +impl_load(const MemberType &team, + const specfem::compute::quadrature &quadrature, + ViewType &element_quadrature) { constexpr bool store_hprime_gll = ViewType::store_hprime_gll; @@ -260,15 +260,51 @@ load_on_device(const MemberType &team, int ix, iz; sub2ind(xz, NGLL, iz, ix); if constexpr (store_hprime_gll) { - element_quadrature.hprime_gll(iz, ix) = quadrature.gll.hprime(iz, ix); + if constexpr (on_device) { + element_quadrature.hprime_gll(iz, ix) = + quadrature.gll.hprime(iz, ix); + } else { + element_quadrature.hprime_gll(iz, ix) = + quadrature.gll.h_hprime(iz, ix); + } } if constexpr (store_weight_times_hprime_gll) { - element_quadrature.hprime_wgll(ix, iz) = - quadrature.gll.hprime(iz, ix) * quadrature.gll.weights(iz); + if constexpr (on_device) { + element_quadrature.hprime_wgll(ix, iz) = + quadrature.gll.hprime(iz, ix) * quadrature.gll.weights(iz); + } else { + element_quadrature.hprime_wgll(ix, iz) = + quadrature.gll.h_hprime(iz, ix) * quadrature.gll.h_weights(iz); + } } }); } +/** + * @defgroup QuadratureDataAccess + * + */ + +/** + * @brief Load quadrature data for a spectral element on the device + * + * @ingroup QuadratureDataAccess + * + * @tparam MemberType Member type. Needs to be a Kokkos::TeamPolicy member type + * @tparam ViewType View type. Needs to be of @ref specfem::element::quadrature + * @param team Team member + * @param quadrature Quadrature data + * @param element_quadrature Quadrature data for the element (output) + */ +template +KOKKOS_FUNCTION void +load_on_device(const MemberType &team, + const specfem::compute::quadrature &quadrature, + ViewType &element_quadrature) { + + impl_load(team, quadrature, element_quadrature); +} + /** * @brief Load quadrature data for a spectral element on the host * @@ -284,27 +320,7 @@ template void load_on_host(const MemberType &team, const specfem::compute::quadrature &quadrature, ViewType &element_quadrature) { - - constexpr bool store_hprime_gll = ViewType::store_hprime_gll; - constexpr bool store_weight_times_hprime_gll = - ViewType::store_weight_times_hprime_gll; - constexpr int NGLL = ViewType::ngll; - - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, NGLL * NGLL), [=](const int &xz) { - int ix, iz; - sub2ind(xz, NGLL, iz, ix); - if constexpr (store_hprime_gll) { - element_quadrature.hprime_gll(iz, ix) = - quadrature.gll.h_hprime(iz, ix); - } - if constexpr (store_weight_times_hprime_gll) { - element_quadrature.hprime_wgll(ix, iz) = - quadrature.gll.h_hprime(iz, ix) * quadrature.gll.h_weights(iz); - } - }); - - return; + impl_load(team, quadrature, element_quadrature); } } // namespace compute diff --git a/include/compute/compute_partial_derivatives.hpp b/include/compute/compute_partial_derivatives.hpp index 700eb79f..4a144084 100644 --- a/include/compute/compute_partial_derivatives.hpp +++ b/include/compute/compute_partial_derivatives.hpp @@ -71,10 +71,10 @@ struct partial_derivatives { * */ -template = 0> -KOKKOS_FORCEINLINE_FUNCTION void impl_load_on_device( +KOKKOS_FORCEINLINE_FUNCTION void impl_load( const specfem::point::simd_index &index, const specfem::compute::partial_derivatives &derivatives, @@ -94,24 +94,39 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_load_on_device( mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - Kokkos::Experimental::where(mask, partial_derivatives.xix) - .copy_from(&derivatives.xix(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, partial_derivatives.gammax) - .copy_from(&derivatives.gammax(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, partial_derivatives.xiz) - .copy_from(&derivatives.xiz(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, partial_derivatives.gammaz) - .copy_from(&derivatives.gammaz(ispec, iz, ix), tag_type()); - if constexpr (StoreJacobian) { - Kokkos::Experimental::where(mask, partial_derivatives.jacobian) - .copy_from(&derivatives.jacobian(ispec, iz, ix), tag_type()); + if constexpr (on_device) { + Kokkos::Experimental::where(mask, partial_derivatives.xix) + .copy_from(&derivatives.xix(ispec, iz, ix), tag_type()); + Kokkos::Experimental::where(mask, partial_derivatives.gammax) + .copy_from(&derivatives.gammax(ispec, iz, ix), tag_type()); + Kokkos::Experimental::where(mask, partial_derivatives.xiz) + .copy_from(&derivatives.xiz(ispec, iz, ix), tag_type()); + Kokkos::Experimental::where(mask, partial_derivatives.gammaz) + .copy_from(&derivatives.gammaz(ispec, iz, ix), tag_type()); + if constexpr (StoreJacobian) { + Kokkos::Experimental::where(mask, partial_derivatives.jacobian) + .copy_from(&derivatives.jacobian(ispec, iz, ix), tag_type()); + } + } else { + Kokkos::Experimental::where(mask, partial_derivatives.xix) + .copy_from(&derivatives.h_xix(ispec, iz, ix), tag_type()); + Kokkos::Experimental::where(mask, partial_derivatives.gammax) + .copy_from(&derivatives.h_gammax(ispec, iz, ix), tag_type()); + Kokkos::Experimental::where(mask, partial_derivatives.xiz) + .copy_from(&derivatives.h_xiz(ispec, iz, ix), tag_type()); + Kokkos::Experimental::where(mask, partial_derivatives.gammaz) + .copy_from(&derivatives.h_gammaz(ispec, iz, ix), tag_type()); + if constexpr (StoreJacobian) { + Kokkos::Experimental::where(mask, partial_derivatives.jacobian) + .copy_from(&derivatives.h_jacobian(ispec, iz, ix), tag_type()); + } } } -template = 0> -KOKKOS_FORCEINLINE_FUNCTION void impl_load_on_device( +KOKKOS_FORCEINLINE_FUNCTION void impl_load( const specfem::point::index &index, const specfem::compute::partial_derivatives &derivatives, PointPartialDerivativesType &partial_derivatives) { @@ -123,73 +138,22 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_load_on_device( constexpr static bool StoreJacobian = PointPartialDerivativesType::store_jacobian; - partial_derivatives.xix = derivatives.xix(ispec, iz, ix); - partial_derivatives.gammax = derivatives.gammax(ispec, iz, ix); - partial_derivatives.xiz = derivatives.xiz(ispec, iz, ix); - partial_derivatives.gammaz = derivatives.gammaz(ispec, iz, ix); - if constexpr (StoreJacobian) { - partial_derivatives.jacobian = derivatives.jacobian(ispec, iz, ix); - } -} - -template = 0> -inline void -impl_load_on_host(const specfem::point::simd_index< - PointPointPartialDerivativesType::dimension> &index, - const specfem::compute::partial_derivatives &derivatives, - PointPointPartialDerivativesType &partial_derivatives) { - - const int ispec = index.ispec; - const int nspec = derivatives.nspec; - const int iz = index.iz; - const int ix = index.ix; - - constexpr static bool StoreJacobian = - PointPointPartialDerivativesType::store_jacobian; - - using simd = typename PointPointPartialDerivativesType::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, partial_derivatives.xix) - .copy_from(&derivatives.h_xix(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, partial_derivatives.gammax) - .copy_from(&derivatives.h_gammax(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, partial_derivatives.xiz) - .copy_from(&derivatives.h_xiz(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, partial_derivatives.gammaz) - .copy_from(&derivatives.h_gammaz(ispec, iz, ix), tag_type()); - if constexpr (StoreJacobian) { - Kokkos::Experimental::where(mask, partial_derivatives.jacobian) - .copy_from(&derivatives.h_jacobian(ispec, iz, ix), tag_type()); - } -} - -template = 0> -inline void impl_load_on_host( - const specfem::point::index &index, - const specfem::compute::partial_derivatives &derivatives, - PointPartialDerivativesType &partial_derivatives) { - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - constexpr static bool StoreJacobian = - PointPartialDerivativesType::store_jacobian; - - partial_derivatives.xix = derivatives.h_xix(ispec, iz, ix); - partial_derivatives.gammax = derivatives.h_gammax(ispec, iz, ix); - partial_derivatives.xiz = derivatives.h_xiz(ispec, iz, ix); - partial_derivatives.gammaz = derivatives.h_gammaz(ispec, iz, ix); - if constexpr (StoreJacobian) { - partial_derivatives.jacobian = derivatives.h_jacobian(ispec, iz, ix); + if constexpr (on_device) { + partial_derivatives.xix = derivatives.xix(ispec, iz, ix); + partial_derivatives.gammax = derivatives.gammax(ispec, iz, ix); + partial_derivatives.xiz = derivatives.xiz(ispec, iz, ix); + partial_derivatives.gammaz = derivatives.gammaz(ispec, iz, ix); + if constexpr (StoreJacobian) { + partial_derivatives.jacobian = derivatives.jacobian(ispec, iz, ix); + } + } else { + partial_derivatives.xix = derivatives.h_xix(ispec, iz, ix); + partial_derivatives.gammax = derivatives.h_gammax(ispec, iz, ix); + partial_derivatives.xiz = derivatives.h_xiz(ispec, iz, ix); + partial_derivatives.gammaz = derivatives.h_gammaz(ispec, iz, ix); + if constexpr (StoreJacobian) { + partial_derivatives.jacobian = derivatives.h_jacobian(ispec, iz, ix); + } } } @@ -276,7 +240,7 @@ KOKKOS_FORCEINLINE_FUNCTION void load_on_device(const IndexType &index, const specfem::compute::partial_derivatives &derivatives, PointPartialDerivativesType &partial_derivatives) { - impl_load_on_device(index, derivatives, partial_derivatives); + impl_load(index, derivatives, partial_derivatives); } /** @@ -302,7 +266,7 @@ inline void load_on_host(const IndexType &index, const specfem::compute::partial_derivatives &derivatives, PointPartialDerivativesType &partial_derivatives) { - impl_load_on_host(index, derivatives, partial_derivatives); + impl_load(index, derivatives, partial_derivatives); } /** diff --git a/include/compute/fields/data_access.tpp b/include/compute/fields/data_access.tpp index 94f27af4..aff301a4 100644 --- a/include/compute/fields/data_access.tpp +++ b/include/compute/fields/data_access.tpp @@ -6,11 +6,11 @@ namespace specfem { namespace compute { -template < +template = 0> -KOKKOS_FORCEINLINE_FUNCTION void impl_load_on_device(const int iglob, +KOKKOS_FORCEINLINE_FUNCTION void impl_load(const int iglob, const WavefieldType &field, ViewType &point_field) { @@ -18,47 +18,47 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_load_on_device(const int iglob, constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < ViewType::components; ++icomp) { point_field.displacement(icomp) = - curr_field.field(iglob, icomp); + curr_field.template get_field(iglob, icomp); } } if constexpr (StoreVelocity) { for (int icomp = 0; icomp < ViewType::components; ++icomp) { point_field.velocity(icomp) = - curr_field.field_dot(iglob, icomp); + curr_field.template get_field_dot(iglob, icomp); } } if constexpr (StoreAcceleration) { for (int icomp = 0; icomp < ViewType::components; ++icomp) { point_field.acceleration(icomp) = - curr_field.field_dot_dot(iglob, icomp); + curr_field.template get_field_dot_dot(iglob, icomp); } } if constexpr (StoreMassMatrix) { for (int icomp = 0; icomp < ViewType::components; ++icomp) { point_field.mass_matrix(icomp) = - curr_field.mass_inverse(iglob, icomp); + curr_field.template get_mass_inverse(iglob, icomp); } } return; } -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_load_on_device(const typename ViewType::simd::mask_type &mask, +impl_load(const typename ViewType::simd::mask_type &mask, const int *iglob, const WavefieldType &field, ViewType &point_field) { @@ -66,10 +66,10 @@ impl_load_on_device(const typename ViewType::simd::mask_type &mask, constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); for (int lane = 0; lane < ViewType::simd::size(); ++lane) { if (!mask[lane]) { @@ -81,28 +81,28 @@ impl_load_on_device(const typename ViewType::simd::mask_type &mask, if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { point_field.displacement(icomp)[lane] = - curr_field.field(iglob_l, icomp); + curr_field.template get_field(iglob_l, icomp); } } if constexpr (StoreVelocity) { for (int icomp = 0; icomp < components; ++icomp) { point_field.velocity(icomp)[lane] = - curr_field.field_dot(iglob_l, icomp); + curr_field.template get_field_dot(iglob_l, icomp); } } if constexpr (StoreAcceleration) { for (int icomp = 0; icomp < components; ++icomp) { point_field.acceleration(icomp)[lane] = - curr_field.field_dot_dot(iglob_l, icomp); + curr_field.template get_field_dot_dot(iglob_l, icomp); } } if constexpr (StoreMassMatrix) { for (int icomp = 0; icomp < components; ++icomp) { point_field.mass_matrix(icomp)[lane] = - curr_field.mass_inverse(iglob_l, icomp); + curr_field.template get_mass_inverse(iglob_l, icomp); } } } @@ -110,19 +110,19 @@ impl_load_on_device(const typename ViewType::simd::mask_type &mask, return; } -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_load_on_device(const specfem::point::simd_assembly_index &index, +impl_load(const specfem::point::simd_assembly_index &index, const WavefieldType &field, ViewType &point_field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; const int iglob = index.iglob; @@ -132,102 +132,137 @@ impl_load_on_device(const specfem::point::simd_assembly_index &index, mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.displacement(icomp)) - .copy_from(&curr_field.field(iglob, icomp), tag_type()); + .copy_from(&(curr_field.template get_field(iglob, icomp)), tag_type()); } } if constexpr (StoreVelocity) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.velocity(icomp)) - .copy_from(&curr_field.field_dot(iglob, icomp), tag_type()); + .copy_from(&(curr_field.template get_field_dot(iglob, icomp)), tag_type()); } } if constexpr (StoreAcceleration) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.acceleration(icomp)) - .copy_from(&curr_field.field_dot_dot(iglob, icomp), tag_type()); + .copy_from(&(curr_field.template get_field_dot_dot(iglob, icomp)), tag_type()); } } if constexpr (StoreMassMatrix) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.mass_matrix(icomp)) - .copy_from(&curr_field.mass_inverse(iglob, icomp), tag_type()); + .copy_from(&(curr_field.template get_mass_inverse(iglob, icomp)), tag_type()); } } return; } -template < +template = 0> -inline void impl_load_on_host(const int iglob, const WavefieldType &field, - ViewType &point_field) { +KOKKOS_FORCEINLINE_FUNCTION void +impl_load(const specfem::point::index &index, + const WavefieldType &field, ViewType &point_field) { + constexpr static auto MediumTag = ViewType::medium_tag; + impl_load(field.template get_iglob(index.ispec, index.iz, index.ix, MediumTag), field, point_field); +} + +template = 0> +KOKKOS_FORCEINLINE_FUNCTION void +impl_load(const specfem::point::assembly_index &index, + const WavefieldType &field, ViewType &point_field) { + constexpr static auto MediumTag = ViewType::medium_tag; + const int iglob = index.iglob; + impl_load(iglob, field, point_field); +} + +template = 0> +KOKKOS_FORCEINLINE_FUNCTION void impl_load( + const specfem::point::simd_index &index, + const WavefieldType &field, ViewType &point_field) { + constexpr static auto MediumTag = ViewType::medium_tag; + int iglob[ViewType::simd::size()]; + + using mask_type = typename ViewType::simd::mask_type; + + mask_type mask([&](std::size_t lane) { return index.mask(lane); }); + + for (int lane = 0; lane < ViewType::simd::size(); ++lane) { + iglob[lane] = index.mask(lane) ? + field.template get_iglob( + index.ispec + lane, index.iz, index.ix, MediumTag) : + field.nglob + 1; + } + + impl_load(mask, iglob, field, point_field); +} + +template = 0> +KOKKOS_FORCEINLINE_FUNCTION void +impl_store(const int iglob, const ViewType &point_field, + const WavefieldType &field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); - if constexpr (StoreDisplacement) { - for (int icomp = 0; icomp < ViewType::components; ++icomp) { - point_field.displacement(icomp) = - curr_field.h_field(iglob, icomp); + for (int icomp = 0; icomp < ViewType::components; ++icomp) { + if constexpr (StoreDisplacement) { + curr_field.template get_field(iglob, icomp) = point_field.displacement(icomp); } - } - - if constexpr (StoreVelocity) { - for (int icomp = 0; icomp < ViewType::components; ++icomp) { - point_field.velocity(icomp) = - curr_field.h_field_dot(iglob, icomp); + if constexpr (StoreVelocity) { + curr_field.template get_field_dot(iglob, icomp) = point_field.velocity(icomp); } - } - - if constexpr (StoreAcceleration) { - for (int icomp = 0; icomp < ViewType::components; ++icomp) { - point_field.acceleration(icomp) = - curr_field.h_field_dot_dot(iglob, icomp); + if constexpr (StoreAcceleration) { + curr_field.template get_field_dot_dot(iglob, icomp) = point_field.acceleration(icomp); } - } - - if constexpr (StoreMassMatrix) { - for (int icomp = 0; icomp < ViewType::components; ++icomp) { - point_field.mass_matrix(icomp) = - curr_field.h_mass_inverse(iglob, icomp); + if constexpr (StoreMassMatrix) { + curr_field.template get_mass_inverse(iglob, icomp) = point_field.mass_matrix(icomp); } } return; } -template < +template = 0> -inline void impl_load_on_host(const typename ViewType::simd::mask_type &mask, - const int *iglob, const WavefieldType &field, - ViewType &point_field) { + typename std::enable_if_t< + ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> +KOKKOS_FORCEINLINE_FUNCTION void +impl_store(const typename ViewType::simd::mask_type &mask, + const int *iglob, const ViewType &point_field, + const WavefieldType &field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; - const auto &curr_field = field.template get_field(); - + const auto &curr_field = field.template get_field(); for (int lane = 0; lane < ViewType::simd::size(); ++lane) { if (!mask[lane]) { continue; @@ -237,29 +272,29 @@ inline void impl_load_on_host(const typename ViewType::simd::mask_type &mask, if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { - point_field.displacement(icomp)[lane] = - curr_field.h_field(iglob_l, icomp); + curr_field.template get_field(iglob_l, icomp) = + point_field.displacement(icomp)[lane]; } } if constexpr (StoreVelocity) { for (int icomp = 0; icomp < components; ++icomp) { - point_field.velocity(icomp)[lane] = - curr_field.h_field_dot(iglob_l, icomp); + curr_field.template get_field_dot(iglob_l, icomp) = + point_field.velocity(icomp)[lane]; } } if constexpr (StoreAcceleration) { for (int icomp = 0; icomp < components; ++icomp) { - point_field.acceleration(icomp)[lane] = - curr_field.h_field_dot_dot(iglob_l, icomp); + curr_field.template get_field_dot_dot(iglob_l, icomp) = + point_field.acceleration(icomp)[lane]; } } if constexpr (StoreMassMatrix) { for (int icomp = 0; icomp < components; ++icomp) { - point_field.mass_matrix(icomp)[lane] = - curr_field.h_mass_inverse(iglob_l, icomp); + curr_field.template get_mass_inverse(iglob_l, icomp) = + point_field.mass_matrix(icomp)[lane]; } } } @@ -267,18 +302,19 @@ inline void impl_load_on_host(const typename ViewType::simd::mask_type &mask, return; } -template < +template = 0> -inline void impl_load_on_host(const specfem::point::simd_assembly_index &index, - const WavefieldType &field, ViewType &point_field) { + typename std::enable_if_t< + ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> +KOKKOS_FORCEINLINE_FUNCTION void +impl_store(const specfem::point::simd_assembly_index &index, + const ViewType &point_field, const WavefieldType &field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; using mask_type = typename ViewType::simd::mask_type; @@ -288,127 +324,75 @@ inline void impl_load_on_host(const specfem::point::simd_assembly_index &index, mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - const auto &curr_field = field.template get_field(); - + const auto &curr_field = field.template get_field(); if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.displacement(icomp)) - .copy_from(&curr_field.h_field(iglob, icomp), tag_type()); + .copy_to(&(curr_field.template get_field(iglob, icomp)), tag_type()); } } if constexpr (StoreVelocity) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.velocity(icomp)) - .copy_from(&curr_field.h_field_dot(iglob, icomp), tag_type()); + .copy_to(&(curr_field.template get_field_dot(iglob, icomp)), tag_type()); } } if constexpr (StoreAcceleration) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.acceleration(icomp)) - .copy_from(&curr_field.h_field_dot_dot(iglob, icomp), tag_type()); + .copy_to(&(curr_field.template get_field_dot_dot(iglob, icomp)), tag_type()); } } if constexpr (StoreMassMatrix) { for (int icomp = 0; icomp < components; ++icomp) { Kokkos::Experimental::where(mask, point_field.mass_matrix(icomp)) - .copy_from(&curr_field.h_mass_inverse(iglob, icomp), tag_type()); + .copy_to(&(curr_field.template get_mass_inverse(iglob, icomp)), tag_type()); } } return; } -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_load_on_device(const specfem::point::index &index, - const WavefieldType &field, ViewType &point_field) { - constexpr static auto MediumType = ViewType::medium_tag; - const int iglob_l = field.index_mapping(index.ispec, index.iz, index.ix); - const int iglob = - field.assembly_index_mapping(iglob_l, static_cast(MediumType)); - impl_load_on_device(iglob, field, point_field); -} - -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_load_on_device(const specfem::point::assembly_index &index, - const WavefieldType &field, ViewType &point_field) { - constexpr static auto MediumType = ViewType::medium_tag; - const int iglob = index.iglob; - impl_load_on_device(iglob, field, point_field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void impl_load_on_device( - const specfem::point::simd_index &index, - const WavefieldType &field, ViewType &point_field) { - constexpr static auto MediumType = ViewType::medium_tag; - int iglob[ViewType::simd::size()]; - - using mask_type = typename ViewType::simd::mask_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); +impl_store(const specfem::point::index &index, + const ViewType &point_field, const WavefieldType &field) { - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = - (index.mask(std::size_t(lane))) - ? field.assembly_index_mapping( - field.index_mapping(index.ispec + lane, index.iz, index.ix), - static_cast(MediumType)) - : field.nglob + 1; - } + constexpr static auto MediumTag = ViewType::medium_tag; - impl_load_on_device(mask, iglob, field, point_field); + impl_store(field.template get_iglob(index.ispec, index.iz, index.ix, MediumTag), point_field, field); } -template < +template = 0> -inline void impl_load_on_host(const specfem::point::index &index, - const WavefieldType &field, ViewType &point_field) { - - constexpr static auto MediumType = ViewType::medium_tag; - const int iglob = field.h_assembly_index_mapping( - field.h_index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); - - impl_load_on_host(iglob, field, point_field); -} +KOKKOS_FORCEINLINE_FUNCTION void +impl_store(const specfem::point::assembly_index &index, + const ViewType &point_field, const WavefieldType &field) { -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_load_on_host(const specfem::point::assembly_index &index, - const WavefieldType &field, ViewType &point_field) { + constexpr static auto MediumTag = ViewType::medium_tag; - constexpr static auto MediumType = ViewType::medium_tag; const int iglob = index.iglob; - impl_load_on_host(iglob, field, point_field); + + impl_store(iglob, point_field, field); } -template < +template = 0> -inline void impl_load_on_host( +KOKKOS_FORCEINLINE_FUNCTION void impl_store( const specfem::point::simd_index &index, - const WavefieldType &field, ViewType &point_field) { + const ViewType &point_field, const WavefieldType &field) { - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; int iglob[ViewType::simd::size()]; using mask_type = typename ViewType::simd::mask_type; @@ -416,68 +400,65 @@ inline void impl_load_on_host( mask_type mask([&](std::size_t lane) { return index.mask(lane); }); for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = - (index.mask(std::size_t(lane))) - ? field.h_assembly_index_mapping( - field.h_index_mapping(index.ispec + lane, index.iz, index.ix), - static_cast(MediumType)) - : field.nglob + 1; + iglob[lane] = index.mask(lane) ? + field.template get_iglob( + index.ispec + lane, index.iz, index.ix, MediumTag) : + field.nglob + 1; } - impl_load_on_host(mask, iglob, field, point_field); + impl_store(mask, iglob, point_field, field); } -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_store_on_device(const int iglob, const ViewType &point_field, - const WavefieldType &field) { +impl_add(const int iglob, const ViewType &point_field, + const WavefieldType &field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - - const auto &curr_field = field.template get_field(); + constexpr static auto MediumTag = ViewType::medium_tag; + const auto &curr_field = field.template get_field(); for (int icomp = 0; icomp < ViewType::components; ++icomp) { if constexpr (StoreDisplacement) { - curr_field.field(iglob, icomp) = point_field.displacement(icomp); + curr_field.template get_field(iglob, icomp) += point_field.displacement(icomp); } if constexpr (StoreVelocity) { - curr_field.field_dot(iglob, icomp) = point_field.velocity(icomp); + curr_field.template get_field_dot(iglob, icomp) += point_field.velocity(icomp); } if constexpr (StoreAcceleration) { - curr_field.field_dot_dot(iglob, icomp) = point_field.acceleration(icomp); + curr_field.template get_field_dot_dot(iglob, icomp) += point_field.acceleration(icomp); } if constexpr (StoreMassMatrix) { - curr_field.mass_inverse(iglob, icomp) = point_field.mass_matrix(icomp); + curr_field.template get_mass_inverse(iglob, icomp) += point_field.mass_matrix(icomp); } } return; } -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_store_on_device(const typename ViewType::simd::mask_type &mask, - const int *iglob, const ViewType &point_field, - const WavefieldType &field) { +impl_add(const typename ViewType::simd::mask_type &mask, + const int *iglob, const ViewType &point_field, + const WavefieldType &field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); for (int lane = 0; lane < ViewType::simd::size(); ++lane) { if (!mask[lane]) { continue; @@ -487,28 +468,28 @@ impl_store_on_device(const typename ViewType::simd::mask_type &mask, if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { - curr_field.field(iglob_l, icomp) = + curr_field.template get_field(iglob_l, icomp) += point_field.displacement(icomp)[lane]; } } if constexpr (StoreVelocity) { for (int icomp = 0; icomp < components; ++icomp) { - curr_field.field_dot(iglob_l, icomp) = + curr_field.template get_field_dot(iglob_l, icomp) += point_field.velocity(icomp)[lane]; } } if constexpr (StoreAcceleration) { for (int icomp = 0; icomp < components; ++icomp) { - curr_field.field_dot_dot(iglob_l, icomp) = + curr_field.template get_field_dot_dot(iglob_l, icomp) += point_field.acceleration(icomp)[lane]; } } if constexpr (StoreMassMatrix) { for (int icomp = 0; icomp < components; ++icomp) { - curr_field.mass_inverse(iglob_l, icomp) = + curr_field.template get_mass_inverse(iglob_l, icomp) += point_field.mass_matrix(icomp)[lane]; } } @@ -517,19 +498,19 @@ impl_store_on_device(const typename ViewType::simd::mask_type &mask, return; } -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_store_on_device(const specfem::point::simd_assembly_index &index, - const ViewType &point_field, const WavefieldType &field) { +impl_add(const specfem::point::simd_assembly_index &index, + const ViewType &point_field, const WavefieldType &field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; using mask_type = typename ViewType::simd::mask_type; @@ -539,87 +520,163 @@ impl_store_on_device(const specfem::point::simd_assembly_index &index, mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::Experimental::where(mask, point_field.displacement(icomp)) - .copy_to(&curr_field.field(iglob, icomp), tag_type()); + typename ViewType::simd::datatype lhs; + Kokkos::Experimental::where(mask, lhs).copy_from( + &(curr_field.template get_field(iglob, icomp)), tag_type()); + + lhs += point_field.displacement(icomp); + Kokkos::Experimental::where(mask, lhs).copy_to( + &(curr_field.template get_field(iglob, icomp)), tag_type()); } } if constexpr (StoreVelocity) { for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::Experimental::where(mask, point_field.velocity(icomp)) - .copy_to(&curr_field.field_dot(iglob, icomp), tag_type()); + typename ViewType::simd::datatype lhs; + Kokkos::Experimental::where(mask, lhs).copy_from( + &(curr_field.template get_field_dot(iglob, icomp)), tag_type()); + + lhs += point_field.velocity(icomp); + Kokkos::Experimental::where(mask, lhs).copy_to( + &(curr_field.template get_field_dot(iglob, icomp)), tag_type()); } } if constexpr (StoreAcceleration) { for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::Experimental::where(mask, point_field.acceleration(icomp)) - .copy_to(&curr_field.field_dot_dot(iglob, icomp), tag_type()); + typename ViewType::simd::datatype lhs; + Kokkos::Experimental::where(mask, lhs).copy_from( + &(curr_field.template get_field_dot_dot(iglob, icomp)), tag_type()); + + lhs += point_field.acceleration(icomp); + Kokkos::Experimental::where(mask, lhs).copy_to( + &(curr_field.template get_field_dot_dot(iglob, icomp)), tag_type()); } } if constexpr (StoreMassMatrix) { for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::Experimental::where(mask, point_field.mass_matrix(icomp)) - .copy_to(&curr_field.mass_inverse(iglob, icomp), tag_type()); + typename ViewType::simd::datatype lhs; + Kokkos::Experimental::where(mask, lhs).copy_from( + &(curr_field.template get_mass_inverse(iglob, icomp)), tag_type()); + + lhs += point_field.mass_matrix(icomp); + Kokkos::Experimental::where(mask, lhs).copy_to( + &(curr_field.template get_mass_inverse(iglob, icomp)), tag_type()); } } return; } -template < +template = 0> +KOKKOS_FORCEINLINE_FUNCTION void +impl_add(const specfem::point::index &index, + const ViewType &point_field, const WavefieldType &field) { + + constexpr static auto MediumTag = ViewType::medium_tag; + impl_add(field.template get_iglob(index.ispec, index.iz, index.ix, MediumTag), point_field, field); +} + +template = 0> +KOKKOS_FORCEINLINE_FUNCTION void +impl_add(const specfem::point::assembly_index &index, + const ViewType &point_field, const WavefieldType &field) { + + constexpr static auto MediumTag = ViewType::medium_tag; + + const int iglob = index.iglob; + + impl_add(iglob, point_field, field); +} + +template = 0> +KOKKOS_FORCEINLINE_FUNCTION void +impl_add(const specfem::point::simd_index &index, + const ViewType &point_field, const WavefieldType &field) { + + constexpr static auto MediumTag = ViewType::medium_tag; + int iglob[ViewType::simd::size()]; + + using mask_type = typename ViewType::simd::mask_type; + + mask_type mask([&](std::size_t lane) { return index.mask(lane); }); + + for (int lane = 0; lane < ViewType::simd::size(); ++lane) { + iglob[lane] = index.mask(lane) ? + field.template get_iglob(index.ispec + lane, index.iz, index.ix, MediumTag) : + field.nglob + 1; + } + + impl_add(mask, iglob, point_field, field); +} + +template = 0> -inline void impl_store_on_host(const int iglob, const ViewType &point_field, - const WavefieldType &field) { +KOKKOS_FORCEINLINE_FUNCTION void +impl_atomic_add(const int iglob, const ViewType &point_field, + const WavefieldType &field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); for (int icomp = 0; icomp < ViewType::components; ++icomp) { if constexpr (StoreDisplacement) { - curr_field.h_field(iglob, icomp) = point_field.displacement(icomp); + Kokkos::atomic_add(&(curr_field.template get_field(iglob, icomp)), + point_field.displacement(icomp)); } if constexpr (StoreVelocity) { - curr_field.h_field_dot(iglob, icomp) = point_field.velocity(icomp); + Kokkos::atomic_add(&(curr_field.template get_field_dot(iglob, icomp)), + point_field.velocity(icomp)); } if constexpr (StoreAcceleration) { - curr_field.h_field_dot_dot(iglob, icomp) = - point_field.acceleration(icomp); + Kokkos::atomic_add(&(curr_field.template get_field_dot_dot(iglob, icomp)), + point_field.acceleration(icomp)); } if constexpr (StoreMassMatrix) { - curr_field.h_mass_inverse(iglob, icomp) = point_field.mass_matrix(icomp); + Kokkos::atomic_add(&(curr_field.template get_mass_inverse(iglob, icomp)), + point_field.mass_matrix(icomp)); } } return; } -template < +template = 0> -inline void impl_store_on_host(const typename ViewType::simd::mask_type &mask, - const int *iglob, const ViewType &point_field, - const WavefieldType &field) { +KOKKOS_FORCEINLINE_FUNCTION void +impl_atomic_add(const typename ViewType::simd::mask_type &mask, + const int *iglob, const ViewType &point_field, + const WavefieldType &field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); for (int lane = 0; lane < ViewType::simd::size(); ++lane) { if (!mask[lane]) { continue; @@ -629,29 +686,29 @@ inline void impl_store_on_host(const typename ViewType::simd::mask_type &mask, if constexpr (StoreDisplacement) { for (int icomp = 0; icomp < components; ++icomp) { - curr_field.h_field(iglob_l, icomp) = - point_field.displacement(icomp)[lane]; + Kokkos::atomic_add(&(curr_field.template get_field(iglob_l, icomp)), + point_field.displacement(icomp)[lane]); } } if constexpr (StoreVelocity) { for (int icomp = 0; icomp < components; ++icomp) { - curr_field.h_field_dot(iglob_l, icomp) = - point_field.velocity(icomp)[lane]; + Kokkos::atomic_add(&(curr_field.template get_field_dot(iglob_l, icomp)), + point_field.velocity(icomp)[lane]); } } if constexpr (StoreAcceleration) { for (int icomp = 0; icomp < components; ++icomp) { - curr_field.h_field_dot_dot(iglob_l, icomp) = - point_field.acceleration(icomp)[lane]; + Kokkos::atomic_add(&(curr_field.template get_field_dot_dot(iglob_l, icomp)), + point_field.acceleration(icomp)[lane]); } } if constexpr (StoreMassMatrix) { for (int icomp = 0; icomp < components; ++icomp) { - curr_field.h_mass_inverse(iglob_l, icomp) = - point_field.mass_matrix(icomp)[lane]; + Kokkos::atomic_add(&(curr_field.template get_mass_inverse(iglob_l, icomp)), + point_field.mass_matrix(icomp)[lane]); } } } @@ -659,101 +716,28 @@ inline void impl_store_on_host(const typename ViewType::simd::mask_type &mask, return; } -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -inline void impl_store_on_host(const specfem::point::simd_assembly_index &index, - const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - using mask_type = typename ViewType::simd::mask_type; - using tag_type = typename ViewType::simd::tag_type; - - const int iglob = index.iglob; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const auto &curr_field = field.template get_field(); - if constexpr (StoreDisplacement) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::Experimental::where(mask, point_field.displacement(icomp)) - .copy_to(&curr_field.h_field(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreVelocity) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::Experimental::where(mask, point_field.velocity(icomp)) - .copy_to(&curr_field.h_field_dot(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreAcceleration) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::Experimental::where(mask, point_field.acceleration(icomp)) - .copy_to(&curr_field.h_field_dot_dot(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreMassMatrix) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::Experimental::where(mask, point_field.acceleration(icomp)) - .copy_to(&curr_field.h_mass_matrix(iglob, icomp), tag_type()); - } - } - - return; -} - -template < +template = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_store_on_device(const specfem::point::index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = field.assembly_index_mapping( - field.index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); - - impl_store_on_device(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_store_on_device(const specfem::point::assembly_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; +KOKKOS_FORCEINLINE_FUNCTION void impl_atomic_add( + const specfem::point::index &index, + const ViewType &point_field, const WavefieldType &field) { - const int iglob = index.iglob; + constexpr static auto MediumTag = ViewType::medium_tag; - impl_store_on_device(iglob, point_field, field); + impl_atomic_add(field.template get_iglob(index.ispec, index.iz, index.ix, MediumTag), point_field, field); } -template < +template = 0> -KOKKOS_FORCEINLINE_FUNCTION void impl_store_on_device( +KOKKOS_FORCEINLINE_FUNCTION void impl_atomic_add( const specfem::point::simd_index &index, const ViewType &point_field, const WavefieldType &field) { - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; int iglob[ViewType::simd::size()]; using mask_type = typename ViewType::simd::mask_type; @@ -761,878 +745,63 @@ KOKKOS_FORCEINLINE_FUNCTION void impl_store_on_device( mask_type mask([&](std::size_t lane) { return index.mask(lane); }); for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = - (index.mask(std::size_t(lane))) - ? field.assembly_index_mapping( - field.index_mapping(index.ispec + lane, index.iz, index.ix), - static_cast(MediumType)) - : field.nglob + 1; + iglob[lane] = index.mask(lane) ? + field.template get_iglob(index.ispec + lane, index.iz, index.ix, MediumTag) : + field.nglob + 1; } - impl_store_on_device(mask, iglob, point_field, field); + impl_atomic_add(mask, iglob, point_field, field); } -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_store_on_host(const specfem::point::index &index, - const ViewType &point_field, - const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = field.h_assembly_index_mapping( - field.h_index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); - - impl_store_on_host(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_store_on_host(const specfem::point::assembly_index &index, - const ViewType &point_field, - const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = index.iglob; - - impl_store_on_host(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -inline void impl_store_on_host( - const specfem::point::simd_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - int iglob[ViewType::simd::size()]; - - using mask_type = typename ViewType::simd::mask_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = - (index.mask(std::size_t(lane))) - ? field.h_index_mapping(index.ispec + lane, index.iz, index.ix) - : field.nglob + 1; - } - - impl_store_on_host(mask, iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_add_on_device(const int iglob, const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - - const auto &curr_field = field.template get_field(); - for (int icomp = 0; icomp < ViewType::components; ++icomp) { - if constexpr (StoreDisplacement) { - curr_field.field(iglob, icomp) += point_field.displacement(icomp); - } - if constexpr (StoreVelocity) { - curr_field.field_dot(iglob, icomp) += point_field.velocity(icomp); - } - if constexpr (StoreAcceleration) { - curr_field.field_dot_dot(iglob, icomp) += point_field.acceleration(icomp); - } - if constexpr (StoreMassMatrix) { - curr_field.mass_inverse(iglob, icomp) += point_field.mass_matrix(icomp); - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_add_on_device(const typename ViewType::simd::mask_type &mask, - const int *iglob, const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - const auto &curr_field = field.template get_field(); - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - if (!mask[lane]) { - continue; - } - - const int iglob_l = iglob[lane]; - - if constexpr (StoreDisplacement) { - for (int icomp = 0; icomp < components; ++icomp) { - curr_field.field(iglob_l, icomp) += - point_field.displacement(icomp)[lane]; - } - } - - if constexpr (StoreVelocity) { - for (int icomp = 0; icomp < components; ++icomp) { - curr_field.field_dot(iglob_l, icomp) += - point_field.velocity(icomp)[lane]; - } - } - - if constexpr (StoreAcceleration) { - for (int icomp = 0; icomp < components; ++icomp) { - curr_field.field_dot_dot(iglob_l, icomp) += - point_field.acceleration(icomp)[lane]; - } - } - - if constexpr (StoreMassMatrix) { - for (int icomp = 0; icomp < components; ++icomp) { - curr_field.mass_inverse(iglob_l, icomp) += - point_field.mass_matrix(icomp)[lane]; - } - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_add_on_device(const specfem::point::simd_assembly_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - using mask_type = typename ViewType::simd::mask_type; - using tag_type = typename ViewType::simd::tag_type; - - const int iglob = index.iglob; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const auto &curr_field = field.template get_field(); - if constexpr (StoreDisplacement) { - for (int icomp = 0; icomp < components; ++icomp) { - typename ViewType::simd::datatype lhs; - Kokkos::Experimental::where(mask, lhs).copy_from( - &curr_field.field(iglob, icomp), tag_type()); - - lhs += point_field.displacement(icomp); - Kokkos::Experimental::where(mask, lhs).copy_to( - &curr_field.field(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreVelocity) { - for (int icomp = 0; icomp < components; ++icomp) { - typename ViewType::simd::datatype lhs; - Kokkos::Experimental::where(mask, lhs).copy_from( - &curr_field.field_dot(iglob, icomp), tag_type()); - - lhs += point_field.velocity(icomp); - Kokkos::Experimental::where(mask, lhs).copy_to( - &curr_field.field_dot(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreAcceleration) { - for (int icomp = 0; icomp < components; ++icomp) { - typename ViewType::simd::datatype lhs; - Kokkos::Experimental::where(mask, lhs).copy_from( - &curr_field.field_dot_dot(iglob, icomp), tag_type()); - - lhs += point_field.acceleration(icomp); - Kokkos::Experimental::where(mask, lhs).copy_to( - &curr_field.field_dot_dot(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreMassMatrix) { - for (int icomp = 0; icomp < components; ++icomp) { - typename ViewType::simd::datatype lhs; - Kokkos::Experimental::where(mask, lhs).copy_from( - &curr_field.mass_inverse(iglob, icomp), tag_type()); - - lhs += point_field.mass_matrix(icomp); - Kokkos::Experimental::where(mask, lhs).copy_to( - &curr_field.mass_inverse(iglob, icomp), tag_type()); - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_add_on_host(const int iglob, const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - - const auto &curr_field = field.template get_field(); - for (int icomp = 0; icomp < ViewType::components; ++icomp) { - if constexpr (StoreDisplacement) { - curr_field.h_field(iglob, icomp) += point_field.displacement(icomp); - } - if constexpr (StoreVelocity) { - curr_field.h_field_dot(iglob, icomp) += point_field.velocity(icomp); - } - if constexpr (StoreAcceleration) { - curr_field.h_field_dot_dot(iglob, icomp) += - point_field.acceleration(icomp); - } - if constexpr (StoreMassMatrix) { - curr_field.h_mass_inverse(iglob, icomp) += point_field.mass_matrix(icomp); - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -inline void impl_add_on_host(const typename ViewType::simd::mask_type &mask, - const int *iglob, const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - const auto &curr_field = field.template get_field(); - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - if (!mask[lane]) { - continue; - } - - const int iglob_l = iglob[lane]; - - if constexpr (StoreDisplacement) { - for (int icomp = 0; icomp < components; ++icomp) { - curr_field.h_field(iglob_l, icomp) += - point_field.displacement(icomp)[lane]; - } - } - - if constexpr (StoreVelocity) { - for (int icomp = 0; icomp < components; ++icomp) { - curr_field.h_field_dot(iglob_l, icomp) += - point_field.velocity(icomp)[lane]; - } - } - - if constexpr (StoreAcceleration) { - for (int icomp = 0; icomp < components; ++icomp) { - curr_field.h_field_dot_dot(iglob_l, icomp) += - point_field.acceleration(icomp)[lane]; - } - } - - if constexpr (StoreMassMatrix) { - for (int icomp = 0; icomp < components; ++icomp) { - curr_field.h_mass_inverse(iglob_l, icomp) += - point_field.mass_matrix(icomp)[lane]; - } - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -inline void impl_add_on_host(const specfem::point::simd_assembly_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - using mask_type = typename ViewType::simd::mask_type; - using tag_type = typename ViewType::simd::tag_type; - - const int iglob = index.iglob; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const auto &curr_field = field.template get_field(); - if constexpr (StoreDisplacement) { - for (int icomp = 0; icomp < components; ++icomp) { - typename ViewType::simd::datatype lhs; - Kokkos::Experimental::where(mask, lhs).copy_from( - &curr_field.h_field(iglob, icomp), tag_type()); - - lhs += point_field.displacement(icomp); - Kokkos::Experimental::where(mask, lhs).copy_to( - &curr_field.h_field(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreVelocity) { - for (int icomp = 0; icomp < components; ++icomp) { - typename ViewType::simd::datatype lhs; - Kokkos::Experimental::where(mask, lhs).copy_from( - &curr_field.h_field_dot(iglob, icomp), tag_type()); - - lhs += point_field.velocity(icomp); - Kokkos::Experimental::where(mask, lhs).copy_to( - &curr_field.h_field_dot(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreAcceleration) { - for (int icomp = 0; icomp < components; ++icomp) { - typename ViewType::simd::datatype lhs; - Kokkos::Experimental::where(mask, lhs).copy_from( - &curr_field.h_field_dot_dot(iglob, icomp), tag_type()); - - lhs += point_field.acceleration(icomp); - Kokkos::Experimental::where(mask, lhs).copy_to( - &curr_field.h_field_dot_dot(iglob, icomp), tag_type()); - } - } - - if constexpr (StoreMassMatrix) { - for (int icomp = 0; icomp < components; ++icomp) { - typename ViewType::simd::datatype lhs; - Kokkos::Experimental::where(mask, lhs).copy_from( - &curr_field.h_mass_inverse(iglob, icomp), tag_type()); - - lhs += point_field.mass_matrix(icomp); - Kokkos::Experimental::where(mask, lhs).copy_to( - &curr_field.h_mass_inverse(iglob, icomp), tag_type()); - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_add_on_device(const specfem::point::index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = field.assembly_index_mapping( - field.index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); - - impl_add_on_device(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_add_on_device(const specfem::point::assembly_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = index.iglob; - - impl_add_on_device(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_add_on_device(const specfem::point::simd_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - int iglob[ViewType::simd::size()]; - - using mask_type = typename ViewType::simd::mask_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = - (index.mask(std::size_t(lane))) - ? field.assembly_index_mapping( - field.index_mapping(index.ispec + lane, index.iz, index.ix), - static_cast(MediumType)) - : field.nglob + 1; - } - - impl_add_on_device(mask, iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_add_on_host(const specfem::point::index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = field.h_assembly_index_mapping( - field.h_index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); - - impl_add_on_host(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_add_on_host(const specfem::point::assembly_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = index.iglob; - - impl_add_on_host(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -inline void impl_add_on_host( - const specfem::point::simd_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - int iglob[ViewType::simd::size()]; - - using mask_type = typename ViewType::simd::mask_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = - (index.mask(std::size_t(lane))) - ? field.h_index_mapping(index.ispec + lane, index.iz, index.ix) - : field.nglob + 1; - } - - impl_add_on_host(mask, iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_atomic_add_on_device(const int iglob, const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - - const auto &curr_field = field.template get_field(); - for (int icomp = 0; icomp < ViewType::components; ++icomp) { - if constexpr (StoreDisplacement) { - Kokkos::atomic_add(&curr_field.field(iglob, icomp), - point_field.displacement(icomp)); - } - if constexpr (StoreVelocity) { - Kokkos::atomic_add(&curr_field.field_dot(iglob, icomp), - point_field.velocity(icomp)); - } - if constexpr (StoreAcceleration) { - Kokkos::atomic_add(&curr_field.field_dot_dot(iglob, icomp), - point_field.acceleration(icomp)); - } - if constexpr (StoreMassMatrix) { - Kokkos::atomic_add(&curr_field.mass_inverse(iglob, icomp), - point_field.mass_matrix(icomp)); - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void -impl_atomic_add_on_device(const typename ViewType::simd::mask_type &mask, - const int *iglob, const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - const auto &curr_field = field.template get_field(); - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - if (!mask[lane]) { - continue; - } - - const int iglob_l = iglob[lane]; - - if constexpr (StoreDisplacement) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::atomic_add(&curr_field.field(iglob_l, icomp), - point_field.displacement(icomp)[lane]); - } - } - - if constexpr (StoreVelocity) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::atomic_add(&curr_field.field_dot(iglob_l, icomp), - point_field.velocity(icomp)[lane]); - } - } - - if constexpr (StoreAcceleration) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::atomic_add(&curr_field.field_dot_dot(iglob_l, icomp), - point_field.acceleration(icomp)[lane]); - } - } - - if constexpr (StoreMassMatrix) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::atomic_add(&curr_field.mass_inverse(iglob_l, icomp), - point_field.mass_matrix(icomp)[lane]); - } - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_atomic_add_on_host(const int iglob, const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - - const auto &curr_field = field.template get_field(); - for (int icomp = 0; icomp < ViewType::components; ++icomp) { - if constexpr (StoreDisplacement) { - Kokkos::atomic_add(&curr_field.h_field(iglob, icomp), - point_field.displacement(icomp)); - } - if constexpr (StoreVelocity) { - Kokkos::atomic_add(&curr_field.h_field_dot(iglob, icomp), - point_field.velocity(icomp)); - } - if constexpr (StoreAcceleration) { - Kokkos::atomic_add(&curr_field.h_field_dot_dot(iglob, icomp), - point_field.acceleration(icomp)); - } - if constexpr (StoreMassMatrix) { - Kokkos::atomic_add(&curr_field.h_mass_inverse(iglob, icomp), - point_field.mass_matrix(icomp)); - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -inline void impl_atomic_add_on_host(const typename ViewType::simd::mask_type &mask, - const int *iglob, const ViewType &point_field, - const WavefieldType &field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - const auto &curr_field = field.template get_field(); - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - if (!mask[lane]) { - continue; - } - - const int iglob_l = iglob[lane]; - - if constexpr (StoreDisplacement) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::atomic_add(&curr_field.h_field(iglob_l, icomp), - point_field.displacement(icomp)[lane]); - } - } - - if constexpr (StoreVelocity) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::atomic_add(&curr_field.h_field_dot(iglob_l, icomp), - point_field.velocity(icomp)[lane]); - } - } - - if constexpr (StoreAcceleration) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::atomic_add(&curr_field.h_field_dot_dot(iglob_l, icomp), - point_field.acceleration(icomp)[lane]); - } - } - - if constexpr (StoreMassMatrix) { - for (int icomp = 0; icomp < components; ++icomp) { - Kokkos::atomic_add(&curr_field.h_mass_inverse(iglob_l, icomp), - point_field.mass_matrix(icomp)[lane]); - } - } - } - - return; -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void impl_atomic_add_on_device( - const specfem::point::index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = field.assembly_index_mapping( - field.index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); - - impl_atomic_add_on_device(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -KOKKOS_FORCEINLINE_FUNCTION void impl_atomic_add_on_device( - const specfem::point::simd_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - int iglob[ViewType::simd::size()]; - - using mask_type = typename ViewType::simd::mask_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = - (index.mask(std::size_t(lane))) - ? field.assembly_index_mapping( - field.index_mapping(index.ispec + lane, index.iz, index.ix), - static_cast(MediumType)) - : field.nglob + 1; - } - - impl_atomic_add_on_device(mask, iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_atomic_add_on_host( - const specfem::point::index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - - const int iglob = field.h_assembly_index_mapping( - field.h_index_mapping(index.ispec, index.iz, index.ix), - static_cast(MediumType)); - - impl_atomic_add_on_host(iglob, point_field, field); -} - -template < - typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isPointFieldType && ViewType::simd::using_simd, int> = 0> -inline void impl_atomic_add_on_host( - const specfem::point::simd_index &index, - const ViewType &point_field, const WavefieldType &field) { - - constexpr static auto MediumType = ViewType::medium_tag; - int iglob[ViewType::simd::size()]; - - using mask_type = typename ViewType::simd::mask_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - iglob[lane] = - (index.mask(std::size_t(lane))) - ? field.h_index_mapping(index.ispec + lane, index.iz, index.ix) - : field.nglob + 1; - } - - impl_atomic_add_on_host(mask, iglob, point_field, field); -} - -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_load_on_device(const MemberType &team, const int ispec, +impl_load(const MemberType &team, const int ispec, const WavefieldType &field, ViewType &element_field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - constexpr static int NGLL = ViewType::ngll; - - static_assert(std::is_same_v, - "Calling team must have a device execution space"); - - const auto &curr_field = field.template get_field(); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, NGLL * NGLL), [&](const int &xz) { - int iz, ix; - sub2ind(xz, NGLL, iz, ix); - const int iglob = field.assembly_index_mapping( - field.index_mapping(ispec, iz, ix), static_cast(MediumType)); - - for (int icomp = 0; icomp < components; ++icomp) { - if constexpr (StoreDisplacement) { - element_field.displacement(iz, ix, icomp) = - curr_field.field(iglob, icomp); - } - if constexpr (StoreVelocity) { - element_field.velocity(iz, ix, icomp) = - curr_field.field_dot(iglob, icomp); - } - if constexpr (StoreAcceleration) { - element_field.acceleration(iz, ix, icomp) = - curr_field.field_dot_dot(iglob, icomp); - } - if constexpr (StoreMassMatrix) { - element_field.mass_matrix(iz, ix, icomp) = - curr_field.mass_inverse(iglob, icomp); - } - } - }); - - return; -} - -template < - typename MemberType, typename WavefieldType, typename ViewType, - typename std::enable_if_t< - ViewType::isElementFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_load_on_host(const MemberType &team, const int ispec, - const WavefieldType &field, ViewType &element_field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; constexpr static int NGLL = ViewType::ngll; - static_assert( - std::is_same_v, - "Calling team must have a host execution space"); + if constexpr (on_device) { + static_assert(std::is_same_v, + "Calling team must have a device execution space"); + } + else { + static_assert(std::is_same_v, + "Calling team must have a device execution space"); + } - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); Kokkos::parallel_for( Kokkos::TeamThreadRange(team, NGLL * NGLL), [&](const int &xz) { int iz, ix; sub2ind(xz, NGLL, iz, ix); - const int iglob = field.h_assembly_index_mapping( - field.h_index_mapping(ispec, iz, ix), static_cast(MediumType)); + const int iglob = field.template get_iglob(ispec, iz, ix, MediumTag); for (int icomp = 0; icomp < components; ++icomp) { if constexpr (StoreDisplacement) { element_field.displacement(iz, ix, icomp) = - curr_field.h_field(iglob, icomp); + curr_field.template get_field(iglob, icomp); } if constexpr (StoreVelocity) { element_field.velocity(iz, ix, icomp) = - curr_field.h_field_dot(iglob, icomp); + curr_field.template get_field_dot(iglob, icomp); } if constexpr (StoreAcceleration) { element_field.acceleration(iz, ix, icomp) = - curr_field.h_field_dot_dot(iglob, icomp); + curr_field.template get_field_dot_dot(iglob, icomp); } if constexpr (StoreMassMatrix) { element_field.mass_matrix(iz, ix, icomp) = - curr_field.h_mass_inverse(iglob, icomp); + curr_field.template get_mass_inverse(iglob, icomp); } } }); @@ -1640,20 +809,20 @@ inline void impl_load_on_host(const MemberType &team, const int ispec, return; } -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_load_on_device(const MemberType &team, const IteratorType &iterator, +impl_load(const MemberType &team, const IteratorType &iterator, const WavefieldType &field, ViewType &chunk_field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; constexpr static int NGLL = ViewType::ngll; @@ -1663,16 +832,21 @@ impl_load_on_device(const MemberType &team, const IteratorType &iterator, std::is_same_v, "Iterator and View must have the same simd type"); - static_assert(std::is_same_v, - "Calling team must have a device execution space"); + if constexpr (on_device) { + static_assert(std::is_same_v, + "Calling team must have a device execution space"); + } + else { + static_assert(std::is_same_v, + "Calling team must have a device execution space"); + } static_assert( Kokkos::SpaceAccessibility::accessible, "Calling team must have access to the memory space of the view"); - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); Kokkos::parallel_for( Kokkos::TeamThreadRange(team, iterator.chunk_size()), [&](const int &i) { const auto iterator_index = iterator(i); @@ -1681,25 +855,24 @@ impl_load_on_device(const MemberType &team, const IteratorType &iterator, const int iz = iterator_index.index.iz; const int ix = iterator_index.index.ix; - const int iglob = field.assembly_index_mapping( - field.index_mapping(ispec, iz, ix), static_cast(MediumType)); + const int iglob = field.template get_iglob(ispec, iz, ix, MediumTag); for (int icomp = 0; icomp < components; ++icomp) { if constexpr (StoreDisplacement) { chunk_field.displacement(ielement, iz, ix, icomp) = - curr_field.field(iglob, icomp); + curr_field.template get_field(iglob, icomp); } if constexpr (StoreVelocity) { chunk_field.velocity(ielement, iz, ix, icomp) = - curr_field.field_dot(iglob, icomp); + curr_field.template get_field_dot(iglob, icomp); } if constexpr (StoreAcceleration) { chunk_field.acceleration(ielement, iz, ix, icomp) = - curr_field.field_dot_dot(iglob, icomp); + curr_field.template get_field_dot_dot(iglob, icomp); } if constexpr (StoreMassMatrix) { chunk_field.mass_matrix(ielement, iz, ix, icomp) = - curr_field.mass_inverse(iglob, icomp); + curr_field.template get_mass_inverse(iglob, icomp); } } }); @@ -1707,20 +880,20 @@ impl_load_on_device(const MemberType &team, const IteratorType &iterator, return; } -template < +template = 0> KOKKOS_FORCEINLINE_FUNCTION void -impl_load_on_device(const MemberType &team, const IteratorType &iterator, +impl_load(const MemberType &team, const IteratorType &iterator, const WavefieldType &field, ViewType &chunk_field) { constexpr static bool StoreDisplacement = ViewType::store_displacement; constexpr static bool StoreVelocity = ViewType::store_velocity; constexpr static bool StoreAcceleration = ViewType::store_acceleration; constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; + constexpr static auto MediumTag = ViewType::medium_tag; constexpr static int components = ViewType::components; constexpr static int NGLL = ViewType::ngll; @@ -1734,11 +907,16 @@ impl_load_on_device(const MemberType &team, const IteratorType &iterator, typename ViewType::memory_space>::accessible, "Calling team must have access to the memory space of the view"); - static_assert(std::is_same_v, - "Calling team must have a device execution space"); + if constexpr (on_device) { + static_assert(std::is_same_v, + "Calling team must have a device execution space"); + } + else { + static_assert(std::is_same_v, + "Calling team must have a device execution space"); + } - const auto &curr_field = field.template get_field(); + const auto &curr_field = field.template get_field(); Kokkos::parallel_for( Kokkos::TeamThreadRange(team, iterator.chunk_size()), [&](const int &i) { const auto iterator_index = iterator(i); @@ -1752,162 +930,24 @@ impl_load_on_device(const MemberType &team, const IteratorType &iterator, continue; } - const int iglob = field.assembly_index_mapping( - field.index_mapping(ispec + lane, iz, ix), - static_cast(MediumType)); - - for (int icomp = 0; icomp < components; ++icomp) { - if constexpr (StoreDisplacement) { - chunk_field.displacement(ielement, iz, ix, icomp)[lane] = - curr_field.field(iglob, icomp); - } - if constexpr (StoreVelocity) { - chunk_field.velocity(ielement, iz, ix, icomp)[lane] = - curr_field.field_dot(iglob, icomp); - } - if constexpr (StoreAcceleration) { - chunk_field.acceleration(ielement, iz, ix, icomp)[lane] = - curr_field.field_dot_dot(iglob, icomp); - } - if constexpr (StoreMassMatrix) { - chunk_field.mass_matrix(ielement, iz, ix, icomp)[lane] = - curr_field.mass_inverse(iglob, icomp); - } - } - } - }); - - return; -} - -template < - typename MemberType, typename WavefieldType, typename IteratorType, - typename ViewType, - typename std::enable_if_t< - ViewType::isChunkFieldType && !ViewType::simd::using_simd, int> = 0> -inline void impl_load_on_host(const MemberType &team, const IteratorType &iterator, - const WavefieldType &field, ViewType &chunk_field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - constexpr static int NGLL = ViewType::ngll; - - static_assert( - std::is_same_v, - "Calling team must have a host execution space"); - - static_assert( - std::is_same_v, - "Iterator and View must have the same simd type"); - - static_assert( - Kokkos::SpaceAccessibility::accessible, - "Calling team must have access to the memory space of the view"); - - const auto &curr_field = field.template get_field(); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, iterator.chunk_size()), [&](const int &i) { - const auto iterator_index = iterator(i); - const int ielement = iterator_index.ielement; - const int ispec = iterator_index.index.ispec; - const int iz = iterator_index.index.iz; - const int ix = iterator_index.index.ix; - - const int iglob = field.h_assembly_index_mapping( - field.h_index_mapping(ispec, iz, ix), static_cast(MediumType)); - - for (int icomp = 0; icomp < components; ++icomp) { - if constexpr (StoreDisplacement) { - chunk_field.displacement(ielement, iz, ix, icomp) = - curr_field.h_field(iglob, icomp); - } - if constexpr (StoreVelocity) { - chunk_field.velocity(ielement, iz, ix, icomp) = - curr_field.h_field_dot(iglob, icomp); - } - if constexpr (StoreAcceleration) { - chunk_field.acceleration(ielement, iz, ix, icomp); - } - if constexpr (StoreMassMatrix) { - chunk_field.mass_matrix(ielement, iz, ix, icomp) = - curr_field.h_mass_inverse(iglob, icomp); - } - } - }); - - return; -} - -template < - typename MemberType, typename WavefieldType, typename IteratorType, - typename ViewType, - typename std::enable_if_t< - ViewType::isChunkFieldType && ViewType::simd::using_simd, int> = 0> -inline void impl_load_on_host(const MemberType &team, const IteratorType &iterator, - const WavefieldType &field, ViewType &chunk_field) { - - constexpr static bool StoreDisplacement = ViewType::store_displacement; - constexpr static bool StoreVelocity = ViewType::store_velocity; - constexpr static bool StoreAcceleration = ViewType::store_acceleration; - constexpr static bool StoreMassMatrix = ViewType::store_mass_matrix; - constexpr static auto MediumType = ViewType::medium_tag; - constexpr static int components = ViewType::components; - - constexpr static int NGLL = ViewType::ngll; - - static_assert( - std::is_same_v, - "Calling team must have a host execution space"); - - static_assert( - std::is_same_v, - "Iterator and View must have the same simd type"); - - static_assert( - Kokkos::SpaceAccessibility::accessible, - "Calling team must have access to the memory space of the view"); - - const auto &curr_field = field.template get_field(); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, iterator.chunk_size()), [&](const int &i) { - const auto iterator_index = iterator(i); - const int ielement = iterator_index.ielement; - const int ispec = iterator_index.index.ispec; - const int iz = iterator_index.index.iz; - const int ix = iterator_index.index.ix; - - for (int lane = 0; lane < ViewType::simd::size(); ++lane) { - if (!iterator_index.index.mask(lane)) { - continue; - } - - const int iglob = field.h_assembly_index_mapping( - field.h_index_mapping(ispec + lane, iz, ix), - static_cast(MediumType)); + const int iglob = field.template get_iglob(ispec + lane, iz, ix, MediumTag); for (int icomp = 0; icomp < components; ++icomp) { if constexpr (StoreDisplacement) { chunk_field.displacement(ielement, iz, ix, icomp)[lane] = - curr_field.h_field(iglob, icomp); + curr_field.template get_field(iglob, icomp); } if constexpr (StoreVelocity) { chunk_field.velocity(ielement, iz, ix, icomp)[lane] = - curr_field.h_field_dot(iglob, icomp); + curr_field.template get_field_dot(iglob, icomp); } if constexpr (StoreAcceleration) { chunk_field.acceleration(ielement, iz, ix, icomp)[lane] = - curr_field.h_field_dot_dot(iglob, icomp); + curr_field.template get_field_dot_dot(iglob, icomp); } if constexpr (StoreMassMatrix) { chunk_field.mass_matrix(ielement, iz, ix, icomp)[lane] = - curr_field.h_mass_inverse(iglob, icomp); + curr_field.template get_mass_inverse(iglob, icomp); } } } diff --git a/include/compute/fields/impl/field_impl.hpp b/include/compute/fields/impl/field_impl.hpp index 1da50d73..888398b5 100644 --- a/include/compute/fields/impl/field_impl.hpp +++ b/include/compute/fields/impl/field_impl.hpp @@ -1,5 +1,4 @@ -#ifndef _COMPUTE_FIELDS_IMPL_FIELD_IMPL_HPP_ -#define _COMPUTE_FIELDS_IMPL_FIELD_IMPL_HPP_ +#pragma once #include "compute/compute_mesh.hpp" #include "compute/element_types/element_types.hpp" @@ -37,6 +36,46 @@ class field_impl { specfem::kokkos::HostMirror2d h_field_dot_dot; specfem::kokkos::DeviceView2d mass_inverse; specfem::kokkos::HostMirror2d h_mass_inverse; + + template + KOKKOS_FORCEINLINE_FUNCTION constexpr type_real & + get_field(const int &iglob, const int &icomp) const { + if constexpr (on_device) { + return field(iglob, icomp); + } else { + return h_field(iglob, icomp); + } + } + + template + KOKKOS_FORCEINLINE_FUNCTION constexpr type_real & + get_field_dot(const int &iglob, const int &icomp) const { + if constexpr (on_device) { + return field_dot(iglob, icomp); + } else { + return h_field_dot(iglob, icomp); + } + } + + template + KOKKOS_FORCEINLINE_FUNCTION constexpr type_real & + get_field_dot_dot(const int &iglob, const int &icomp) const { + if constexpr (on_device) { + return field_dot_dot(iglob, icomp); + } else { + return h_field_dot_dot(iglob, icomp); + } + } + + template + KOKKOS_FORCEINLINE_FUNCTION constexpr type_real & + get_mass_inverse(const int &iglob, const int &icomp) const { + if constexpr (on_device) { + return mass_inverse(iglob, icomp); + } else { + return h_mass_inverse(iglob, icomp); + } + } }; } // namespace impl @@ -54,5 +93,3 @@ void deep_copy(impl::field_impl &dst, } // namespace compute } // namespace specfem - -#endif /* _COMPUTE_FIELDS_IMPL_FIELD_IMPL_HPP_ */ diff --git a/include/compute/fields/impl/field_impl.tpp b/include/compute/fields/impl/field_impl.tpp index ebb9db43..eb2391f2 100644 --- a/include/compute/fields/impl/field_impl.tpp +++ b/include/compute/fields/impl/field_impl.tpp @@ -1,5 +1,4 @@ -#ifndef _COMPUTE_FIELDS_IMPL_FIELD_IMPL_TPP_ -#define _COMPUTE_FIELDS_IMPL_FIELD_IMPL_TPP_ +#pragma once #include "compute/fields/impl/field_impl.hpp" #include "compute/element_types/element_types.hpp" @@ -111,7 +110,6 @@ void specfem::compute::impl::field_impl::sync_fields() } } -#endif /* _COMPUTE_FIELDS_IMPL_FIELD_IMPL_TPP_ */ // template // KOKKOS_INLINE_FUNCTION type_real &specfem::compute::(const int &iglob, diff --git a/include/compute/fields/simulation_field.hpp b/include/compute/fields/simulation_field.hpp index 2fe7fb9b..ba37c829 100644 --- a/include/compute/fields/simulation_field.hpp +++ b/include/compute/fields/simulation_field.hpp @@ -111,6 +111,23 @@ struct simulation_field { } } + /** + * @brief Returns the assembled index given element index. + * + */ + template + KOKKOS_INLINE_FUNCTION constexpr int + get_iglob(const int &ispec, const int &iz, const int &ix, + const specfem::element::medium_tag MediumTag) const { + if constexpr (on_device) { + return assembly_index_mapping(index_mapping(ispec, iz, ix), + static_cast(MediumTag)); + } else { + return h_assembly_index_mapping(h_index_mapping(ispec, iz, ix), + static_cast(MediumTag)); + } + } + int nglob = 0; ///< Number of global degrees of freedom int nspec; ///< Number of spectral elements int ngllz; ///< Number of quadrature points in z direction @@ -172,7 +189,7 @@ template (index, field, point_field); } /** @@ -195,7 +212,7 @@ template (index, field, point_field); } /** @@ -218,7 +235,7 @@ template (index, point_field, field); } /** @@ -240,7 +257,7 @@ template = 0> inline void store_on_host(const IndexType &index, const ViewType &point_field, const WavefieldContainer &field) { - impl_store_on_host(index, point_field, field); + impl_store(index, point_field, field); } /** @@ -263,7 +280,7 @@ template (index, point_field, field); } /** @@ -285,7 +302,7 @@ template = 0> inline void add_on_host(const IndexType &index, const ViewType &point_field, const WavefieldContainer &field) { - impl_add_on_host(index, point_field, field); + impl_add(index, point_field, field); } /** @@ -308,7 +325,7 @@ template (index, point_field, field); } /** @@ -331,7 +348,7 @@ template (index, point_field, field); } /** @@ -353,7 +370,7 @@ template (member, index, field, element_field); } /** @@ -375,7 +392,7 @@ template (member, index, field, element_field); } /** @@ -400,7 +417,7 @@ template (member, iterator, field, chunk_field); } /** @@ -425,7 +442,7 @@ template (member, iterator, field, chunk_field); } } // namespace compute diff --git a/include/compute/kernels/interface.hpp b/include/compute/kernels/interface.hpp index ec0e4570..47df4ddd 100644 --- a/include/compute/kernels/interface.hpp +++ b/include/compute/kernels/interface.hpp @@ -2,4 +2,3 @@ #include "kernels.hpp" #include "medium/kernels_container.hpp" -#include "medium/material_kernels.hpp" diff --git a/include/compute/kernels/kernels.hpp b/include/compute/kernels/kernels.hpp index bd97d676..03674c55 100644 --- a/include/compute/kernels/kernels.hpp +++ b/include/compute/kernels/kernels.hpp @@ -3,7 +3,7 @@ #include "compute/element_types/element_types.hpp" #include "compute/impl/value_containers.hpp" #include "enumerations/medium.hpp" -#include "medium/material_kernels.hpp" +#include "medium/kernels_container.hpp" #include "mesh/materials/materials.hpp" #include "point/coordinates.hpp" #include "point/kernels.hpp" @@ -17,7 +17,7 @@ namespace compute { * */ struct kernels - : public impl::value_containers { + : public impl::value_containers { public: /** * @name Constructors @@ -49,11 +49,12 @@ struct kernels * */ void copy_to_host() { - impl::value_containers::copy_to_host(); + impl::value_containers::copy_to_host(); } void copy_to_device() { - impl::value_containers::copy_to_device(); + impl::value_containers< + specfem::medium::kernels_container>::copy_to_device(); } }; @@ -89,7 +90,7 @@ KOKKOS_FUNCTION void load_on_device(const IndexType &index, IndexType l_index = index; l_index.ispec = ispec; - kernels.get_container().load_device_kernels( + kernels.get_container().load_device_values( l_index, point_kernels); return; @@ -122,7 +123,7 @@ void load_on_host(const IndexType &index, const kernels &kernels, IndexType l_index = index; l_index.ispec = ispec; - kernels.get_container().load_host_kernels( + kernels.get_container().load_host_values( l_index, point_kernels); return; @@ -155,7 +156,7 @@ void store_on_host(const IndexType &index, const PointKernelType &point_kernels, IndexType l_index = index; l_index.ispec = ispec; - kernels.get_container().update_kernels_on_host( + kernels.get_container().store_host_values( l_index, point_kernels); return; @@ -189,7 +190,7 @@ KOKKOS_FUNCTION void store_on_device(const IndexType &index, IndexType l_index = index; l_index.ispec = ispec; - kernels.get_container().update_kernels_on_device( + kernels.get_container().store_device_values( l_index, point_kernels); return; @@ -225,7 +226,7 @@ KOKKOS_FUNCTION void add_on_device(const IndexType &index, IndexType l_index = index; l_index.ispec = ispec; - kernels.get_container().add_kernels_on_device( + kernels.get_container().add_device_values( l_index, point_kernels); return; @@ -259,7 +260,7 @@ void add_on_host(const IndexType &index, const PointKernelType &point_kernels, IndexType l_index = index; l_index.ispec = ispec; - kernels.get_container().add_kernels_on_host( + kernels.get_container().add_host_values( l_index, point_kernels); return; diff --git a/include/compute/properties/interface.hpp b/include/compute/properties/interface.hpp index d7ae8a21..5129b34a 100644 --- a/include/compute/properties/interface.hpp +++ b/include/compute/properties/interface.hpp @@ -1,5 +1,4 @@ #pragma once -#include "medium/material_properties.hpp" #include "medium/properties_container.hpp" #include "properties.hpp" diff --git a/include/compute/properties/properties.hpp b/include/compute/properties/properties.hpp index 2addf56a..2fc86b56 100644 --- a/include/compute/properties/properties.hpp +++ b/include/compute/properties/properties.hpp @@ -6,7 +6,6 @@ #include "kokkos_abstractions.h" #include "macros.hpp" #include "medium/material.hpp" -#include "medium/material_properties.hpp" #include "medium/properties_container.hpp" #include "point/coordinates.hpp" #include "specfem_setup.hpp" @@ -23,7 +22,7 @@ namespace compute { * */ struct properties - : public impl::value_containers { + : public impl::value_containers { /** * @name Constructors */ @@ -59,12 +58,12 @@ struct properties */ void copy_to_host() { impl::value_containers< - specfem::medium::material_properties>::copy_to_host(); + specfem::medium::properties_container>::copy_to_host(); } void copy_to_device() { impl::value_containers< - specfem::medium::material_properties>::copy_to_device(); + specfem::medium::properties_container>::copy_to_device(); } }; @@ -109,7 +108,7 @@ load_on_device(const IndexType &lcoord, static_assert(DimensionType == specfem::dimension::type::dim2, "Only 2D properties are supported"); - properties.get_container().load_device_properties( + properties.get_container().load_device_values( l_index, point_properties); } @@ -149,7 +148,7 @@ void load_on_host(const IndexType &lcoord, static_assert(DimensionType == specfem::dimension::type::dim2, "Only 2D properties are supported"); - properties.get_container().load_host_properties( + properties.get_container().load_host_values( l_index, point_properties); } @@ -188,8 +187,8 @@ void store_on_host(const IndexType &lcoord, static_assert(DimensionType == specfem::dimension::type::dim2, "Only 2D properties are supported"); - properties.get_container().assign(l_index, - point_properties); + properties.get_container().store_host_values( + l_index, point_properties); } } // namespace compute } // namespace specfem diff --git a/include/medium/dim2/acoustic/isotropic/frechet_derivative.hpp b/include/medium/dim2/acoustic/isotropic/frechet_derivative.hpp index 25f48a34..59b2f215 100644 --- a/include/medium/dim2/acoustic/isotropic/frechet_derivative.hpp +++ b/include/medium/dim2/acoustic/isotropic/frechet_derivative.hpp @@ -30,11 +30,11 @@ impl_compute_frechet_derivatives( const auto rho_kl = (adjoint_derivatives.du(0, 0) * backward_derivatives.du(0, 0) + adjoint_derivatives.du(1, 0) * backward_derivatives.du(1, 0)) * - properties.rho_inverse * dt; + properties.rho_inverse() * dt; const auto kappa_kl = specfem::algorithms::dot(adjoint_field.acceleration, backward_field.displacement) * - static_cast(1.0) / properties.kappa * dt; + static_cast(1.0) / properties.kappa() * dt; return { rho_kl, kappa_kl }; } diff --git a/include/medium/dim2/acoustic/isotropic/kernels_container.hpp b/include/medium/dim2/acoustic/isotropic/kernels_container.hpp index e8b64e67..a0aee70b 100644 --- a/include/medium/dim2/acoustic/isotropic/kernels_container.hpp +++ b/include/medium/dim2/acoustic/isotropic/kernels_container.hpp @@ -1,412 +1,28 @@ #pragma once -#include "enumerations/medium.hpp" -#include "kokkos_abstractions.h" -#include "point/coordinates.hpp" -#include "point/kernels.hpp" -#include +#include "medium/properties_container.hpp" +#include "point/interface.hpp" +#include namespace specfem { namespace medium { template <> -class kernels_container { -public: - constexpr static auto value_type = specfem::element::medium_tag::acoustic; - constexpr static auto property_type = - specfem::element::property_tag::isotropic; - int nspec; - int ngllz; - int ngllx; - - using ViewType = Kokkos::View; - ViewType rho; - ViewType::HostMirror h_rho; - ViewType kappa; - ViewType::HostMirror h_kappa; - ViewType rho_prime; - ViewType::HostMirror h_rho_prime; - ViewType alpha; - ViewType::HostMirror h_alpha; - - kernels_container() = default; - - kernels_container(const int nspec, const int ngllz, const int ngllx) - : nspec(nspec), ngllz(ngllz), ngllx(ngllx), - rho("specfem::medium::acoustic::rho", nspec, ngllz, ngllx), - kappa("specfem::medium::acoustic::kappa", nspec, ngllz, ngllx), - rho_prime("specfem::medium::acoustic::rho_prime", nspec, ngllz, ngllx), - alpha("specfem::medium::acoustic::alpha", nspec, ngllz, ngllx), - h_rho(Kokkos::create_mirror_view(rho)), - h_kappa(Kokkos::create_mirror_view(kappa)), - h_rho_prime(Kokkos::create_mirror_view(rho_prime)), - h_alpha(Kokkos::create_mirror_view(alpha)) { - - initialize(); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void load_device_kernels( - const specfem::point::index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - kernels.rho = rho(ispec, iz, ix); - kernels.kappa = kappa(ispec, iz, ix); - kernels.rhop = rho_prime(ispec, iz, ix); - kernels.alpha = alpha(ispec, iz, ix); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void load_device_kernels( - const specfem::point::simd_index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_from(&rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.kappa) - .copy_from(&kappa(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.rhop) - .copy_from(&rho_prime(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.alpha) - .copy_from(&alpha(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void load_host_kernels( - const specfem::point::index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - kernels.rho = h_rho(ispec, iz, ix); - kernels.kappa = h_kappa(ispec, iz, ix); - kernels.rhop = h_rho_prime(ispec, iz, ix); - kernels.alpha = h_alpha(ispec, iz, ix); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void load_host_kernels( - const specfem::point::simd_index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_from(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.kappa) - .copy_from(&h_kappa(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.rhop) - .copy_from(&h_rho_prime(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.alpha) - .copy_from(&h_alpha(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void update_kernels_on_device( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - rho(ispec, iz, ix) = kernels.rho; - kappa(ispec, iz, ix) = kernels.kappa; - rho_prime(ispec, iz, ix) = kernels.rhop; - alpha(ispec, iz, ix) = kernels.alpha; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void update_kernels_on_device( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_to(&rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.kappa) - .copy_to(&kappa(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.rhop) - .copy_to(&rho_prime(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.alpha) - .copy_to(&alpha(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void update_kernels_on_host( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho(ispec, iz, ix) = kernels.rho; - h_kappa(ispec, iz, ix) = kernels.kappa; - h_rho_prime(ispec, iz, ix) = kernels.rhop; - h_alpha(ispec, iz, ix) = kernels.alpha; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void update_kernels_on_host( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_to(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.kappa) - .copy_to(&h_kappa(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.rhop) - .copy_to(&h_rho_prime(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.alpha) - .copy_to(&h_alpha(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void add_kernels_on_device( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - rho(ispec, iz, ix) += kernels.rho; - kappa(ispec, iz, ix) += kernels.kappa; - rho_prime(ispec, iz, ix) += kernels.rhop; - alpha(ispec, iz, ix) += kernels.alpha; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void add_kernels_on_device( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - simd_type lhs; - - Kokkos::Experimental::where(mask, lhs).copy_from(&rho(ispec, iz, ix), - tag_type()); - lhs += kernels.rho; - Kokkos::Experimental::where(mask, lhs).copy_to(&rho(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&kappa(ispec, iz, ix), - tag_type()); - lhs += kernels.kappa; - Kokkos::Experimental::where(mask, lhs).copy_to(&kappa(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&rho_prime(ispec, iz, ix), - tag_type()); - lhs += kernels.rhop; - Kokkos::Experimental::where(mask, lhs).copy_to(&rho_prime(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&alpha(ispec, iz, ix), - tag_type()); - lhs += kernels.alpha; - Kokkos::Experimental::where(mask, lhs).copy_to(&alpha(ispec, iz, ix), - tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void add_kernels_on_host( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho(ispec, iz, ix) += kernels.rho; - h_kappa(ispec, iz, ix) += kernels.kappa; - h_rho_prime(ispec, iz, ix) += kernels.rhop; - h_alpha(ispec, iz, ix) += kernels.alpha; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void add_kernels_on_host( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - simd_type lhs; - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_rho(ispec, iz, ix), - tag_type()); - lhs += kernels.rho; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_rho(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_kappa(ispec, iz, ix), - tag_type()); - lhs += kernels.kappa; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_kappa(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from( - &h_rho_prime(ispec, iz, ix), tag_type()); - lhs += kernels.rhop; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_rho_prime(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_alpha(ispec, iz, ix), - tag_type()); - lhs += kernels.alpha; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_alpha(ispec, iz, ix), - tag_type()); - } - - void copy_to_host() { - Kokkos::deep_copy(h_rho, rho); - Kokkos::deep_copy(h_kappa, kappa); - Kokkos::deep_copy(h_rho_prime, rho_prime); - Kokkos::deep_copy(h_alpha, alpha); - } - - void copy_to_device() { - Kokkos::deep_copy(rho, h_rho); - Kokkos::deep_copy(kappa, h_kappa); - Kokkos::deep_copy(rho_prime, h_rho_prime); - Kokkos::deep_copy(alpha, h_alpha); - } - - void initialize() { - Kokkos::parallel_for( - "specfem::medium::acoustic::initialize", - Kokkos::MDRangePolicy >({ 0, 0, 0 }, - { nspec, ngllz, ngllx }), - KOKKOS_CLASS_LAMBDA(const int ispec, const int iz, const int ix) { - this->rho(ispec, iz, ix) = 0.0; - this->kappa(ispec, iz, ix) = 0.0; - this->rho_prime(ispec, iz, ix) = 0.0; - this->alpha(ispec, iz, ix) = 0.0; - }); - } +struct kernels_container + : public impl_kernels_container { + using base_type = + impl_kernels_container; + using base_type::base_type; + constexpr static int _counter = __COUNTER__; + + DEFINE_MEDIUM_VIEW(rho) + DEFINE_MEDIUM_VIEW(kappa) + DEFINE_MEDIUM_VIEW(rhop) + DEFINE_MEDIUM_VIEW(alpha) }; } // namespace medium diff --git a/include/medium/dim2/acoustic/isotropic/mass_matrix.tpp b/include/medium/dim2/acoustic/isotropic/mass_matrix.tpp index 4533bf91..7af19384 100644 --- a/include/medium/dim2/acoustic/isotropic/mass_matrix.tpp +++ b/include/medium/dim2/acoustic/isotropic/mass_matrix.tpp @@ -14,5 +14,5 @@ specfem::medium::impl_mass_matrix_component( specfem::dimension::type::dim2, true, UseSIMD> &partial_derivatives) { return specfem::datatype::ScalarPointViewType( - partial_derivatives.jacobian / properties.kappa); + partial_derivatives.jacobian / properties.kappa()); } diff --git a/include/medium/dim2/acoustic/isotropic/properties.hpp b/include/medium/dim2/acoustic/isotropic/material.hpp similarity index 83% rename from include/medium/dim2/acoustic/isotropic/properties.hpp rename to include/medium/dim2/acoustic/isotropic/material.hpp index de132117..4249d721 100644 --- a/include/medium/dim2/acoustic/isotropic/properties.hpp +++ b/include/medium/dim2/acoustic/isotropic/material.hpp @@ -1,7 +1,6 @@ #pragma once #include "enumerations/medium.hpp" -#include "medium/properties.hpp" #include "point/properties.hpp" #include "specfem_setup.hpp" #include @@ -17,8 +16,8 @@ namespace medium { * */ template <> -class properties { +class impl_material { public: constexpr static auto dimension = @@ -41,9 +40,9 @@ class propertiesQkappa <= 0.0 || this->Qmu <= 0.0) { @@ -59,8 +58,8 @@ class properties + bool operator==(const impl_material &other) const { return (std::abs(this->density - other.density) < 1e-6 && @@ -76,8 +75,8 @@ class properties + bool operator!=(const impl_material &other) const { return !(*this == other); } @@ -86,10 +85,10 @@ class properties struct properties_container { - - constexpr static auto dimension = specfem::dimension::type::dim2; - constexpr static auto value_type = specfem::element::medium_tag::acoustic; - constexpr static auto property_type = - specfem::element::property_tag::isotropic; - - using ViewType = typename Kokkos::View; - - int nspec; ///< total number of acoustic spectral elements - int ngllz; ///< number of quadrature points in z dimension - int ngllx; ///< number of quadrature points in x dimension - ViewType rho_inverse; - ViewType::HostMirror h_rho_inverse; - ViewType kappa; - ViewType::HostMirror h_kappa; - - properties_container() = default; - - properties_container(const int nspec, const int ngllz, const int ngllx) - : nspec(nspec), ngllz(ngllz), ngllx(ngllx), - rho_inverse("specfem::compute::properties::rho_inverse", nspec, ngllz, - ngllx), - h_rho_inverse(Kokkos::create_mirror_view(rho_inverse)), - kappa("specfem::compute::properties::kappa", nspec, ngllz, ngllx), - h_kappa(Kokkos::create_mirror_view(kappa)) {} - - template < - typename PointProperties, - typename std::enable_if_t = 0> - KOKKOS_FORCEINLINE_FUNCTION void - load_device_properties(const specfem::point::index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - property.rho_inverse = rho_inverse(ispec, iz, ix); - property.kappa = kappa(ispec, iz, ix); - property.kappa_inverse = static_cast(1.0) / property.kappa; - property.rho_vpinverse = - sqrt(property.rho_inverse * property.kappa_inverse); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - KOKKOS_FORCEINLINE_FUNCTION void - load_device_properties(const specfem::point::simd_index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho_inverse) - .copy_from(&rho_inverse(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.kappa) - .copy_from(&kappa(ispec, iz, ix), tag_type()); - - property.kappa_inverse = static_cast(1.0) / property.kappa; - property.rho_vpinverse = - Kokkos::sqrt(property.rho_inverse * property.kappa_inverse); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void - load_host_properties(const specfem::point::index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - property.rho_inverse = h_rho_inverse(ispec, iz, ix); - property.kappa = h_kappa(ispec, iz, ix); - property.kappa_inverse = static_cast(1.0) / property.kappa; - property.rho_vpinverse = - sqrt(property.rho_inverse * property.kappa_inverse); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void - load_host_properties(const specfem::point::simd_index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho_inverse) - .copy_from(&h_rho_inverse(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.kappa) - .copy_from(&h_kappa(ispec, iz, ix), tag_type()); - - property.kappa_inverse = static_cast(1.0) / property.kappa; - property.rho_vpinverse = - Kokkos::sqrt(property.rho_inverse * property.kappa_inverse); - } - - void copy_to_device() { - Kokkos::deep_copy(rho_inverse, h_rho_inverse); - Kokkos::deep_copy(kappa, h_kappa); - } - - void copy_to_host() { - Kokkos::deep_copy(h_rho_inverse, rho_inverse); - Kokkos::deep_copy(h_kappa, kappa); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void assign(const specfem::point::index &index, - const PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho_inverse(ispec, iz, ix) = property.rho_inverse; - h_kappa(ispec, iz, ix) = property.kappa; - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void assign(const specfem::point::simd_index &index, - const PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho_inverse) - .copy_to(&h_rho_inverse(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.kappa) - .copy_to(&h_kappa(ispec, iz, ix), tag_type()); - } + specfem::element::property_tag::isotropic> + : public impl_properties_container< + specfem::element::medium_tag::acoustic, + specfem::element::property_tag::isotropic, 2> { + using base_type = + impl_properties_container; + using base_type::base_type; + constexpr static int _counter = __COUNTER__; + + DEFINE_MEDIUM_VIEW(rho_inverse) + DEFINE_MEDIUM_VIEW(kappa) }; } // namespace medium diff --git a/include/medium/dim2/acoustic/isotropic/source.hpp b/include/medium/dim2/acoustic/isotropic/source.hpp index 6ba010ad..b5e7e43b 100644 --- a/include/medium/dim2/acoustic/isotropic/source.hpp +++ b/include/medium/dim2/acoustic/isotropic/source.hpp @@ -27,7 +27,7 @@ KOKKOS_INLINE_FUNCTION auto impl_compute_source_contribution( result.acceleration(0) = point_source.stf(0) * point_source.lagrange_interpolant(0) / - point_properties.kappa; + point_properties.kappa(); return result; } diff --git a/include/medium/dim2/acoustic/isotropic/stress.hpp b/include/medium/dim2/acoustic/isotropic/stress.hpp index 895a43e5..38f2c5d6 100644 --- a/include/medium/dim2/acoustic/isotropic/stress.hpp +++ b/include/medium/dim2/acoustic/isotropic/stress.hpp @@ -26,8 +26,8 @@ impl_compute_stress( specfem::datatype::VectorPointViewType T; - T(0, 0) = properties.rho_inverse * du(0, 0); - T(1, 0) = properties.rho_inverse * du(1, 0); + T(0, 0) = properties.rho_inverse() * du(0, 0); + T(1, 0) = properties.rho_inverse() * du(1, 0); return { T }; } diff --git a/include/medium/dim2/elastic/anisotropic/frechet_derivative.hpp b/include/medium/dim2/elastic/anisotropic/frechet_derivative.hpp index 2f83ec85..97ee5a04 100644 --- a/include/medium/dim2/elastic/anisotropic/frechet_derivative.hpp +++ b/include/medium/dim2/elastic/anisotropic/frechet_derivative.hpp @@ -79,13 +79,13 @@ impl_compute_frechet_derivatives( // Computing the rest of the integral. // rho from equation 14 - rho_kl = static_cast(-1.0) * properties.rho * dt * rho_kl; - c11_kl = static_cast(-1.0) * c11_kl * properties.c11 * dt; - c13_kl = static_cast(-1.0) * c13_kl * properties.c13 * dt; - c15_kl = static_cast(-1.0) * c15_kl * properties.c15 * dt; - c33_kl = static_cast(-1.0) * c33_kl * properties.c33 * dt; - c35_kl = static_cast(-1.0) * c35_kl * properties.c35 * dt; - c55_kl = static_cast(-1.0) * c55_kl * properties.c55 * dt; + rho_kl = static_cast(-1.0) * properties.rho() * dt * rho_kl; + c11_kl = static_cast(-1.0) * c11_kl * properties.c11() * dt; + c13_kl = static_cast(-1.0) * c13_kl * properties.c13() * dt; + c15_kl = static_cast(-1.0) * c15_kl * properties.c15() * dt; + c33_kl = static_cast(-1.0) * c33_kl * properties.c33() * dt; + c35_kl = static_cast(-1.0) * c35_kl * properties.c35() * dt; + c55_kl = static_cast(-1.0) * c55_kl * properties.c55() * dt; return { rho_kl, c11_kl, c13_kl, c15_kl, c33_kl, c35_kl, c55_kl }; diff --git a/include/medium/dim2/elastic/anisotropic/kernels_container.hpp b/include/medium/dim2/elastic/anisotropic/kernels_container.hpp index b03e170c..90c86d75 100644 --- a/include/medium/dim2/elastic/anisotropic/kernels_container.hpp +++ b/include/medium/dim2/elastic/anisotropic/kernels_container.hpp @@ -1,510 +1,31 @@ #pragma once -#include "enumerations/medium.hpp" -#include "kokkos_abstractions.h" -#include "point/coordinates.hpp" -#include "point/kernels.hpp" -#include + +#include "medium/properties_container.hpp" +#include "point/interface.hpp" +#include namespace specfem { namespace medium { template <> -class kernels_container { -public: - constexpr static auto value_type = specfem::element::medium_tag::elastic; - constexpr static auto property_type = - specfem::element::property_tag::anisotropic; - int nspec; - int ngllz; - int ngllx; - - using ViewType = Kokkos::View; - - ViewType rho; - ViewType::HostMirror h_rho; - ViewType c11; - ViewType::HostMirror h_c11; - ViewType c13; - ViewType::HostMirror h_c13; - ViewType c15; - ViewType::HostMirror h_c15; - ViewType c33; - ViewType::HostMirror h_c33; - ViewType c35; - ViewType::HostMirror h_c35; - ViewType c55; - ViewType::HostMirror h_c55; - - kernels_container() = default; - - kernels_container(const int nspec, const int ngllz, const int ngllx) - : nspec(nspec), ngllz(ngllz), ngllx(ngllx), - rho("specfem::compute::properties::rho", nspec, ngllz, ngllx), - h_rho(Kokkos::create_mirror_view(rho)), - c11("specfem::compute::properties::c11", nspec, ngllz, ngllx), - h_c11(Kokkos::create_mirror_view(c11)), - c13("specfem::compute::properties::c13", nspec, ngllz, ngllx), - h_c13(Kokkos::create_mirror_view(c13)), - c15("specfem::compute::properties::c15", nspec, ngllz, ngllx), - h_c15(Kokkos::create_mirror_view(c15)), - c33("specfem::compute::properties::c33", nspec, ngllz, ngllx), - h_c33(Kokkos::create_mirror_view(c33)), - c35("specfem::compute::properties::c35", nspec, ngllz, ngllx), - h_c35(Kokkos::create_mirror_view(c35)), - c55("specfem::compute::properties::c55", nspec, ngllz, ngllx), - h_c55(Kokkos::create_mirror_view(c55)) { - - initialize(); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void load_device_kernels( - const specfem::point::index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - kernels.rho = rho(ispec, iz, ix); - kernels.c11 = c11(ispec, iz, ix); - kernels.c13 = c13(ispec, iz, ix); - kernels.c15 = c15(ispec, iz, ix); - kernels.c33 = c33(ispec, iz, ix); - kernels.c35 = c35(ispec, iz, ix); - kernels.c55 = c55(ispec, iz, ix); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void load_device_kernels( - const specfem::point::simd_index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_from(&rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c11) - .copy_from(&c11(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c13) - .copy_from(&c13(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c15) - .copy_from(&c15(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c33) - .copy_from(&c33(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c35) - .copy_from(&c35(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c55) - .copy_from(&c55(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void - load_host_kernels(specfem::point::index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - kernels.rho = h_rho(ispec, iz, ix); - kernels.c11 = h_c11(ispec, iz, ix); - kernels.c13 = h_c13(ispec, iz, ix); - kernels.c15 = h_c15(ispec, iz, ix); - kernels.c33 = h_c33(ispec, iz, ix); - kernels.c35 = h_c35(ispec, iz, ix); - kernels.c55 = h_c55(ispec, iz, ix); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void load_host_kernels( - specfem::point::simd_index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_from(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c11) - .copy_from(&h_c11(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c13) - .copy_from(&h_c13(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c15) - .copy_from(&h_c15(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c33) - .copy_from(&h_c33(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c35) - .copy_from(&h_c35(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c55) - .copy_from(&h_c55(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void update_kernels_on_device( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - rho(ispec, iz, ix) = kernels.rho; - c11(ispec, iz, ix) = kernels.c11; - c13(ispec, iz, ix) = kernels.c13; - c15(ispec, iz, ix) = kernels.c15; - c33(ispec, iz, ix) = kernels.c33; - c35(ispec, iz, ix) = kernels.c35; - c55(ispec, iz, ix) = kernels.c55; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void update_kernels_on_device( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_to(&rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c11) - .copy_to(&c11(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c13) - .copy_to(&c13(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c15) - .copy_to(&c15(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c33) - .copy_to(&c33(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c35) - .copy_to(&c35(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c55) - .copy_to(&c55(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void update_kernels_on_host( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho(ispec, iz, ix) = kernels.rho; - h_c11(ispec, iz, ix) = kernels.c11; - h_c13(ispec, iz, ix) = kernels.c13; - h_c15(ispec, iz, ix) = kernels.c15; - h_c33(ispec, iz, ix) = kernels.c33; - h_c35(ispec, iz, ix) = kernels.c35; - h_c55(ispec, iz, ix) = kernels.c55; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void update_kernels_on_host( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - Kokkos::Experimental::where(mask, kernels.rho) - .copy_to(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c11) - .copy_to(&h_c11(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c13) - .copy_to(&h_c13(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c15) - .copy_to(&h_c15(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c33) - .copy_to(&h_c33(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c35) - .copy_to(&h_c35(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.c55) - .copy_to(&h_c55(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void add_kernels_on_device( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - rho(ispec, iz, ix) += kernels.rho; - c11(ispec, iz, ix) += kernels.c11; - c13(ispec, iz, ix) += kernels.c13; - c15(ispec, iz, ix) += kernels.c15; - c33(ispec, iz, ix) += kernels.c33; - c35(ispec, iz, ix) += kernels.c35; - c55(ispec, iz, ix) += kernels.c55; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void add_kernels_on_device( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - simd_type lhs; - - Kokkos::Experimental::where(mask, lhs).copy_from(&rho(ispec, iz, ix), - tag_type()); - lhs += kernels.rho; - Kokkos::Experimental::where(mask, lhs).copy_to(&rho(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&c11(ispec, iz, ix), - tag_type()); - lhs += kernels.c11; - Kokkos::Experimental::where(mask, lhs).copy_to(&c11(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&c13(ispec, iz, ix), - tag_type()); - lhs += kernels.c13; - Kokkos::Experimental::where(mask, lhs).copy_to(&c13(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&c15(ispec, iz, ix), - tag_type()); - lhs += kernels.c15; - Kokkos::Experimental::where(mask, lhs).copy_to(&c15(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&c33(ispec, iz, ix), - tag_type()); - lhs += kernels.c33; - Kokkos::Experimental::where(mask, lhs).copy_to(&c33(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&c35(ispec, iz, ix), - tag_type()); - lhs += kernels.c35; - Kokkos::Experimental::where(mask, lhs).copy_to(&c35(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&c55(ispec, iz, ix), - tag_type()); - lhs += kernels.c55; - Kokkos::Experimental::where(mask, lhs).copy_to(&c55(ispec, iz, ix), - tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void add_kernels_on_host( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho(ispec, iz, ix) += kernels.rho; - h_c11(ispec, iz, ix) += kernels.c11; - h_c13(ispec, iz, ix) += kernels.c13; - h_c15(ispec, iz, ix) += kernels.c15; - h_c33(ispec, iz, ix) += kernels.c33; - h_c35(ispec, iz, ix) += kernels.c35; - h_c55(ispec, iz, ix) += kernels.c55; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void add_kernels_on_host( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - simd_type lhs; - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_rho(ispec, iz, ix), - tag_type()); - lhs += kernels.rho; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_rho(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_c11(ispec, iz, ix), - tag_type()); - lhs += kernels.c11; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_c11(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_c13(ispec, iz, ix), - tag_type()); - lhs += kernels.c13; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_c13(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_c15(ispec, iz, ix), - tag_type()); - lhs += kernels.c15; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_c15(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_c33(ispec, iz, ix), - tag_type()); - lhs += kernels.c33; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_c33(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_c35(ispec, iz, ix), - tag_type()); - lhs += kernels.c35; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_c35(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_c55(ispec, iz, ix), - tag_type()); - lhs += kernels.c55; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_c55(ispec, iz, ix), - tag_type()); - } - - void copy_to_host() { - Kokkos::deep_copy(h_rho, rho); - Kokkos::deep_copy(h_c11, c11); - Kokkos::deep_copy(h_c13, c13); - Kokkos::deep_copy(h_c15, c15); - Kokkos::deep_copy(h_c33, c33); - Kokkos::deep_copy(h_c35, c35); - Kokkos::deep_copy(h_c55, c55); - } - - void copy_to_device() { - Kokkos::deep_copy(rho, h_rho); - Kokkos::deep_copy(c11, h_c11); - Kokkos::deep_copy(c13, h_c13); - Kokkos::deep_copy(c15, h_c15); - Kokkos::deep_copy(c33, h_c33); - Kokkos::deep_copy(c35, h_c35); - Kokkos::deep_copy(c55, h_c55); - } - - void initialize() { - Kokkos::parallel_for( - "specfem::medium::elastic::anisotropic::initialize", - Kokkos::MDRangePolicy >({ 0, 0, 0 }, - { nspec, ngllz, ngllx }), - KOKKOS_CLASS_LAMBDA(const int ispec, const int iz, const int ix) { - this->rho(ispec, iz, ix) = 0.0; - this->c11(ispec, iz, ix) = 0.0; - this->c13(ispec, iz, ix) = 0.0; - this->c15(ispec, iz, ix) = 0.0; - this->c33(ispec, iz, ix) = 0.0; - this->c35(ispec, iz, ix) = 0.0; - this->c55(ispec, iz, ix) = 0.0; - }); - } +struct kernels_container + : public impl_kernels_container { + using base_type = + impl_kernels_container; + using base_type::base_type; + constexpr static int _counter = __COUNTER__; + + DEFINE_MEDIUM_VIEW(rho) + DEFINE_MEDIUM_VIEW(c11) + DEFINE_MEDIUM_VIEW(c13) + DEFINE_MEDIUM_VIEW(c15) + DEFINE_MEDIUM_VIEW(c33) + DEFINE_MEDIUM_VIEW(c35) + DEFINE_MEDIUM_VIEW(c55) }; } // namespace medium diff --git a/include/medium/dim2/elastic/anisotropic/properties.hpp b/include/medium/dim2/elastic/anisotropic/material.hpp similarity index 83% rename from include/medium/dim2/elastic/anisotropic/properties.hpp rename to include/medium/dim2/elastic/anisotropic/material.hpp index ac3c8eae..1472157e 100644 --- a/include/medium/dim2/elastic/anisotropic/properties.hpp +++ b/include/medium/dim2/elastic/anisotropic/material.hpp @@ -1,7 +1,6 @@ #pragma once #include "enumerations/specfem_enums.hpp" -#include "medium/properties.hpp" #include "point/properties.hpp" #include "specfem_setup.hpp" #include @@ -17,8 +16,8 @@ namespace medium { * */ template <> -class properties { +class impl_material { public: constexpr static auto dimension = specfem::dimension::type::dim2; ///< Dimension of the material @@ -46,11 +45,12 @@ class properties - &other) const { + bool operator==( + const impl_material &other) + const { return (std::abs(this->density - other.density) < 1e-6 && std::abs(this->c11 - other.c11) < 1e-6 && std::abs(this->c13 - other.c13) < 1e-6 && @@ -100,9 +101,10 @@ class properties - &other) const { + bool operator!=( + const impl_material &other) + const { return !(*this == other); } diff --git a/include/medium/dim2/elastic/anisotropic/properties_container.hpp b/include/medium/dim2/elastic/anisotropic/properties_container.hpp index ca78ed05..2cf27a5f 100644 --- a/include/medium/dim2/elastic/anisotropic/properties_container.hpp +++ b/include/medium/dim2/elastic/anisotropic/properties_container.hpp @@ -9,327 +9,27 @@ namespace medium { template <> struct properties_container { - - constexpr static auto dimension = specfem::dimension::type::dim2; - constexpr static auto value_type = specfem::element::medium_tag::elastic; - constexpr static auto property_type = - specfem::element::property_tag::anisotropic; - - using ViewType = typename Kokkos::View; - - int nspec; ///< total number of acoustic spectral elements - int ngllz; ///< number of quadrature points in z dimension - int ngllx; ///< number of quadrature points in x dimension - - ViewType rho; - ViewType::HostMirror h_rho; - ViewType c11; - ViewType::HostMirror h_c11; - ViewType c13; - ViewType::HostMirror h_c13; - ViewType c15; - ViewType::HostMirror h_c15; - ViewType c33; - ViewType::HostMirror h_c33; - ViewType c35; - ViewType::HostMirror h_c35; - ViewType c55; - ViewType::HostMirror h_c55; - ViewType c12; - ViewType::HostMirror h_c12; - ViewType c23; - ViewType::HostMirror h_c23; - ViewType c25; - ViewType::HostMirror h_c25; - - properties_container() = default; - - properties_container(const int nspec, const int ngllz, const int ngllx) - : nspec(nspec), ngllz(ngllz), ngllx(ngllx), - rho("specfem::compute::properties::rho", nspec, ngllz, ngllx), - h_rho(Kokkos::create_mirror_view(rho)), - c11("specfem::compute::properties::c11", nspec, ngllz, ngllx), - h_c11(Kokkos::create_mirror_view(c11)), - c12("specfem::compute::properties::c12", nspec, ngllz, ngllx), - h_c12(Kokkos::create_mirror_view(c12)), - c13("specfem::compute::properties::c13", nspec, ngllz, ngllx), - h_c13(Kokkos::create_mirror_view(c13)), - c15("specfem::compute::properties::c15", nspec, ngllz, ngllx), - h_c15(Kokkos::create_mirror_view(c15)), - c33("specfem::compute::properties::c33", nspec, ngllz, ngllx), - h_c33(Kokkos::create_mirror_view(c33)), - c35("specfem::compute::properties::c35", nspec, ngllz, ngllx), - h_c35(Kokkos::create_mirror_view(c35)), - c55("specfem::compute::properties::c55", nspec, ngllz, ngllx), - h_c55(Kokkos::create_mirror_view(c55)), - c23("specfem::compute::properties::c23", nspec, ngllz, ngllx), - h_c23(Kokkos::create_mirror_view(c23)), - c25("specfem::compute::properties::c25", nspec, ngllz, ngllx), - h_c25(Kokkos::create_mirror_view(c25)) {} - - template < - typename PointProperties, - typename std::enable_if_t = 0> - KOKKOS_FORCEINLINE_FUNCTION void - load_device_properties(const specfem::point::index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - property.rho = rho(ispec, iz, ix); - property.c11 = c11(ispec, iz, ix); - property.c12 = c12(ispec, iz, ix); - property.c13 = c13(ispec, iz, ix); - property.c15 = c15(ispec, iz, ix); - property.c33 = c33(ispec, iz, ix); - property.c35 = c35(ispec, iz, ix); - property.c55 = c55(ispec, iz, ix); - property.c23 = c23(ispec, iz, ix); - property.c25 = c25(ispec, iz, ix); - - property.rho_vp = sqrt(property.rho * property.c33); - property.rho_vs = sqrt(property.rho * property.c55); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - KOKKOS_FORCEINLINE_FUNCTION void - load_device_properties(const specfem::point::simd_index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho) - .copy_from(&rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c11) - .copy_from(&c11(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c12) - .copy_from(&c12(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c13) - .copy_from(&c13(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c15) - .copy_from(&c15(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c33) - .copy_from(&c33(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c35) - .copy_from(&c35(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c55) - .copy_from(&c55(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c23) - .copy_from(&c23(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c25) - .copy_from(&c25(ispec, iz, ix), tag_type()); - - property.rho_vp = Kokkos::sqrt(property.rho * property.c33); - property.rho_vs = Kokkos::sqrt(property.rho * property.c55); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void - load_host_properties(const specfem::point::index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - property.rho = h_rho(ispec, iz, ix); - property.c11 = h_c11(ispec, iz, ix); - property.c12 = h_c12(ispec, iz, ix); - property.c13 = h_c13(ispec, iz, ix); - property.c15 = h_c15(ispec, iz, ix); - property.c33 = h_c33(ispec, iz, ix); - property.c35 = h_c35(ispec, iz, ix); - property.c55 = h_c55(ispec, iz, ix); - property.c23 = h_c23(ispec, iz, ix); - property.c25 = h_c25(ispec, iz, ix); - - property.rho_vp = sqrt(property.rho * property.c33); - property.rho_vs = sqrt(property.rho * property.c55); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void - load_host_properties(const specfem::point::simd_index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho) - .copy_from(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c11) - .copy_from(&h_c11(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c13) - .copy_from(&h_c13(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c15) - .copy_from(&h_c15(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c33) - .copy_from(&h_c33(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c35) - .copy_from(&h_c35(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c55) - .copy_from(&h_c55(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c23) - .copy_from(&h_c23(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c25) - .copy_from(&h_c25(ispec, iz, ix), tag_type()); - - property.rho_vp = Kokkos::sqrt(property.rho * property.c33); - property.rho_vs = Kokkos::sqrt(property.rho * property.c55); - } - - void copy_to_device() { - Kokkos::deep_copy(rho, h_rho); - Kokkos::deep_copy(c11, h_c11); - Kokkos::deep_copy(c13, h_c13); - Kokkos::deep_copy(c15, h_c15); - Kokkos::deep_copy(c12, h_c12); - Kokkos::deep_copy(c33, h_c33); - Kokkos::deep_copy(c35, h_c35); - Kokkos::deep_copy(c55, h_c55); - Kokkos::deep_copy(c23, h_c23); - Kokkos::deep_copy(c25, h_c25); - } - - void copy_to_host() { - Kokkos::deep_copy(h_rho, rho); - Kokkos::deep_copy(h_c11, c11); - Kokkos::deep_copy(h_c13, c13); - Kokkos::deep_copy(h_c15, c15); - Kokkos::deep_copy(h_c12, c12); - Kokkos::deep_copy(h_c33, c33); - Kokkos::deep_copy(h_c35, c35); - Kokkos::deep_copy(h_c55, c55); - Kokkos::deep_copy(h_c23, c23); - Kokkos::deep_copy(h_c25, c25); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void assign(const specfem::point::index &index, - const PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho(ispec, iz, ix) = property.rho; - h_c11(ispec, iz, ix) = property.c11; - h_c13(ispec, iz, ix) = property.c13; - h_c15(ispec, iz, ix) = property.c15; - h_c12(ispec, iz, ix) = property.c12; - h_c33(ispec, iz, ix) = property.c33; - h_c35(ispec, iz, ix) = property.c35; - h_c55(ispec, iz, ix) = property.c55; - h_c23(ispec, iz, ix) = property.c23; - h_c25(ispec, iz, ix) = property.c25; - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void assign(const specfem::point::simd_index &index, - const PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho) - .copy_to(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c11) - .copy_to(&h_c11(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c13) - .copy_to(&h_c13(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c15) - .copy_to(&h_c15(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c12) - .copy_to(&h_c12(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c33) - .copy_to(&h_c33(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c35) - .copy_to(&h_c35(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c55) - .copy_to(&h_c55(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c23) - .copy_to(&h_c23(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.c25) - .copy_to(&h_c25(ispec, iz, ix), tag_type()); - } + specfem::element::property_tag::anisotropic> + : public impl_properties_container< + specfem::element::medium_tag::elastic, + specfem::element::property_tag::anisotropic, 10> { + using base_type = + impl_properties_container; + using base_type::base_type; + constexpr static int _counter = __COUNTER__; + + DEFINE_MEDIUM_VIEW(c11) + DEFINE_MEDIUM_VIEW(c13) + DEFINE_MEDIUM_VIEW(c15) + DEFINE_MEDIUM_VIEW(c33) + DEFINE_MEDIUM_VIEW(c35) + DEFINE_MEDIUM_VIEW(c55) + DEFINE_MEDIUM_VIEW(c12) + DEFINE_MEDIUM_VIEW(c23) + DEFINE_MEDIUM_VIEW(c25) + DEFINE_MEDIUM_VIEW(rho) }; } // namespace medium diff --git a/include/medium/dim2/elastic/anisotropic/stress.hpp b/include/medium/dim2/elastic/anisotropic/stress.hpp index 39ba0382..563cd2f4 100644 --- a/include/medium/dim2/elastic/anisotropic/stress.hpp +++ b/include/medium/dim2/elastic/anisotropic/stress.hpp @@ -28,16 +28,16 @@ impl_compute_stress( // P_SV case // sigma_xx - sigma_xx = properties.c11 * du(0, 0) + properties.c13 * du(1, 1) + - properties.c15 * (du(1, 0) + du(0, 1)); + sigma_xx = properties.c11() * du(0, 0) + properties.c13() * du(1, 1) + + properties.c15() * (du(1, 0) + du(0, 1)); // sigma_zz - sigma_zz = properties.c13 * du(0, 0) + properties.c33 * du(1, 1) + - properties.c35 * (du(1, 0) + du(0, 1)); + sigma_zz = properties.c13() * du(0, 0) + properties.c33() * du(1, 1) + + properties.c35() * (du(1, 0) + du(0, 1)); // sigma_xz - sigma_xz = properties.c15 * du(0, 0) + properties.c35 * du(1, 1) + - properties.c55 * (du(1, 0) + du(0, 1)); + sigma_xz = properties.c15() * du(0, 0) + properties.c35() * du(1, 1) + + properties.c55() * (du(1, 0) + du(0, 1)); specfem::datatype::VectorPointViewType T; diff --git a/include/medium/dim2/elastic/anisotropic/wavefield.hpp b/include/medium/dim2/elastic/anisotropic/wavefield.hpp index 4dfd2050..b0f4ff6f 100644 --- a/include/medium/dim2/elastic/anisotropic/wavefield.hpp +++ b/include/medium/dim2/elastic/anisotropic/wavefield.hpp @@ -66,7 +66,7 @@ KOKKOS_FUNCTION void impl_compute_wavefield( // cannot compute pressure for an anisotropic material if c12 or c23 // are zero - if (point_property.c12 < 1.e-7 || point_property.c23 < 1.e-7) { + if (point_property.c12() < 1.e-7 || point_property.c23() < 1.e-7) { Kokkos::abort("C_12 or C_23 are zero, cannot compute pressure. " "Check your material properties. Or, deactivate the " "pressure computation."); @@ -85,19 +85,19 @@ KOKKOS_FUNCTION void impl_compute_wavefield( // P_SV case // sigma_xx - const auto sigma_xx = point_property.c11 * du(0, 0) + - point_property.c13 * du(1, 1) + - point_property.c15 * (du(1, 0) + du(0, 1)); + const auto sigma_xx = point_property.c11() * du(0, 0) + + point_property.c13() * du(1, 1) + + point_property.c15() * (du(1, 0) + du(0, 1)); // sigma_zz - const auto sigma_zz = point_property.c13 * du(0, 0) + - point_property.c33 * du(1, 1) + - point_property.c35 * (du(1, 0) + du(0, 1)); + const auto sigma_zz = point_property.c13() * du(0, 0) + + point_property.c33() * du(1, 1) + + point_property.c35() * (du(1, 0) + du(0, 1)); // sigma_yy - const auto sigma_yy = point_property.c12 * du(0, 0) + - point_property.c23 * du(1, 1) + - point_property.c25 * (du(1, 0) + du(0, 1)); + const auto sigma_yy = point_property.c12() * du(0, 0) + + point_property.c23() * du(1, 1) + + point_property.c25() * (du(1, 0) + du(0, 1)); wavefield(iterator_index.ielement, index.iz, index.ix, 0) = -1.0 * (sigma_xx + sigma_zz + sigma_yy) / 3.0; diff --git a/include/medium/dim2/elastic/isotropic/frechet_derivative.hpp b/include/medium/dim2/elastic/isotropic/frechet_derivative.hpp index 3949495b..8853fd43 100644 --- a/include/medium/dim2/elastic/isotropic/frechet_derivative.hpp +++ b/include/medium/dim2/elastic/isotropic/frechet_derivative.hpp @@ -34,7 +34,7 @@ impl_compute_frechet_derivatives( specfem::globals::simulation_wave == specfem::wave::sh, "Only P-SV and SH waves are supported."); - const auto kappa = properties.lambdaplus2mu - properties.mu; + const auto kappa = properties.lambdaplus2mu() - properties.mu(); if (specfem::globals::simulation_wave == specfem::wave::p_sv) { @@ -121,8 +121,8 @@ impl_compute_frechet_derivatives( // Finishing the kernels kappa_kl = static_cast(-1.0) * kappa * dt * kappa_kl; - mu_kl = static_cast(-2.0) * properties.mu * dt * mu_kl; - rho_kl = static_cast(-1.0) * properties.rho * dt * rho_kl; + mu_kl = static_cast(-2.0) * properties.mu() * dt * mu_kl; + rho_kl = static_cast(-1.0) * properties.rho() * dt * rho_kl; // rho' kernel, first term in Equation 20 const auto rhop_kl = rho_kl + kappa_kl + mu_kl; @@ -130,14 +130,14 @@ impl_compute_frechet_derivatives( // beta (shear wave) kernel, second term in Equation 20 const auto beta_kl = static_cast(2.0) * (mu_kl - static_cast(4.0 / 3.0) * - properties.mu / kappa * kappa_kl); + properties.mu() / kappa * kappa_kl); // alpha (compressional wave) kernel, third and last term in Eq. 20 // of Tromp et al 2005. const auto alpha_kl = static_cast(2.0) * (static_cast(1.0) + - static_cast(4.0 / 3.0) * properties.mu / kappa) * + static_cast(4.0 / 3.0) * properties.mu() / kappa) * kappa_kl; return { rho_kl, mu_kl, kappa_kl, rhop_kl, alpha_kl, beta_kl }; @@ -164,13 +164,13 @@ impl_compute_frechet_derivatives( */ const auto mu_kl = - static_cast(-2.0) * properties.mu * dt * + static_cast(-2.0) * properties.mu() * dt * static_cast(0.5) * // du#y_dx * duy_dx + (adjoint_derivatives.du(0, 0) * backward_derivatives.du(0, 0) + // du#y_dz * duy_dz adjoint_derivatives.du(1, 0) * backward_derivatives.du(1, 0)); - const auto rho_kl = static_cast(-1.0) * properties.rho * dt * + const auto rho_kl = static_cast(-1.0) * properties.rho() * dt * specfem::algorithms::dot(adjoint_field.acceleration, backward_field.displacement); const auto kappa_kl = decltype(mu_kl)(0.0); diff --git a/include/medium/dim2/elastic/isotropic/kernels_container.hpp b/include/medium/dim2/elastic/isotropic/kernels_container.hpp index 65073f82..888def47 100644 --- a/include/medium/dim2/elastic/isotropic/kernels_container.hpp +++ b/include/medium/dim2/elastic/isotropic/kernels_container.hpp @@ -1,479 +1,30 @@ #pragma once -#include "enumerations/medium.hpp" -#include "kokkos_abstractions.h" -#include "point/coordinates.hpp" -#include "point/kernels.hpp" -#include +#include "medium/properties_container.hpp" +#include "point/interface.hpp" +#include namespace specfem { namespace medium { template <> -class kernels_container { -public: - constexpr static auto value_type = specfem::element::medium_tag::elastic; - constexpr static auto property_type = - specfem::element::property_tag::isotropic; - int nspec; - int ngllz; - int ngllx; - - using ViewType = Kokkos::View; - - ViewType rho; - ViewType::HostMirror h_rho; - ViewType mu; - ViewType::HostMirror h_mu; - ViewType kappa; - ViewType::HostMirror h_kappa; - ViewType rhop; - ViewType::HostMirror h_rhop; - ViewType alpha; - ViewType::HostMirror h_alpha; - ViewType beta; - ViewType::HostMirror h_beta; - - kernels_container() = default; - - kernels_container(const int nspec, const int ngllz, const int ngllx) - : nspec(nspec), ngllz(ngllz), ngllx(ngllx), - rho("specfem::medium::elastic::rho", nspec, ngllz, ngllx), - mu("specfem::medium::elastic::mu", nspec, ngllz, ngllx), - kappa("specfem::medium::elastic::kappa", nspec, ngllz, ngllx), - rhop("specfem::medium::elastic::rhop", nspec, ngllz, ngllx), - alpha("specfem::medium::elastic::alpha", nspec, ngllz, ngllx), - beta("specfem::medium::elastic::beta", nspec, ngllz, ngllx), - h_rho(Kokkos::create_mirror_view(rho)), - h_mu(Kokkos::create_mirror_view(mu)), - h_kappa(Kokkos::create_mirror_view(kappa)), - h_rhop(Kokkos::create_mirror_view(rhop)), - h_alpha(Kokkos::create_mirror_view(alpha)), - h_beta(Kokkos::create_mirror_view(beta)) { - - initialize(); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void load_device_kernels( - const specfem::point::index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - kernels.rho = rho(ispec, iz, ix); - kernels.mu = mu(ispec, iz, ix); - kernels.kappa = kappa(ispec, iz, ix); - kernels.rhop = rhop(ispec, iz, ix); - kernels.alpha = alpha(ispec, iz, ix); - kernels.beta = beta(ispec, iz, ix); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void load_device_kernels( - const specfem::point::simd_index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_from(&rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.mu) - .copy_from(&mu(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.kappa) - .copy_from(&kappa(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.rhop) - .copy_from(&rhop(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.alpha) - .copy_from(&alpha(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.beta) - .copy_from(&beta(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void - load_host_kernels(specfem::point::index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - kernels.rho = h_rho(ispec, iz, ix); - kernels.mu = h_mu(ispec, iz, ix); - kernels.kappa = h_kappa(ispec, iz, ix); - kernels.rhop = h_rhop(ispec, iz, ix); - kernels.alpha = h_alpha(ispec, iz, ix); - kernels.beta = h_beta(ispec, iz, ix); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void load_host_kernels( - specfem::point::simd_index &index, - PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_from(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.mu) - .copy_from(&h_mu(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.kappa) - .copy_from(&h_kappa(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.rhop) - .copy_from(&h_rhop(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.alpha) - .copy_from(&h_alpha(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.beta) - .copy_from(&h_beta(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void update_kernels_on_device( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - rho(ispec, iz, ix) = kernels.rho; - mu(ispec, iz, ix) = kernels.mu; - kappa(ispec, iz, ix) = kernels.kappa; - rhop(ispec, iz, ix) = kernels.rhop; - alpha(ispec, iz, ix) = kernels.alpha; - beta(ispec, iz, ix) = kernels.beta; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void update_kernels_on_device( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_to(&rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.mu) - .copy_to(&mu(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.kappa) - .copy_to(&kappa(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.rhop) - .copy_to(&rhop(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.alpha) - .copy_to(&alpha(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.beta) - .copy_to(&beta(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void update_kernels_on_host( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho(ispec, iz, ix) = kernels.rho; - h_mu(ispec, iz, ix) = kernels.mu; - h_kappa(ispec, iz, ix) = kernels.kappa; - h_rhop(ispec, iz, ix) = kernels.rhop; - h_alpha(ispec, iz, ix) = kernels.alpha; - h_beta(ispec, iz, ix) = kernels.beta; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void update_kernels_on_host( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - Kokkos::Experimental::where(mask, kernels.rho) - .copy_to(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.mu) - .copy_to(&h_mu(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.kappa) - .copy_to(&h_kappa(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.rhop) - .copy_to(&h_rhop(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.alpha) - .copy_to(&h_alpha(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, kernels.beta) - .copy_to(&h_beta(ispec, iz, ix), tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void add_kernels_on_device( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - rho(ispec, iz, ix) += kernels.rho; - mu(ispec, iz, ix) += kernels.mu; - kappa(ispec, iz, ix) += kernels.kappa; - rhop(ispec, iz, ix) += kernels.rhop; - alpha(ispec, iz, ix) += kernels.alpha; - beta(ispec, iz, ix) += kernels.beta; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - KOKKOS_INLINE_FUNCTION void add_kernels_on_device( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - simd_type lhs; - - Kokkos::Experimental::where(mask, lhs).copy_from(&rho(ispec, iz, ix), - tag_type()); - lhs += kernels.rho; - Kokkos::Experimental::where(mask, lhs).copy_to(&rho(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&mu(ispec, iz, ix), - tag_type()); - lhs += kernels.mu; - Kokkos::Experimental::where(mask, lhs).copy_to(&mu(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&kappa(ispec, iz, ix), - tag_type()); - lhs += kernels.kappa; - Kokkos::Experimental::where(mask, lhs).copy_to(&kappa(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&rhop(ispec, iz, ix), - tag_type()); - lhs += kernels.rhop; - Kokkos::Experimental::where(mask, lhs).copy_to(&rhop(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&alpha(ispec, iz, ix), - tag_type()); - lhs += kernels.alpha; - Kokkos::Experimental::where(mask, lhs).copy_to(&alpha(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&beta(ispec, iz, ix), - tag_type()); - lhs += kernels.beta; - Kokkos::Experimental::where(mask, lhs).copy_to(&beta(ispec, iz, ix), - tag_type()); - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void add_kernels_on_host( - const specfem::point::index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho(ispec, iz, ix) += kernels.rho; - h_mu(ispec, iz, ix) += kernels.mu; - h_kappa(ispec, iz, ix) += kernels.kappa; - h_rhop(ispec, iz, ix) += kernels.rhop; - h_alpha(ispec, iz, ix) += kernels.alpha; - h_beta(ispec, iz, ix) += kernels.beta; - } - - template < - typename PointKernelType, - typename std::enable_if_t = 0> - void add_kernels_on_host( - const specfem::point::simd_index &index, - const PointKernelType &kernels) const { - - static_assert(PointKernelType::medium_tag == value_type); - static_assert(PointKernelType::property_tag == property_type); - - using simd_type = typename PointKernelType::simd::datatype; - using mask_type = typename PointKernelType::simd::mask_type; - using tag_type = typename PointKernelType::simd::tag_type; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - simd_type lhs; - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_rho(ispec, iz, ix), - tag_type()); - lhs += kernels.rho; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_rho(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_mu(ispec, iz, ix), - tag_type()); - lhs += kernels.mu; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_mu(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_kappa(ispec, iz, ix), - tag_type()); - lhs += kernels.kappa; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_kappa(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_rhop(ispec, iz, ix), - tag_type()); - lhs += kernels.rhop; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_rhop(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_alpha(ispec, iz, ix), - tag_type()); - lhs += kernels.alpha; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_alpha(ispec, iz, ix), - tag_type()); - - Kokkos::Experimental::where(mask, lhs).copy_from(&h_beta(ispec, iz, ix), - tag_type()); - lhs += kernels.beta; - Kokkos::Experimental::where(mask, lhs).copy_to(&h_beta(ispec, iz, ix), - tag_type()); - } - - void copy_to_host() { - Kokkos::deep_copy(h_rho, rho); - Kokkos::deep_copy(h_mu, mu); - Kokkos::deep_copy(h_kappa, kappa); - Kokkos::deep_copy(h_rhop, rhop); - Kokkos::deep_copy(h_alpha, alpha); - Kokkos::deep_copy(h_beta, beta); - } - - void copy_to_device() { - Kokkos::deep_copy(rho, h_rho); - Kokkos::deep_copy(mu, h_mu); - Kokkos::deep_copy(kappa, h_kappa); - Kokkos::deep_copy(rhop, h_rhop); - Kokkos::deep_copy(alpha, h_alpha); - Kokkos::deep_copy(beta, h_beta); - } - - void initialize() { - Kokkos::parallel_for( - "specfem::medium::elastic::isotropic::initialize", - Kokkos::MDRangePolicy >({ 0, 0, 0 }, - { nspec, ngllz, ngllx }), - KOKKOS_CLASS_LAMBDA(const int ispec, const int iz, const int ix) { - this->rho(ispec, iz, ix) = 0.0; - this->mu(ispec, iz, ix) = 0.0; - this->kappa(ispec, iz, ix) = 0.0; - this->rhop(ispec, iz, ix) = 0.0; - this->alpha(ispec, iz, ix) = 0.0; - this->beta(ispec, iz, ix) = 0.0; - }); - } +struct kernels_container + : public impl_kernels_container { + using base_type = + impl_kernels_container; + using base_type::base_type; + constexpr static int _counter = __COUNTER__; + + DEFINE_MEDIUM_VIEW(rho) + DEFINE_MEDIUM_VIEW(mu) + DEFINE_MEDIUM_VIEW(kappa) + DEFINE_MEDIUM_VIEW(rhop) + DEFINE_MEDIUM_VIEW(alpha) + DEFINE_MEDIUM_VIEW(beta) }; } // namespace medium diff --git a/include/medium/dim2/elastic/isotropic/mass_matrix.tpp b/include/medium/dim2/elastic/isotropic/mass_matrix.tpp index f49f3d8f..28b8ae7a 100644 --- a/include/medium/dim2/elastic/isotropic/mass_matrix.tpp +++ b/include/medium/dim2/elastic/isotropic/mass_matrix.tpp @@ -15,11 +15,11 @@ specfem::medium::impl_mass_matrix_component( if constexpr (specfem::globals::simulation_wave == specfem::wave::p_sv) { return specfem::datatype::ScalarPointViewType( - partial_derivatives.jacobian * properties.rho, - partial_derivatives.jacobian * properties.rho); + partial_derivatives.jacobian * properties.rho(), + partial_derivatives.jacobian * properties.rho()); } else if constexpr (specfem::globals::simulation_wave == specfem::wave::sh) { return specfem::datatype::ScalarPointViewType( - partial_derivatives.jacobian * properties.rho, 0); + partial_derivatives.jacobian * properties.rho(), 0); } else { static_assert("Unknown wave type"); return specfem::datatype::ScalarPointViewType( diff --git a/include/medium/dim2/elastic/isotropic/properties.hpp b/include/medium/dim2/elastic/isotropic/material.hpp similarity index 86% rename from include/medium/dim2/elastic/isotropic/properties.hpp rename to include/medium/dim2/elastic/isotropic/material.hpp index 6dc7162d..c7f02dda 100644 --- a/include/medium/dim2/elastic/isotropic/properties.hpp +++ b/include/medium/dim2/elastic/isotropic/material.hpp @@ -1,7 +1,6 @@ #pragma once #include "enumerations/medium.hpp" -#include "medium/properties.hpp" #include "point/properties.hpp" #include "specfem_setup.hpp" #include @@ -17,8 +16,8 @@ namespace medium { * */ template <> -class properties { +class impl_material { public: constexpr static auto dimension = specfem::dimension::type::dim2; ///< Dimension of the material @@ -41,9 +40,9 @@ class properties + bool operator==(const impl_material &other) const { return (std::abs(this->density - other.density) < 1e-6 && @@ -91,8 +90,8 @@ class properties + bool operator!=(const impl_material &other) const { return !(*this == other); } diff --git a/include/medium/dim2/elastic/isotropic/properties_container.hpp b/include/medium/dim2/elastic/isotropic/properties_container.hpp index 4be8eda7..217c5b23 100644 --- a/include/medium/dim2/elastic/isotropic/properties_container.hpp +++ b/include/medium/dim2/elastic/isotropic/properties_container.hpp @@ -9,226 +9,19 @@ namespace medium { template <> struct properties_container { - - constexpr static auto dimension = specfem::dimension::type::dim2; - constexpr static auto value_type = specfem::element::medium_tag::elastic; - constexpr static auto property_type = - specfem::element::property_tag::isotropic; - - using ViewType = typename Kokkos::View; - - int nspec; ///< total number of acoustic spectral elements - int ngllz; ///< number of quadrature points in z dimension - int ngllx; ///< number of quadrature points in x dimension - ViewType rho; - ViewType::HostMirror h_rho; - ViewType mu; - ViewType::HostMirror h_mu; - ViewType lambdaplus2mu; - ViewType::HostMirror h_lambdaplus2mu; - - properties_container() = default; - - properties_container(const int nspec, const int ngllz, const int ngllx) - : nspec(nspec), ngllz(ngllz), ngllx(ngllx), - rho("specfem::compute::properties::rho", nspec, ngllz, ngllx), - h_rho(Kokkos::create_mirror_view(rho)), - mu("specfem::compute::properties::mu", nspec, ngllz, ngllx), - h_mu(Kokkos::create_mirror_view(mu)), - lambdaplus2mu("specfem::compute::properties::lambdaplus2mu", nspec, - ngllz, ngllx), - h_lambdaplus2mu(Kokkos::create_mirror_view(lambdaplus2mu)) {} - - template < - typename PointProperties, - typename std::enable_if_t = 0> - KOKKOS_FORCEINLINE_FUNCTION void - load_device_properties(const specfem::point::index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - property.rho = rho(ispec, iz, ix); - property.mu = mu(ispec, iz, ix); - property.lambdaplus2mu = lambdaplus2mu(ispec, iz, ix); - property.lambda = property.lambdaplus2mu - 2 * property.mu; - property.rho_vp = sqrt(property.rho * property.lambdaplus2mu); - property.rho_vs = sqrt(property.rho * property.mu); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - KOKKOS_FORCEINLINE_FUNCTION void - load_device_properties(const specfem::point::simd_index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho) - .copy_from(&rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.mu) - .copy_from(&mu(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.lambdaplus2mu) - .copy_from(&lambdaplus2mu(ispec, iz, ix), tag_type()); - - property.lambda = property.lambdaplus2mu - 2 * property.mu; - property.rho_vp = Kokkos::sqrt(property.rho * property.lambdaplus2mu); - property.rho_vs = Kokkos::sqrt(property.rho * property.mu); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void - load_host_properties(const specfem::point::index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - property.rho = h_rho(ispec, iz, ix); - property.mu = h_mu(ispec, iz, ix); - property.lambdaplus2mu = h_lambdaplus2mu(ispec, iz, ix); - property.lambda = property.lambdaplus2mu - 2 * property.mu; - property.rho_vp = sqrt(property.rho * property.lambdaplus2mu); - property.rho_vs = sqrt(property.rho * property.mu); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void - load_host_properties(const specfem::point::simd_index &index, - PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho) - .copy_from(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.mu) - .copy_from(&h_mu(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.lambdaplus2mu) - .copy_from(&h_lambdaplus2mu(ispec, iz, ix), tag_type()); - - property.lambda = property.lambdaplus2mu - 2 * property.mu; - property.rho_vp = Kokkos::sqrt(property.rho * property.lambdaplus2mu); - property.rho_vs = Kokkos::sqrt(property.rho * property.mu); - } - - void copy_to_device() { - Kokkos::deep_copy(rho, h_rho); - Kokkos::deep_copy(mu, h_mu); - Kokkos::deep_copy(lambdaplus2mu, h_lambdaplus2mu); - } - - void copy_to_host() { - Kokkos::deep_copy(h_rho, rho); - Kokkos::deep_copy(h_mu, mu); - Kokkos::deep_copy(h_lambdaplus2mu, lambdaplus2mu); - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void assign(const specfem::point::index &index, - const PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - h_rho(ispec, iz, ix) = property.rho; - h_mu(ispec, iz, ix) = property.mu; - h_lambdaplus2mu(ispec, iz, ix) = property.lambdaplus2mu; - } - - template < - typename PointProperties, - typename std::enable_if_t = 0> - inline void assign(const specfem::point::simd_index &index, - const PointProperties &property) const { - - static_assert(PointProperties::dimension == dimension, - "Dimension mismatch"); - static_assert(PointProperties::medium_tag == value_type, - "Medium tag mismatch"); - static_assert(PointProperties::property_tag == property_type, - "Property tag mismatch"); - - using simd = typename PointProperties::simd; - using mask_type = typename simd::mask_type; - using tag_type = typename simd::tag_type; - - const int ispec = index.ispec; - const int iz = index.iz; - const int ix = index.ix; - - mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); - - Kokkos::Experimental::where(mask, property.rho) - .copy_to(&h_rho(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.mu) - .copy_to(&h_mu(ispec, iz, ix), tag_type()); - Kokkos::Experimental::where(mask, property.lambdaplus2mu) - .copy_to(&h_lambdaplus2mu(ispec, iz, ix), tag_type()); - } + specfem::element::property_tag::isotropic> + : public impl_properties_container< + specfem::element::medium_tag::elastic, + specfem::element::property_tag::isotropic, 3> { + using base_type = + impl_properties_container; + using base_type::base_type; + constexpr static int _counter = __COUNTER__; + + DEFINE_MEDIUM_VIEW(lambdaplus2mu) + DEFINE_MEDIUM_VIEW(mu) + DEFINE_MEDIUM_VIEW(rho) }; } // namespace medium diff --git a/include/medium/dim2/elastic/isotropic/stress.hpp b/include/medium/dim2/elastic/isotropic/stress.hpp index a90f6cce..e7088a7f 100644 --- a/include/medium/dim2/elastic/isotropic/stress.hpp +++ b/include/medium/dim2/elastic/isotropic/stress.hpp @@ -29,13 +29,15 @@ impl_compute_stress( // P_SV case // sigma_xx - sigma_xx = properties.lambdaplus2mu * du(0, 0) + properties.lambda * du(1, 1); + sigma_xx = + properties.lambdaplus2mu() * du(0, 0) + properties.lambda() * du(1, 1); // sigma_zz - sigma_zz = properties.lambdaplus2mu * du(1, 1) + properties.lambda * du(0, 0); + sigma_zz = + properties.lambdaplus2mu() * du(1, 1) + properties.lambda() * du(0, 0); // sigma_xz - sigma_xz = properties.mu * (du(0, 1) + du(1, 0)); + sigma_xz = properties.mu() * (du(0, 1) + du(1, 0)); specfem::datatype::VectorPointViewType T; diff --git a/include/medium/dim2/elastic/isotropic/stress_integrand.tpp b/include/medium/dim2/elastic/isotropic/stress_integrand.tpp index e6d2f0d8..12f668a9 100644 --- a/include/medium/dim2/elastic/isotropic/stress_integrand.tpp +++ b/include/medium/dim2/elastic/isotropic/stress_integrand.tpp @@ -26,22 +26,22 @@ specfem::medium::impl_compute_stress_integrands( // P_SV case // sigma_xx sigma_xx = - properties.lambdaplus2mu * du(0, 0) + properties.lambda * du(1, 1); + properties.lambdaplus2mu() * du(0, 0) + properties.lambda() * du(1, 1); // sigma_zz sigma_zz = - properties.lambdaplus2mu * du(1, 1) + properties.lambda * du(0, 0); + properties.lambdaplus2mu() * du(1, 1) + properties.lambda() * du(0, 0); // sigma_xz - sigma_xz = properties.mu * (du(0, 1) + du(1, 0)); + sigma_xz = properties.mu() * (du(0, 1) + du(1, 0)); } else if (specfem::globals::simulation_wave == specfem::wave::sh) { // SH-case // sigma_xx - sigma_xx = properties.mu * du(0, 0); // would be sigma_xy in + sigma_xx = properties.mu() * du(0, 0); // would be sigma_xy in // CPU-version // sigma_xz - sigma_xz = properties.mu * du(1, 0); // sigma_zy + sigma_xz = properties.mu() * du(1, 0); // sigma_zy } specfem::datatype::VectorPointViewType F; diff --git a/include/medium/dim2/elastic/isotropic/wavefield.hpp b/include/medium/dim2/elastic/isotropic/wavefield.hpp index 5deb6b24..7f4da876 100644 --- a/include/medium/dim2/elastic/isotropic/wavefield.hpp +++ b/include/medium/dim2/elastic/isotropic/wavefield.hpp @@ -80,7 +80,7 @@ KOKKOS_FUNCTION void impl_compute_wavefield( // -1.0 * (sigma_xx + sigma_zz + sigma_yy) / 3.0; wavefield(iterator_index.ielement, index.iz, index.ix, 0) = -1.0 * - ((point_property.lambda + (2.0 / 3.0) * point_property.mu) * + ((point_property.lambda() + (2.0 / 3.0) * point_property.mu()) * (du(0, 0) + du(1, 1))); }); diff --git a/include/medium/impl/medium_data.hpp b/include/medium/impl/medium_data.hpp new file mode 100644 index 00000000..d1151257 --- /dev/null +++ b/include/medium/impl/medium_data.hpp @@ -0,0 +1,240 @@ +#pragma once + +#include "enumerations/medium.hpp" + +#define DEFINE_MEDIUM_VIEW(prop) \ + constexpr static int i_##prop = __COUNTER__ - _counter - 1; \ + KOKKOS_INLINE_FUNCTION type_real &prop(const int &ispec, const int &iz, \ + const int &ix) const { \ + return base_type::data(ispec, iz, ix, i_##prop); \ + } \ + KOKKOS_INLINE_FUNCTION type_real &h_##prop(const int &ispec, const int &iz, \ + const int &ix) const { \ + return base_type::h_data(ispec, iz, ix, i_##prop); \ + } + +namespace specfem { +namespace medium { + +namespace impl { +template +struct medium_data { + using view_type = typename Kokkos::View; + constexpr static auto nprops = N; + constexpr static auto dimension = specfem::dimension::type::dim2; + constexpr static auto medium_tag = MediumTag; + constexpr static auto property_tag = PropertyTag; + + int nspec; ///< total number of acoustic spectral elements + int ngllz; ///< number of quadrature points in z dimension + int ngllx; ///< number of quadrature points in x dimension + + view_type data; + typename view_type::HostMirror h_data; + + medium_data() = default; + + medium_data(const int nspec, const int ngllz, const int ngllx) + : nspec(nspec), ngllz(ngllz), ngllx(ngllx), + data("specfem::medium::impl::container::data", nspec, ngllz, ngllx, N), + h_data(Kokkos::create_mirror_view(data)) {} + +private: + template + KOKKOS_FORCEINLINE_FUNCTION constexpr type_real & + get_data(const int &ispec, const int &iz, const int &ix, const int &i) const { + if constexpr (on_device) { + return data(ispec, iz, ix, i); + } else { + return h_data(ispec, iz, ix, i); + } + } + + template + KOKKOS_FORCEINLINE_FUNCTION void + load_values(const specfem::point::index &index, + PointValues &values) const { + + static_assert(PointValues::dimension == dimension, "Dimension mismatch"); + static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); + static_assert(PointValues::property_tag == property_tag, + "Property tag mismatch"); + + const int ispec = index.ispec; + const int iz = index.iz; + const int ix = index.ix; + + for (int i = 0; i < nprops; i++) { + values.data[i] = get_data(ispec, iz, ix, i); + } + } + + template + KOKKOS_FORCEINLINE_FUNCTION void + load_values(const specfem::point::simd_index &index, + PointValues &values) const { + + static_assert(PointValues::dimension == dimension, "Dimension mismatch"); + static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); + static_assert(PointValues::property_tag == property_tag, + "Property tag mismatch"); + + using simd = typename PointValues::simd; + using mask_type = typename simd::mask_type; + using tag_type = typename simd::tag_type; + + const int ispec = index.ispec; + const int iz = index.iz; + const int ix = index.ix; + + mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); + + for (int i = 0; i < nprops; i++) { + Kokkos::Experimental::where(mask, values.data[i]) + .copy_from(&get_data(ispec, iz, ix, i), tag_type()); + } + } + + template + KOKKOS_FORCEINLINE_FUNCTION void + store_values(const specfem::point::index &index, + const PointValues &values) const { + + static_assert(PointValues::dimension == dimension, "Dimension mismatch"); + static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); + static_assert(PointValues::property_tag == property_tag, + "Property tag mismatch"); + + const int ispec = index.ispec; + const int iz = index.iz; + const int ix = index.ix; + + for (int i = 0; i < nprops; i++) { + get_data(ispec, iz, ix, i) = values.data[i]; + } + } + + template + KOKKOS_FORCEINLINE_FUNCTION void + store_values(const specfem::point::simd_index &index, + const PointValues &values) const { + + static_assert(PointValues::dimension == dimension, "Dimension mismatch"); + static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); + static_assert(PointValues::property_tag == property_tag, + "Property tag mismatch"); + + using simd = typename PointValues::simd; + using mask_type = typename simd::mask_type; + using tag_type = typename simd::tag_type; + + const int ispec = index.ispec; + const int iz = index.iz; + const int ix = index.ix; + + mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); + + for (int i = 0; i < nprops; i++) { + Kokkos::Experimental::where(mask, values.data[i]) + .copy_to(&get_data(ispec, iz, ix, i), tag_type()); + } + } + + template + KOKKOS_FORCEINLINE_FUNCTION void + add_values(const specfem::point::index &index, + const PointValues &values) const { + + static_assert(PointValues::dimension == dimension, "Dimension mismatch"); + static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); + static_assert(PointValues::property_tag == property_tag, + "Property tag mismatch"); + + const int ispec = index.ispec; + const int iz = index.iz; + const int ix = index.ix; + + for (int i = 0; i < nprops; i++) { + get_data(ispec, iz, ix, i) += values.data[i]; + } + } + + template + KOKKOS_FORCEINLINE_FUNCTION void + add_values(const specfem::point::simd_index &index, + const PointValues &values) const { + + static_assert(PointValues::dimension == dimension, "Dimension mismatch"); + static_assert(PointValues::medium_tag == medium_tag, "Medium tag mismatch"); + static_assert(PointValues::property_tag == property_tag, + "Property tag mismatch"); + + using simd = typename PointValues::simd; + using simd_type = typename simd::datatype; + using mask_type = typename simd::mask_type; + using tag_type = typename simd::tag_type; + + const int ispec = index.ispec; + const int iz = index.iz; + const int ix = index.ix; + + simd_type lhs; + + mask_type mask([&, this](std::size_t lane) { return index.mask(lane); }); + + for (int i = 0; i < nprops; i++) { + Kokkos::Experimental::where(mask, lhs).copy_from( + &get_data(ispec, iz, ix, i), tag_type()); + lhs += values.data[i]; + Kokkos::Experimental::where(mask, lhs).copy_to( + &get_data(ispec, iz, ix, i), tag_type()); + } + } + +public: + template + KOKKOS_FORCEINLINE_FUNCTION void + load_device_values(const IndexType &index, PointValues &values) const { + load_values(index, values); + } + + template + KOKKOS_FORCEINLINE_FUNCTION void load_host_values(const IndexType &index, + PointValues &values) const { + load_values(index, values); + } + + template + KOKKOS_FORCEINLINE_FUNCTION void + store_device_values(const IndexType &index, PointValues &values) const { + store_values(index, values); + } + + template + KOKKOS_FORCEINLINE_FUNCTION void + store_host_values(const IndexType &index, PointValues &values) const { + store_values(index, values); + } + + template + KOKKOS_FORCEINLINE_FUNCTION void + add_device_values(const IndexType &index, PointValues &values) const { + add_values(index, values); + } + + template + KOKKOS_FORCEINLINE_FUNCTION void add_host_values(const IndexType &index, + PointValues &values) const { + add_values(index, values); + } + + void copy_to_device() { Kokkos::deep_copy(data, h_data); } + + void copy_to_host() { Kokkos::deep_copy(h_data, data); } +}; +} // namespace impl + +} // namespace medium +} // namespace specfem diff --git a/include/medium/kernels_container.hpp b/include/medium/kernels_container.hpp index 938be54f..9e56546b 100644 --- a/include/medium/kernels_container.hpp +++ b/include/medium/kernels_container.hpp @@ -1,14 +1,33 @@ #pragma once -#include "enumerations/medium.hpp" -#include "kokkos_abstractions.h" -#include "point/coordinates.hpp" -#include "point/kernels.hpp" -#include +#include "impl/medium_data.hpp" namespace specfem { namespace medium { +template +struct impl_kernels_container + : public impl::medium_data { + using base_type = impl::medium_data; + using base_type::base_type; + + impl_kernels_container( + const Kokkos::View elements, + const int ngllz, const int ngllx, + const specfem::kokkos::HostView1d property_index_mapping) + : impl_kernels_container(elements.extent(0), ngllz, ngllx) { + + const int nelement = elements.extent(0); + int count = 0; + for (int i = 0; i < nelement; ++i) { + const int ispec = elements(i); + property_index_mapping(ispec) = count; + count++; + } + } +}; + template class kernels_container; diff --git a/include/medium/material.hpp b/include/medium/material.hpp index cc41078a..302cfa1e 100644 --- a/include/medium/material.hpp +++ b/include/medium/material.hpp @@ -1,11 +1,7 @@ #pragma once -#include "dim2/acoustic/isotropic/properties.hpp" -#include "dim2/elastic/anisotropic/properties.hpp" -#include "dim2/elastic/isotropic/properties.hpp" #include "enumerations/specfem_enums.hpp" #include "point/properties.hpp" -#include "properties.hpp" #include "specfem_setup.hpp" #include #include @@ -21,7 +17,17 @@ namespace medium { */ template -class material : public specfem::medium::properties { +class impl_material; + +/** + * @brief Material properties for a given medium and property + * + * @tparam MediumTag Medium tag for the material + * @tparam PropertyTag Property tag for the material + */ +template +class material : public impl_material { public: constexpr static auto medium_tag = MediumTag; ///< Medium tag constexpr static auto property_tag = PropertyTag; ///< Property tag @@ -45,7 +51,7 @@ class material : public specfem::medium::properties { */ template material(Args &&...args) - : specfem::medium::properties( + : specfem::medium::impl_material( std::forward(args)...) {} ///@} @@ -63,3 +69,7 @@ class material : public specfem::medium::properties { } // namespace medium } // namespace specfem + +#include "dim2/acoustic/isotropic/material.hpp" +#include "dim2/elastic/anisotropic/material.hpp" +#include "dim2/elastic/isotropic/material.hpp" diff --git a/include/medium/material_kernels.hpp b/include/medium/material_kernels.hpp deleted file mode 100644 index 3b3196c1..00000000 --- a/include/medium/material_kernels.hpp +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include "compute/properties/properties.hpp" -#include "enumerations/interface.hpp" -#include "kernels_container.hpp" -#include "kokkos_abstractions.h" -#include - -namespace specfem { -namespace medium { - -template -class material_kernels : public kernels_container { -public: - constexpr static auto value_type = type; - constexpr static auto property_type = property; - - material_kernels() = default; - - material_kernels( - const Kokkos::View elements, - const int ngllz, const int ngllx, - const specfem::kokkos::HostView1d property_index_mapping) - : specfem::medium::kernels_container( - elements.extent(0), ngllz, ngllx) { - - const int nelement = elements.extent(0); - int count = 0; - for (int i = 0; i < nelement; ++i) { - const int ispec = elements(i); - property_index_mapping(ispec) = count; - count++; - } - } -}; -} // namespace medium -} // namespace specfem diff --git a/include/medium/material_properties.hpp b/include/medium/material_properties.hpp deleted file mode 100644 index 5c146cb1..00000000 --- a/include/medium/material_properties.hpp +++ /dev/null @@ -1,62 +0,0 @@ -#pragma once - -#include "compute/compute_mesh.hpp" -#include "enumerations/interface.hpp" -#include "medium/properties_container.hpp" -#include "mesh/materials/materials.hpp" -#include "mesh/tags/tags.hpp" - -namespace specfem { -namespace medium { - -template -struct material_properties - : public specfem::medium::properties_container { - constexpr static auto value_type = type; - constexpr static auto property_type = property; - constexpr static auto dimension = specfem::dimension::type::dim2; - - material_properties() = default; - - material_properties( - const Kokkos::View elements, - const int ngllz, const int ngllx, - const specfem::mesh::materials &materials, const bool has_gll_model, - const specfem::kokkos::HostView1d property_index_mapping) - : specfem::medium::properties_container( - elements.extent(0), ngllz, ngllx) { - - const int nelement = elements.extent(0); - int count = 0; - for (int i = 0; i < nelement; ++i) { - const int ispec = elements(i); - property_index_mapping(ispec) = count; - if (!has_gll_model) { - for (int iz = 0; iz < ngllz; ++iz) { - for (int ix = 0; ix < ngllx; ++ix) { - // Get the material at index from mesh::materials - auto material = - std::get >( - materials[ispec]); - - // Assign the material property to the property container - auto point_property = material.get_properties(); - this->assign(specfem::point::index(count, iz, ix), - point_property); - } - } - } - count++; - } - - if (!has_gll_model) { - this->copy_to_device(); - } - - return; - } -}; - -} // namespace medium -} // namespace specfem diff --git a/include/medium/properties.hpp b/include/medium/properties.hpp deleted file mode 100644 index 33d3d44d..00000000 --- a/include/medium/properties.hpp +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include "enumerations/medium.hpp" - -namespace specfem { -namespace medium { -/** - * @brief Material properties for a given medium and property - * - * @tparam MediumTag Medium tag for the material - * @tparam PropertyTag Property tag for the material - */ -template -class properties; -} // namespace medium -} // namespace specfem diff --git a/include/medium/properties_container.hpp b/include/medium/properties_container.hpp index 2b3c9d89..97802f89 100644 --- a/include/medium/properties_container.hpp +++ b/include/medium/properties_container.hpp @@ -1,10 +1,65 @@ #pragma once -#include "enumerations/medium.hpp" +#include "impl/medium_data.hpp" namespace specfem { namespace medium { +template +struct impl_properties_container + : public impl::medium_data { + using base_type = impl::medium_data; + using base_type::base_type; + + impl_properties_container( + const Kokkos::View elements, + const int ngllz, const int ngllx, + const specfem::mesh::materials &materials, const bool has_gll_model, + const specfem::kokkos::HostView1d property_index_mapping) + : impl_properties_container(elements.extent(0), ngllz, ngllx) { + + const int nelement = elements.extent(0); + int count = 0; + for (int i = 0; i < nelement; ++i) { + const int ispec = elements(i); + property_index_mapping(ispec) = count; + if (!has_gll_model) { + for (int iz = 0; iz < ngllz; ++iz) { + for (int ix = 0; ix < ngllx; ++ix) { + // Get the material at index from mesh::materials + auto material = + std::get >( + materials[ispec]); + + // Assign the material property to the property container + auto point_property = material.get_properties(); + this->store_host_values( + specfem::point::index(count, iz, ix), + point_property); + } + } + } + count++; + } + + if (!has_gll_model) { + this->copy_to_device(); + } + + return; + } + + template + KOKKOS_FORCEINLINE_FUNCTION void + add_device_values(const IndexType &index, PointValues &values) const = delete; + + template + KOKKOS_FORCEINLINE_FUNCTION void + add_host_values(const IndexType &index, PointValues &values) const = delete; +}; + template struct properties_container { diff --git a/include/mesh/materials/materials.hpp b/include/mesh/materials/materials.hpp index 06f447f1..fa500572 100644 --- a/include/mesh/materials/materials.hpp +++ b/include/mesh/materials/materials.hpp @@ -44,7 +44,7 @@ struct materials { struct material { int n_materials; ///< Number of elements std::vector > - material_properties; ///< Material properties + element_materials; ///< Material properties material() = default; diff --git a/include/mesh/materials/materials.tpp b/include/mesh/materials/materials.tpp index 867ed038..66259400 100644 --- a/include/mesh/materials/materials.tpp +++ b/include/mesh/materials/materials.tpp @@ -14,6 +14,6 @@ specfem::mesh::materials::material::material( const std::vector > &l_materials) : n_materials(n_materials), - material_properties(l_materials) {} + element_materials(l_materials) {} #endif /* end of include guard: _MESH_MATERIALS_TPP */ diff --git a/include/point/impl/point_data.hpp b/include/point/impl/point_data.hpp new file mode 100644 index 00000000..43f58276 --- /dev/null +++ b/include/point/impl/point_data.hpp @@ -0,0 +1,74 @@ +#pragma once + +#include "datatypes/simd.hpp" +#include "enumerations/medium.hpp" + +#define DEFINE_POINT_VALUE(prop) \ + constexpr static int i_##prop = __COUNTER__ - _counter - 1; \ + KOKKOS_INLINE_FUNCTION constexpr value_type prop() const { \ + return base_type::data[i_##prop]; \ + } \ + KOKKOS_INLINE_FUNCTION constexpr void prop(value_type &val) { \ + base_type::data[i_##prop] = val; \ + } + +namespace specfem { +namespace point { + +namespace impl { +template struct point_data { + using simd = specfem::datatype::simd; ///< SIMD type + using value_type = typename simd::datatype; + constexpr static auto nprops = N; + + value_type data[N]; + + KOKKOS_FUNCTION + point_data() = default; + + /** + * @brief array constructor + * + */ + KOKKOS_FUNCTION + point_data(const value_type *value) { + for (int i = 0; i < N; ++i) { + data[i] = value[i]; + } + } + + /** + * @brief value constructor + * + */ + template = 0> + KOKKOS_FUNCTION point_data(Args... args) : data{ args... } {} + + /** + * @brief Equality operator + * + */ + KOKKOS_FUNCTION + bool operator==(const point_data &rhs) const { + for (int i = 0; i < N; ++i) { + if (data[i] != rhs.data[i]) { + return false; + } + } + return true; + } + + /** + * @brief Inequality operator + * + */ + KOKKOS_FUNCTION + bool operator!=(const point_data &rhs) const { + return !(*this == rhs); + } +}; +} // namespace impl + +} // namespace point +} // namespace specfem diff --git a/include/point/kernels.hpp b/include/point/kernels.hpp index 43b0fdcb..26ee4db4 100644 --- a/include/point/kernels.hpp +++ b/include/point/kernels.hpp @@ -1,9 +1,6 @@ #pragma once -#include "datatypes/simd.hpp" -#include "enumerations/dimension.hpp" -#include "enumerations/medium.hpp" -#include +#include "impl/point_data.hpp" namespace specfem { namespace point { @@ -31,107 +28,40 @@ struct kernels; template struct kernels { -public: + specfem::element::property_tag::isotropic, UseSIMD> + : public impl::point_data<6, UseSIMD> { + /** * @name Typedefs * */ ///@{ - using simd = - typename specfem::datatype::simd; ///< SIMD type - using value_type = - typename simd::datatype; ///< Underlying data type to store the kernels - ///@} + using base_type = impl::point_data<6, UseSIMD>; + using value_type = typename base_type::value_type; -public: - /** - * @name Compile time constants - * - */ - ///@{ + constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::elastic; constexpr static auto property_tag = specfem::element::property_tag::isotropic; - constexpr static auto dimension = specfem::dimension::type::dim2; - ///@} - /** - * @name Misfit Kernels - * - */ - ///@{ - value_type rho; ///< \f$ K_{\rho} \f$ - value_type mu; ///< \f$ K_{\mu} \f$ - value_type kappa; ///< \f$ K_{\kappa} \f$ - value_type alpha; ///< \f$ K_{\alpha} \f$ - value_type beta; ///< \f$ K_{\beta} \f$ - value_type rhop; ///< \f$ K_{\rho'} \f$ + constexpr static bool is_point_properties = true; + constexpr static int _counter = __COUNTER__; ///@} - /** - * @name Constructors - * - */ - ///@{ - /** - * @brief Default constructor - * - */ - KOKKOS_FUNCTION - kernels() = default; - - /** - * @brief single value constructor - * - */ - KOKKOS_FUNCTION - kernels(const value_type value) - : rho(value), mu(value), kappa(value), alpha(value), beta(value), - rhop(value) {} + using base_type::base_type; /** - * @brief Constructor + * @name Misfit Kernels * - * @param rho \f$ K_{\rho} \f$ - * @param mu \f$ K_{\mu} \f$ - * @param kappa \f$ K_{\kappa} \f$ - * @param rhop \f$ K_{\rho'} \f$ - * @param alpha \f$ K_{\alpha} \f$ - * @param beta \f$ K_{\beta} \f$ */ - KOKKOS_FUNCTION - kernels(const value_type rho, const value_type mu, const value_type kappa, - const value_type rhop, const value_type alpha, const value_type beta) - : rho(rho), mu(mu), kappa(kappa), rhop(rhop), alpha(alpha), beta(beta) {} - + ///@{ + DEFINE_POINT_VALUE(rho) ///< \f$ K_{\rho} \f$ + DEFINE_POINT_VALUE(mu) ///< \f$ K_{\mu} \f$ + DEFINE_POINT_VALUE(kappa) ///< \f$ K_{\kappa} \f$ + DEFINE_POINT_VALUE(rhop) ///< \f$ K_{\rho'} \f$ + DEFINE_POINT_VALUE(alpha) ///< \f$ K_{\alpha} \f$ + DEFINE_POINT_VALUE(beta) ///< \f$ K_{\beta} \f$ ///@} - - /** - * @brief Equality operator - * - */ - KOKKOS_FUNCTION - bool operator==(const kernels &rhs) const { - return rho == rhs.rho && mu == rhs.mu && kappa == rhs.kappa && - rhop == rhs.rhop && alpha == rhs.alpha && beta == rhs.beta; - } - - /** - * @brief Inequality operator - * - */ - KOKKOS_FUNCTION - bool operator!=(const kernels &rhs) const { return !(*this == rhs); } - - KOKKOS_FUNCTION - bool operator==(const value_type value) { - return rho == value && mu == value && kappa == value && rhop == value && - alpha == value && beta == value; - } - - KOKKOS_FUNCTION - bool operator!=(const value_type value) { return !(*this == value); } }; // end elastic isotropic @@ -144,109 +74,41 @@ struct kernels struct kernels { -public: + specfem::element::property_tag::anisotropic, UseSIMD> + : public impl::point_data<7, UseSIMD> { + /** * @name Typedefs * */ ///@{ - using simd = - typename specfem::datatype::simd; ///< SIMD type - using value_type = - typename simd::datatype; ///< Underlying data type to store the kernels - ///@} + using base_type = impl::point_data<7, UseSIMD>; + using value_type = typename base_type::value_type; -public: - /** - * @name Compile time constants - * - */ - ///@{ + constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::elastic; constexpr static auto property_tag = specfem::element::property_tag::anisotropic; - constexpr static auto dimension = specfem::dimension::type::dim2; - ///@} - /** - * @name Misfit Kernels - * - */ - ///@{ - value_type rho; ///< \f$ K_{\rho} \f$ - value_type c11; ///< \f$ K_{c_{11}} \f$ - value_type c13; ///< \f$ K_{c_{13}} \f$ - value_type c15; ///< \f$ K_{c_{15}} \f$ - value_type c33; ///< \f$ K_{c_{33}} \f$ - value_type c35; ///< \f$ K_{c_{35}} \f$ - value_type c55; ///< \f$ K_{c_{55}} \f$ + constexpr static bool is_point_properties = true; + constexpr static int _counter = __COUNTER__; ///@} - /** - * @name Constructors - * - */ - ///@{ - /** - * @brief Default constructor - * - */ - KOKKOS_FUNCTION - kernels() = default; + using base_type::base_type; /** - * @brief single value constructor + * @name Misfit Kernels * */ - KOKKOS_FUNCTION - kernels(const value_type value) - : rho(value), c11(value), c13(value), c15(value), c33(value), c35(value), - c55(value) {} - - /** - * @brief Constructor - * - * @param rho \f$ K_{\rho} \f$ - * @param c11 \f$ K_{c_{11}} \f$ - * @param c13 \f$ K_{c_{13}} \f$ - * @param c15 \f$ K_{c_{15}} \f$ - * @param c33 \f$ K_{c_{33}} \f$ - * @param c35 \f$ K_{c_{35}} \f$ - * @param c55 \f$ K_{c_{55}} \f$ - */ - KOKKOS_FUNCTION - kernels(const value_type rho, const value_type c11, const value_type c13, - const value_type c15, const value_type c33, const value_type c35, - const value_type c55) - : rho(rho), c11(c11), c13(c13), c15(c15), c33(c33), c35(c35), c55(c55) {} + ///@{ + DEFINE_POINT_VALUE(rho) ///< \f$ K_{\rho} \f$ + DEFINE_POINT_VALUE(c11) ///< \f$ K_{c_{11}} \f$ + DEFINE_POINT_VALUE(c13) ///< \f$ K_{c_{13}} \f$ + DEFINE_POINT_VALUE(c15) ///< \f$ K_{c_{15}} \f$ + DEFINE_POINT_VALUE(c33) ///< \f$ K_{c_{33}} \f$ + DEFINE_POINT_VALUE(c35) ///< \f$ K_{c_{35}} \f$ + DEFINE_POINT_VALUE(c55) ///< \f$ K_{c_{55}} \f$ ///@} - - /** - * @brief Equality operator - * - */ - KOKKOS_FUNCTION - bool operator==(const kernels &rhs) const { - return rho == rhs.rho && c11 == rhs.c11 && c13 == rhs.c13 && - c15 == rhs.c15 && c33 == rhs.c33 && c35 == rhs.c35 && c55 == rhs.c55; - } - - /** - * @brief Inequality operator - * - */ - KOKKOS_FUNCTION - bool operator!=(const kernels &rhs) const { return !(*this == rhs); } - - KOKKOS_FUNCTION - bool operator==(const value_type value) { - return rho == value && c11 == value && c13 == value && c15 == value && - c33 == value && c35 == value && c55 == value; - } - - KOKKOS_FUNCTION - bool operator!=(const value_type value) { return !(*this == value); } }; // end elastic anisotropic @@ -259,62 +121,27 @@ struct kernels struct kernels { -public: + specfem::element::property_tag::isotropic, UseSIMD> + : public impl::point_data<4, UseSIMD> { + /** * @name Typedefs * */ ///@{ - using simd = typename specfem::datatype::simd; ///< SIMD - ///< type - using value_type = typename simd::datatype; ///< Underlying data type to store - ///< the kernels - ///@} + using base_type = impl::point_data<4, UseSIMD>; + using value_type = typename base_type::value_type; -public: - /** - * @name Compile time constants - * - */ - ///@{ + constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::acoustic; constexpr static auto property_tag = specfem::element::property_tag::isotropic; - constexpr static auto dimension = specfem::dimension::type::dim2; - ///@} -public: - /** - * @name Misfit Kernels - * - */ - ///@{ - value_type rho; ///< \f$ K_{\rho} \f$ - value_type kappa; ///< \f$ K_{\kappa} \f$ - value_type rhop; ///< \f$ K_{\rho'} \f$ - value_type alpha; ///< \f$ K_{\alpha} \f$ + constexpr static bool is_point_properties = true; + constexpr static int _counter = __COUNTER__; ///@} - /** - * @name Constructors - * - */ - ///@{ - /** - * @brief Default constructor - * - */ - KOKKOS_FUNCTION - kernels() = default; - - /** - * @brief single value constructor - * - */ - KOKKOS_FUNCTION - kernels(const value_type value) - : rho(value), kappa(value), rhop(value), alpha(value) {} + using base_type::base_type; /** * @brief Constructor @@ -324,36 +151,18 @@ struct kernels(2.0) * kappa; - } - ///@} - - /** - * @brief Equality operator - * - */ - KOKKOS_FUNCTION - bool operator==(const kernels &rhs) const { - return rho == rhs.rho && kappa == rhs.kappa && rhop == rhs.rhop && - alpha == rhs.alpha; - } + : kernels(rho, kappa, rho * kappa, static_cast(2.0) * kappa) {} /** - * @brief Inequality operator + * @name Misfit Kernels * */ - KOKKOS_FUNCTION - bool operator!=(const kernels &rhs) const { return !(*this == rhs); } - - KOKKOS_FUNCTION - bool operator==(const value_type value) { - return rho == value && kappa == value && rhop == value && alpha == value; - } - - KOKKOS_FUNCTION - bool operator!=(const value_type value) { return !(*this == value); } + ///@{ + DEFINE_POINT_VALUE(rho) ///< \f$ K_{\rho} \f$ + DEFINE_POINT_VALUE(kappa) ///< \f$ K_{\kappa} \f$ + DEFINE_POINT_VALUE(rhop) ///< \f$ K_{\rho'} \f$ + DEFINE_POINT_VALUE(alpha) ///< \f$ K_{\alpha} \f$ + ///@} }; } // namespace point diff --git a/include/point/properties.hpp b/include/point/properties.hpp index a6409523..b0fa45ce 100644 --- a/include/point/properties.hpp +++ b/include/point/properties.hpp @@ -1,8 +1,6 @@ -#ifndef _POINT_PROPERTIES_HPP -#define _POINT_PROPERTIES_HPP +#pragma once -#include "datatypes/simd.hpp" -#include "enumerations/medium.hpp" +#include "impl/point_data.hpp" namespace specfem { namespace point { @@ -28,216 +26,96 @@ struct properties; template struct properties { + specfem::element::property_tag::isotropic, UseSIMD> + : public impl::point_data<3, UseSIMD> { /** * @name Typedefs * */ ///@{ - constexpr static bool is_point_properties = true; - using simd = specfem::datatype::simd; ///< SIMD type + using base_type = impl::point_data<3, UseSIMD>; + using value_type = typename base_type::value_type; + constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::elastic; constexpr static auto property_tag = specfem::element::property_tag::isotropic; - using value_type = - typename simd::datatype; ///< Value type to store properties - ///@} - - value_type mu; ///< shear modulus @f$ \mu @f$ - value_type rho; ///< density @f$ \rho @f$ - - value_type rho_vp; ///< P-wave velocity @f$ \rho v_p @f$ - value_type rho_vs; ///< S-wave velocity @f$ \rho v_s @f$ - value_type lambda; ///< Lame's parameter @f$ \lambda @f$ - value_type lambdaplus2mu; ///< Lame's parameter @f$ \lambda + 2\mu @f$ - -private: - KOKKOS_FUNCTION - properties(const value_type &lambdaplus2mu, const value_type &mu, - const value_type &rho, std::false_type) - : lambdaplus2mu(lambdaplus2mu), mu(mu), rho(rho), - rho_vp(sqrt(rho * lambdaplus2mu)), rho_vs(sqrt(rho * mu)), - lambda(lambdaplus2mu - 2.0 * mu) {} - KOKKOS_FUNCTION - properties(const value_type &lambdaplus2mu, const value_type &mu, - const value_type &rho, std::true_type) - : lambdaplus2mu(lambdaplus2mu), mu(mu), rho(rho), - rho_vp(Kokkos::sqrt(rho * lambdaplus2mu)), - rho_vs(Kokkos::sqrt(rho * mu)), - lambda(lambdaplus2mu - (static_cast(2.0)) * mu) {} + constexpr static bool is_point_properties = true; + constexpr static int _counter = __COUNTER__; + ///@} -public: - /** - * @name Constructors - * - */ - ///@{ - /** - * @brief Default constructor - * - */ - KOKKOS_FUNCTION - properties() = default; + using base_type::base_type; - /** - * @brief Construct a new properties object - * - * @param lambdaplus2mu @f$ \lambda + 2\mu @f$ - * @param mu @f$ \mu @f$ - * @param rho @f$ \rho @f$ - */ - KOKKOS_FUNCTION - properties(const value_type &lambdaplus2mu, const value_type &mu, - const value_type &rho) - : properties(lambdaplus2mu, mu, rho, - std::integral_constant{}) {} + DEFINE_POINT_VALUE(lambdaplus2mu) ///< Lame's parameter @f$ \lambda + 2\mu @f$ + DEFINE_POINT_VALUE(mu) ///< shear modulus @f$ \mu @f$ + DEFINE_POINT_VALUE(rho) ///< density @f$ \rho @f$ - /** - * @brief single value constructor - * - */ - KOKKOS_FUNCTION - properties(const value_type value) - : properties(value, value, value, - std::integral_constant{}) {} - ///@} - - /** - * @brief Equality operator - * - */ - KOKKOS_FUNCTION - bool operator==(const properties &rhs) const { - return rho == rhs.rho && mu == rhs.mu && lambdaplus2mu == rhs.lambdaplus2mu; + KOKKOS_INLINE_FUNCTION constexpr value_type rho_vp() const { + return Kokkos::sqrt(rho() * lambdaplus2mu()); ///< P-wave velocity @f$ \rho + ///< v_p @f$ } - /** - * @brief Inequality operator - * - */ - KOKKOS_FUNCTION - bool operator!=(const properties &rhs) const { return !(*this == rhs); } - - KOKKOS_FUNCTION - bool operator==(const value_type value) { - return rho == value && mu == value && lambdaplus2mu == value; + KOKKOS_INLINE_FUNCTION constexpr value_type rho_vs() const { + return Kokkos::sqrt(rho() * mu()); ///< S-wave velocity @f$ \rho v_s @f$ } - KOKKOS_FUNCTION - bool operator!=(const value_type value) { return !(*this == value); } + KOKKOS_INLINE_FUNCTION constexpr value_type lambda() const { + return lambdaplus2mu() - (static_cast(2.0)) * + mu(); ///< Lame's parameter @f$ \lambda @f$ + } }; template struct properties { + specfem::element::property_tag::anisotropic, UseSIMD> + : public impl::point_data<10, UseSIMD> { /** * @name Typedefs * */ ///@{ - constexpr static bool is_point_properties = true; - using simd = specfem::datatype::simd; ///< SIMD type + using base_type = impl::point_data<10, UseSIMD>; + using value_type = typename base_type::value_type; + constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::elastic; constexpr static auto property_tag = specfem::element::property_tag::anisotropic; - using value_type = - typename simd::datatype; ///< Value type to store properties + + constexpr static bool is_point_properties = true; + constexpr static int _counter = __COUNTER__; ///@} + using base_type::base_type; + /** * @name Elastic constants * */ ///@{ - value_type c11; ///< @f$ c_{11} @f$ - value_type c13; ///< @f$ c_{13} @f$ - value_type c15; ///< @f$ c_{15} @f$ - value_type c33; ///< @f$ c_{33} @f$ - value_type c35; ///< @f$ c_{35} @f$ - value_type c55; ///< @f$ c_{55} @f$ - value_type c12; ///< @f$ c_{12} @f$ - value_type c23; ///< @f$ c_{23} @f$ - value_type c25; ///< @f$ c_{25} @f$ + DEFINE_POINT_VALUE(c11) ///< @f$ c_{11} @f$ + DEFINE_POINT_VALUE(c13) ///< @f$ c_{13} @f$ + DEFINE_POINT_VALUE(c15) ///< @f$ c_{15} @f$ + DEFINE_POINT_VALUE(c33) ///< @f$ c_{33} @f$ + DEFINE_POINT_VALUE(c35) ///< @f$ c_{35} @f$ + DEFINE_POINT_VALUE(c55) ///< @f$ c_{55} @f$ + DEFINE_POINT_VALUE(c12) ///< @f$ c_{12} @f$ + DEFINE_POINT_VALUE(c23) ///< @f$ c_{23} @f$ + DEFINE_POINT_VALUE(c25) ///< @f$ c_{25} @f$ + DEFINE_POINT_VALUE(rho) ///< Density @f$ \rho @f$ ///@} - value_type rho; ///< Density @f$ \rho @f$ - value_type rho_vp; ///< P-wave velocity @f$ \rho v_p @f$ - value_type rho_vs; ///< S-wave velocity @f$ \rho v_s @f$ - -private: - KOKKOS_FUNCTION - properties(const value_type &c11, const value_type &c13, - const value_type &c15, const value_type &c33, - const value_type &c35, const value_type &c55, - const value_type &c12, const value_type &c23, - const value_type &c25, const value_type &rho, std::false_type) - : c11(c11), c13(c13), c15(c15), c33(c33), c35(c35), c55(c55), c12(c12), - c23(c23), c25(c25), rho(rho), rho_vp(sqrt(rho * c33)), - rho_vs(sqrt(rho * c55)) {} - - KOKKOS_FUNCTION - properties(const value_type &c11, const value_type &c13, - const value_type &c15, const value_type &c33, - const value_type &c35, const value_type &c55, - const value_type &c12, const value_type &c23, - const value_type &c25, const value_type &rho, std::true_type) - : c11(c11), c13(c13), c15(c15), c33(c33), c35(c35), c55(c55), c12(c12), - c23(c23), c25(c25), rho(rho), rho_vp(Kokkos::sqrt(rho * c33)), - rho_vs(Kokkos::sqrt(rho * c55)) {} - -public: - KOKKOS_FUNCTION - properties() = default; - - KOKKOS_FUNCTION - properties(const value_type &c11, const value_type &c13, - const value_type &c15, const value_type &c33, - const value_type &c35, const value_type &c55, - const value_type &c12, const value_type &c23, - const value_type &c25, const type_real &rho) - : properties(c11, c13, c15, c33, c35, c55, c12, c23, c25, rho, - std::integral_constant{}) {} - - /** - * @brief single value constructor - * - */ - KOKKOS_FUNCTION - properties(const value_type value) - : properties(value, value, value, value, value, value, value, value, - value, value, std::integral_constant{}) {} - - /** - * @brief Equality operator - * - */ - KOKKOS_FUNCTION - bool operator==(const properties &rhs) const { - return rho == rhs.rho && c11 == rhs.c11 && c13 == rhs.c13 && - c15 == rhs.c15 && c33 == rhs.c33 && c35 == rhs.c35 && c55 == rhs.c55; + KOKKOS_INLINE_FUNCTION constexpr value_type rho_vp() const { + return Kokkos::sqrt(rho() * c33()); ///< P-wave velocity @f$ \rho v_p @f$ } - /** - * @brief Inequality operator - * - */ - KOKKOS_FUNCTION - bool operator!=(const properties &rhs) const { return !(*this == rhs); } - - KOKKOS_FUNCTION - bool operator==(const value_type value) { - return rho == value && c11 == value && c13 == value && c15 == value && - c33 == value && c35 == value && c55 == value; + KOKKOS_INLINE_FUNCTION constexpr value_type rho_vs() const { + return Kokkos::sqrt(rho() * c55()); ///< S-wave velocity @f$ \rho v_s @f$ } - - KOKKOS_FUNCTION - bool operator!=(const value_type value) { return !(*this == value); } }; /** @@ -248,103 +126,40 @@ struct properties struct properties { - + specfem::element::property_tag::isotropic, UseSIMD> + : public impl::point_data<2, UseSIMD> { /** * @name Typedefs * */ ///@{ - constexpr static bool is_point_properties = true; - using simd = specfem::datatype::simd; ///< SIMD type + using base_type = impl::point_data<2, UseSIMD>; + using value_type = typename base_type::value_type; + constexpr static auto dimension = specfem::dimension::type::dim2; constexpr static auto medium_tag = specfem::element::medium_tag::acoustic; constexpr static auto property_tag = specfem::element::property_tag::isotropic; - using value_type = - typename simd::datatype; ///< Value type to store properties + constexpr static bool is_point_properties = true; + constexpr static int _counter = __COUNTER__; ///@} - value_type kappa_inverse; ///< @f$ \frac{1}{\lambda + 2\mu} @f$ - value_type rho_inverse; ///< @f$ \frac{1}{\rho} @f$ - value_type kappa; ///< Bulk modulus @f$ \kappa @f$ - - value_type rho_vpinverse; ///< @f$ \frac{1}{\rho v_p} @f$ - -private: - KOKKOS_FUNCTION - properties(const value_type &rho_inverse, const value_type &kappa, - std::false_type) - : kappa_inverse(1.0 / kappa), rho_inverse(rho_inverse), kappa(kappa), - rho_vpinverse(sqrt(rho_inverse * kappa_inverse)) {} - - KOKKOS_FUNCTION - properties(const value_type &rho_inverse, const value_type &kappa, - std::true_type) - : kappa_inverse((static_cast(1.0)) / kappa), - rho_inverse(rho_inverse), kappa(kappa), - rho_vpinverse(Kokkos::sqrt(rho_inverse * kappa_inverse)) {} - -public: - /** - * @name Constructors - * - */ - ///@{ - - /** - * @brief Default constructor - * - */ - KOKKOS_FUNCTION - properties() = default; + using base_type::base_type; - /** - * @brief Construct a new properties object - * - * @param rho_inverse @f$ \frac{1}{\rho} @f$ - * @param kappa Bulk modulus @f$ \kappa @f$ - */ - KOKKOS_FUNCTION - properties(const value_type &rho_inverse, const value_type &kappa) - : properties(rho_inverse, kappa, - std::integral_constant{}) {} - /** - * @brief single value constructor - * - */ - KOKKOS_FUNCTION - properties(const value_type value) - : properties(value, value, std::integral_constant{}) {} - ///@} + DEFINE_POINT_VALUE(rho_inverse) ///< @f$ \frac{1}{\rho} @f$ + DEFINE_POINT_VALUE(kappa) ///< Bulk modulus @f$ \kappa @f$ - /** - * @brief Equality operator - * - */ - KOKKOS_FUNCTION - bool operator==(const properties &rhs) const { - return rho_inverse == rhs.rho_inverse && kappa == rhs.kappa; + KOKKOS_INLINE_FUNCTION constexpr value_type kappa_inverse() const { + return (static_cast(1.0)) / + kappa(); ///< @f$ \frac{1}{\lambda + 2\mu} @f$ } - /** - * @brief Inequality operator - * - */ - KOKKOS_FUNCTION - bool operator!=(const properties &rhs) const { return !(*this == rhs); } - - KOKKOS_FUNCTION - bool operator==(const value_type value) { - return rho_inverse == value && kappa == value; + KOKKOS_INLINE_FUNCTION constexpr value_type rho_vpinverse() const { + return Kokkos::sqrt(rho_inverse() * kappa_inverse()); ///< @f$ \frac{1}{\rho + ///< v_p} @f$ } - - KOKKOS_FUNCTION - bool operator!=(const value_type value) { return !(*this == value); } }; } // namespace point } // namespace specfem - -#endif diff --git a/include/policies/chunk.hpp b/include/policies/chunk.hpp index 018b99a6..a112f5f5 100644 --- a/include/policies/chunk.hpp +++ b/include/policies/chunk.hpp @@ -63,14 +63,14 @@ struct chunk_index_type { template struct mapped_chunk_index_type : public chunk_index_type { - using Base = chunk_index_type; + using base_type = chunk_index_type; int imap; ///< Index of the mapped element KOKKOS_INLINE_FUNCTION mapped_chunk_index_type(const int ielement, const specfem::point::index index, const int imap) - : Base(ielement, index), imap(imap) {} + : base_type(ielement, index), imap(imap) {} }; } // namespace impl @@ -252,20 +252,20 @@ class mapped_chunk; template class mapped_chunk : public chunk { - using Base = chunk; + using base_type = chunk; using mapped_index_type = - typename impl::mapped_chunk_index_type; ///< Index + typename impl::mapped_chunk_index_type; ///< Index public: KOKKOS_INLINE_FUNCTION mapped_chunk(const ViewType &indices, const ViewType &mapping, const int ngllz, const int ngllx) - : Base(indices, ngllz, ngllx), mapping(mapping) {} + : base_type(indices, ngllz, ngllx), mapping(mapping) {} KOKKOS_INLINE_FUNCTION mapped_index_type operator()(const int i) const { - const auto base_index = Base::operator()(i); + const auto base_index = base_type::operator()(i); return mapped_index_type(base_index.ielement, base_index.index, mapping(base_index.ielement)); } @@ -405,8 +405,8 @@ struct element_chunk template struct mapped_element_chunk : public element_chunk { using simd = typename ParallelConfig::simd; - using Base = element_chunk; - using IndexViewType = typename Base::IndexViewType; + using base_type = element_chunk; + using IndexViewType = typename base_type::IndexViewType; using mapped_iterator_type = specfem::iterator::mapped_chunk { mapped_element_chunk(const IndexViewType &view, const IndexViewType &mapping, int ngllz, int ngllx) - : Base(view, ngllz, ngllx), mapping(mapping) {} + : base_type(view, ngllz, ngllx), mapping(mapping) {} KOKKOS_INLINE_FUNCTION mapped_iterator_type mapped_league_iterator(const int start_index) const { const int start = start_index; - const int end = - (start + Base::chunk_size * Base::simd_size > Base::elements.extent(0)) - ? Base::elements.extent(0) - : start + Base::chunk_size * Base::simd_size; + const int end = (start + base_type::chunk_size * base_type::simd_size > + base_type::elements.extent(0)) + ? base_type::elements.extent(0) + : start + base_type::chunk_size * base_type::simd_size; const auto elem_indices = - Kokkos::subview(Base::elements, Kokkos::make_pair(start, end)); + Kokkos::subview(base_type::elements, Kokkos::make_pair(start, end)); const auto map_indices = Kokkos::subview(mapping, Kokkos::make_pair(start, end)); - return mapped_iterator_type(elem_indices, map_indices, Base::ngllz, - Base::ngllx); + return mapped_iterator_type(elem_indices, map_indices, base_type::ngllz, + base_type::ngllx); } protected: diff --git a/src/IO/kernel/writer.cpp b/src/IO/kernel/writer.cpp index dd3181b1..e6cc64e0 100644 --- a/src/IO/kernel/writer.cpp +++ b/src/IO/kernel/writer.cpp @@ -1,10 +1,13 @@ #include "IO/kernel/writer.hpp" #include "IO/ASCII/ASCII.hpp" #include "IO/HDF5/HDF5.hpp" +#include "IO/impl/medium_writer.tpp" #include "IO/kernel/writer.tpp" // Explicit instantiation -template class specfem::IO::kernel_writer >; +template class specfem::IO::kernel_writer< + specfem::IO::HDF5 >; -template class specfem::IO::kernel_writer >; +template class specfem::IO::kernel_writer< + specfem::IO::ASCII >; diff --git a/src/IO/mesh.cpp b/src/IO/mesh.cpp index 29021be5..8a10399c 100644 --- a/src/IO/mesh.cpp +++ b/src/IO/mesh.cpp @@ -194,12 +194,12 @@ specfem::IO::read_mesh(const std::string filename, std::to_string(mesh.materials.n_materials) + "\n\n"); const auto l_elastic_isotropic = - mesh.materials.elastic_isotropic.material_properties; + mesh.materials.elastic_isotropic.element_materials; const auto l_acoustic_isotropic = - mesh.materials.acoustic_isotropic.material_properties; + mesh.materials.acoustic_isotropic.element_materials; const auto l_elastic_anisotropic = - mesh.materials.elastic_anisotropic.material_properties; + mesh.materials.elastic_anisotropic.element_materials; for (const auto material : l_elastic_isotropic) { mpi->cout(material.print()); diff --git a/src/IO/property/writer.cpp b/src/IO/property/writer.cpp index 0bd22db4..b34a5452 100644 --- a/src/IO/property/writer.cpp +++ b/src/IO/property/writer.cpp @@ -1,6 +1,7 @@ #include "IO/property/writer.hpp" #include "IO/ASCII/ASCII.hpp" #include "IO/HDF5/HDF5.hpp" +#include "IO/impl/medium_writer.tpp" #include "IO/property/writer.tpp" // Explicit instantiation diff --git a/src/compute/compute_kernels.cpp b/src/compute/compute_kernels.cpp index c0580fdc..c507dea0 100644 --- a/src/compute/compute_kernels.cpp +++ b/src/compute/compute_kernels.cpp @@ -31,17 +31,17 @@ specfem::compute::kernels::kernels( h_property_index_mapping(ispec) = -1; } - acoustic_isotropic = specfem::medium::material_kernels< + acoustic_isotropic = specfem::medium::kernels_container< specfem::element::medium_tag::acoustic, specfem::element::property_tag::isotropic>( acoustic_elements, ngllz, ngllx, h_property_index_mapping); - elastic_isotropic = specfem::medium::material_kernels< + elastic_isotropic = specfem::medium::kernels_container< specfem::element::medium_tag::elastic, specfem::element::property_tag::isotropic>( elastic_isotropic_elements, ngllz, ngllx, h_property_index_mapping); - elastic_anisotropic = specfem::medium::material_kernels< + elastic_anisotropic = specfem::medium::kernels_container< specfem::element::medium_tag::elastic, specfem::element::property_tag::anisotropic>( elastic_anisotropic_elements, ngllz, ngllx, h_property_index_mapping); diff --git a/src/compute/compute_properties.cpp b/src/compute/compute_properties.cpp index f21b2c0f..51b1eeb9 100644 --- a/src/compute/compute_properties.cpp +++ b/src/compute/compute_properties.cpp @@ -31,19 +31,19 @@ specfem::compute::properties::properties( h_property_index_mapping(ispec) = -1; } - acoustic_isotropic = specfem::medium::material_properties< + acoustic_isotropic = specfem::medium::properties_container< specfem::element::medium_tag::acoustic, specfem::element::property_tag::isotropic>( acoustic_elements, ngllz, ngllx, materials, has_gll_model, h_property_index_mapping); - elastic_isotropic = specfem::medium::material_properties< + elastic_isotropic = specfem::medium::properties_container< specfem::element::medium_tag::elastic, specfem::element::property_tag::isotropic>( elastic_isotropic_elements, ngllz, ngllx, materials, has_gll_model, h_property_index_mapping); - elastic_anisotropic = specfem::medium::material_properties< + elastic_anisotropic = specfem::medium::properties_container< specfem::element::medium_tag::elastic, specfem::element::property_tag::anisotropic>( elastic_anisotropic_elements, ngllz, ngllx, materials, has_gll_model, diff --git a/src/mesh/materials/materials.cpp b/src/mesh/materials/materials.cpp index d2fb9ba0..6eeac712 100644 --- a/src/mesh/materials/materials.cpp +++ b/src/mesh/materials/materials.cpp @@ -22,7 +22,7 @@ specfem::mesh::materials::operator[](const int index) const { material_specification.property == specfem::element::property_tag::isotropic) { return this->elastic_isotropic - .material_properties[material_specification.index]; + .element_materials[material_specification.index]; // Return elastic anisotropic } else if (material_specification.type == @@ -30,7 +30,7 @@ specfem::mesh::materials::operator[](const int index) const { material_specification.property == specfem::element::property_tag::anisotropic) { return this->elastic_anisotropic - .material_properties[material_specification.index]; + .element_materials[material_specification.index]; // Return acoustic isotropic } else if (material_specification.type == @@ -38,7 +38,7 @@ specfem::mesh::materials::operator[](const int index) const { material_specification.property == specfem::element::property_tag::isotropic) { return this->acoustic_isotropic - .material_properties[material_specification.index]; + .element_materials[material_specification.index]; // Throw an error if the material type is not supported } else { throw std::runtime_error("Material type not supported"); diff --git a/tests/unit-tests/assembly/compute_wavefield/test_helper.hpp b/tests/unit-tests/assembly/compute_wavefield/test_helper.hpp index 52d45d06..58ad19e6 100644 --- a/tests/unit-tests/assembly/compute_wavefield/test_helper.hpp +++ b/tests/unit-tests/assembly/compute_wavefield/test_helper.hpp @@ -111,7 +111,7 @@ class test_helper 1.0e-4) { diff --git a/tests/unit-tests/assembly/kernels/kernels.cpp b/tests/unit-tests/assembly/kernels/kernels.cpp index f3aca0af..4eb8ba30 100644 --- a/tests/unit-tests/assembly/kernels/kernels.cpp +++ b/tests/unit-tests/assembly/kernels/kernels.cpp @@ -24,12 +24,12 @@ std::string get_error_message( message << "\n\t Expected: " << value; message << "\n\t Got: \n"; - message << "\t\trho = " << point_kernel.rho << "\n"; - message << "\t\tmu = " << point_kernel.mu << "\n"; - message << "\t\tkappa = " << point_kernel.kappa << "\n"; - message << "\t\trhop = " << point_kernel.rhop << "\n"; - message << "\t\talpha = " << point_kernel.alpha << "\n"; - message << "\t\tbeta = " << point_kernel.beta << "\n"; + message << "\t\trho = " << point_kernel.rho() << "\n"; + message << "\t\tmu = " << point_kernel.mu() << "\n"; + message << "\t\tkappa = " << point_kernel.kappa() << "\n"; + message << "\t\trhop = " << point_kernel.rhop() << "\n"; + message << "\t\talpha = " << point_kernel.alpha() << "\n"; + message << "\t\tbeta = " << point_kernel.beta() << "\n"; return message.str(); } @@ -44,13 +44,13 @@ std::string get_error_message( message << "\n\t Expected: " << value; message << "\n\t Got: \n"; - message << "\t\trho = " << point_kernel.rho << "\n"; - message << "\t\tc11 = " << point_kernel.c11 << "\n"; - message << "\t\tc13 = " << point_kernel.c13 << "\n"; - message << "\t\tc15 = " << point_kernel.c15 << "\n"; - message << "\t\tc33 = " << point_kernel.c33 << "\n"; - message << "\t\tc35 = " << point_kernel.c35 << "\n"; - message << "\t\tc55 = " << point_kernel.c55 << "\n"; + message << "\t\trho = " << point_kernel.rho() << "\n"; + message << "\t\tc11 = " << point_kernel.c11() << "\n"; + message << "\t\tc13 = " << point_kernel.c13() << "\n"; + message << "\t\tc15 = " << point_kernel.c15() << "\n"; + message << "\t\tc33 = " << point_kernel.c33() << "\n"; + message << "\t\tc35 = " << point_kernel.c35() << "\n"; + message << "\t\tc55 = " << point_kernel.c55() << "\n"; return message.str(); } @@ -65,10 +65,10 @@ std::string get_error_message( message << "\n\t Expected: " << value; message << "\n\t Got: \n"; - message << "\t\trho = " << point_kernel.rho << "\n"; - message << "\t\tkappa = " << point_kernel.kappa << "\n"; - message << "\t\trhop = " << point_kernel.rhop << "\n"; - message << "\t\talpha = " << point_kernel.alpha << "\n"; + message << "\t\trho = " << point_kernel.rho() << "\n"; + message << "\t\tkappa = " << point_kernel.kappa() << "\n"; + message << "\t\trhop = " << point_kernel.rhop() << "\n"; + message << "\t\talpha = " << point_kernel.alpha() << "\n"; return message.str(); } @@ -105,12 +105,12 @@ get_point_kernel(const int ispec, const int iz, const int ix, specfem::element::property_tag::isotropic, false> point_kernel; - point_kernel.rho = elastic_isotropic.h_rho(ispec_l, iz, ix); - point_kernel.mu = elastic_isotropic.h_mu(ispec_l, iz, ix); - point_kernel.kappa = elastic_isotropic.h_kappa(ispec_l, iz, ix); - point_kernel.rhop = elastic_isotropic.h_rhop(ispec_l, iz, ix); - point_kernel.alpha = elastic_isotropic.h_alpha(ispec_l, iz, ix); - point_kernel.beta = elastic_isotropic.h_beta(ispec_l, iz, ix); + point_kernel.rho(elastic_isotropic.h_rho(ispec_l, iz, ix)); + point_kernel.mu(elastic_isotropic.h_mu(ispec_l, iz, ix)); + point_kernel.kappa(elastic_isotropic.h_kappa(ispec_l, iz, ix)); + point_kernel.rhop(elastic_isotropic.h_rhop(ispec_l, iz, ix)); + point_kernel.alpha(elastic_isotropic.h_alpha(ispec_l, iz, ix)); + point_kernel.beta(elastic_isotropic.h_beta(ispec_l, iz, ix)); return point_kernel; } @@ -129,12 +129,12 @@ get_point_kernel( specfem::element::property_tag::isotropic, false> point_kernel_l; - point_kernel_l.rho = point_kernel.rho[lane]; - point_kernel_l.mu = point_kernel.mu[lane]; - point_kernel_l.kappa = point_kernel.kappa[lane]; - point_kernel_l.rhop = point_kernel.rhop[lane]; - point_kernel_l.alpha = point_kernel.alpha[lane]; - point_kernel_l.beta = point_kernel.beta[lane]; + point_kernel_l.rho(point_kernel.rho()[lane]); + point_kernel_l.mu(point_kernel.mu()[lane]); + point_kernel_l.kappa(point_kernel.kappa()[lane]); + point_kernel_l.rhop(point_kernel.rhop()[lane]); + point_kernel_l.alpha(point_kernel.alpha()[lane]); + point_kernel_l.beta(point_kernel.beta()[lane]); return point_kernel_l; } @@ -155,13 +155,13 @@ get_point_kernel(const int ispec, const int iz, const int ix, specfem::element::property_tag::anisotropic, false> point_kernel; - point_kernel.rho = elastic_anisotropic.h_rho(ispec_l, iz, ix); - point_kernel.c11 = elastic_anisotropic.h_c11(ispec_l, iz, ix); - point_kernel.c13 = elastic_anisotropic.h_c13(ispec_l, iz, ix); - point_kernel.c15 = elastic_anisotropic.h_c15(ispec_l, iz, ix); - point_kernel.c33 = elastic_anisotropic.h_c33(ispec_l, iz, ix); - point_kernel.c35 = elastic_anisotropic.h_c35(ispec_l, iz, ix); - point_kernel.c55 = elastic_anisotropic.h_c55(ispec_l, iz, ix); + point_kernel.rho(elastic_anisotropic.h_rho(ispec_l, iz, ix)); + point_kernel.c11(elastic_anisotropic.h_c11(ispec_l, iz, ix)); + point_kernel.c13(elastic_anisotropic.h_c13(ispec_l, iz, ix)); + point_kernel.c15(elastic_anisotropic.h_c15(ispec_l, iz, ix)); + point_kernel.c33(elastic_anisotropic.h_c33(ispec_l, iz, ix)); + point_kernel.c35(elastic_anisotropic.h_c35(ispec_l, iz, ix)); + point_kernel.c55(elastic_anisotropic.h_c55(ispec_l, iz, ix)); return point_kernel; } @@ -180,13 +180,13 @@ get_point_kernel( specfem::element::property_tag::anisotropic, false> point_kernel_l; - point_kernel_l.rho = point_kernel.rho[lane]; - point_kernel_l.c11 = point_kernel.c11[lane]; - point_kernel_l.c13 = point_kernel.c13[lane]; - point_kernel_l.c15 = point_kernel.c15[lane]; - point_kernel_l.c33 = point_kernel.c33[lane]; - point_kernel_l.c35 = point_kernel.c35[lane]; - point_kernel_l.c55 = point_kernel.c55[lane]; + point_kernel_l.rho(point_kernel.rho()[lane]); + point_kernel_l.c11(point_kernel.c11()[lane]); + point_kernel_l.c13(point_kernel.c13()[lane]); + point_kernel_l.c15(point_kernel.c15()[lane]); + point_kernel_l.c33(point_kernel.c33()[lane]); + point_kernel_l.c35(point_kernel.c35()[lane]); + point_kernel_l.c55(point_kernel.c55()[lane]); return point_kernel_l; } @@ -207,10 +207,10 @@ get_point_kernel(const int ispec, const int iz, const int ix, specfem::element::property_tag::isotropic, false> point_kernel; - point_kernel.rho = acoustic_isotropic.h_rho(ispec_l, iz, ix); - point_kernel.kappa = acoustic_isotropic.h_kappa(ispec_l, iz, ix); - point_kernel.alpha = acoustic_isotropic.h_alpha(ispec_l, iz, ix); - point_kernel.rhop = acoustic_isotropic.h_rho_prime(ispec_l, iz, ix); + point_kernel.rho(acoustic_isotropic.h_rho(ispec_l, iz, ix)); + point_kernel.kappa(acoustic_isotropic.h_kappa(ispec_l, iz, ix)); + point_kernel.alpha(acoustic_isotropic.h_alpha(ispec_l, iz, ix)); + point_kernel.rhop(acoustic_isotropic.h_rhop(ispec_l, iz, ix)); return point_kernel; } @@ -229,10 +229,10 @@ get_point_kernel( specfem::element::property_tag::isotropic, false> point_kernel_l; - point_kernel_l.rho = point_kernel.rho[lane]; - point_kernel_l.kappa = point_kernel.kappa[lane]; - point_kernel_l.alpha = point_kernel.alpha[lane]; - point_kernel_l.rhop = point_kernel.rhop[lane]; + point_kernel_l.rho(point_kernel.rho()[lane]); + point_kernel_l.kappa(point_kernel.kappa()[lane]); + point_kernel_l.alpha(point_kernel.alpha()[lane]); + point_kernel_l.rhop(point_kernel.rhop()[lane]); return point_kernel_l; } @@ -254,6 +254,10 @@ void check_to_value(const specfem::compute::element_types &element_types, constexpr int simd_size = specfem::datatype::simd::size(); + using PointType = specfem::point::kernels; + constexpr int nprops = PointType::nprops; + for (int i = 0; i < ispecs.extent(0); ++i) { for (int iz = 0; iz < ngllz; iz++) { for (int ix = 0; ix < ngllx; ix++) { @@ -265,14 +269,16 @@ void check_to_value(const specfem::compute::element_types &element_types, const auto point_kernel = get_point_kernel( ielement + j, iz, ix, kernels); const type_real value = values_to_store(i); - if (point_kernel != value) { - std::ostringstream message; + for (int l = 0; l < nprops; l++) { + if (point_kernel.data[l] != value) { + std::ostringstream message; - message << "\n \t Error at ispec = " << ielement + j - << ", iz = " << iz << ", ix = " << ix; - message << get_error_message(point_kernel, value); + message << "\n \t Error at ispec = " << ielement + j + << ", iz = " << iz << ", ix = " << ix; + message << get_error_message(point_kernel, value); - throw std::runtime_error(message.str()); + throw std::runtime_error(message.str()); + } } } } @@ -313,7 +319,10 @@ void execute_store_or_add(specfem::compute::kernels &kernels, const auto index = get_index(ielement, n_simd_elements, iz, ix); const type_real value = values_to_store(i); - PointType point(value); + PointType point; + for (int l = 0; l < PointType::nprops; l++) { + point.data[l] = value; + } if constexpr (Store) { specfem::compute::store_on_device(index, point, kernels_l); } else if constexpr (Add) { @@ -411,6 +420,7 @@ void check_load_on_device( const int element_size = elements.extent(0); const int step = element_size / N; + constexpr int nprops = PointType::nprops; for (int i = 0; i < N; i++) { ispecs_h(i) = elements(i * step); @@ -459,29 +469,34 @@ void check_load_on_device( if constexpr (using_simd) { for (int lane = 0; lane < n_simd_elements; lane++) { const auto point_kernel_l = get_point_kernel(lane, point_kernel); - if (point_kernel_l != value_l) { + for (int l = 0; l < nprops; l++) { + if (point_kernel_l.data[l] != value_l) { + std::ostringstream message; + + message << "\n \t Error in function load_on_device"; + + message << "\n \t Error at ispec = " << ielement + << ", iz = " << 0 << ", ix = " << 0; + message << get_error_message(point_kernel_l, value_l); + + throw std::runtime_error(message.str()); + } + } + } + } else if constexpr (!using_simd) { + for (int l = 0; l < nprops; l++) { + if (point_kernel.data[l] != value_l) { std::ostringstream message; message << "\n \t Error in function load_on_device"; message << "\n \t Error at ispec = " << ielement << ", iz = " << 0 << ", ix = " << 0; - message << get_error_message(point_kernel_l, value_l); + message << get_error_message(point_kernel, value_l); throw std::runtime_error(message.str()); } } - } else if constexpr (!using_simd) { - if (point_kernel != value_l) { - std::ostringstream message; - message << "\n \t Error in function load_on_device"; - - message << "\n \t Error at ispec = " << ielement << ", iz = " << 0 - << ", ix = " << 0; - message << get_error_message(point_kernel, value_l); - - throw std::runtime_error(message.str()); - } } } } diff --git a/tests/unit-tests/assembly/properties/properties.cpp b/tests/unit-tests/assembly/properties/properties.cpp index c047bb02..abe828ac 100644 --- a/tests/unit-tests/assembly/properties/properties.cpp +++ b/tests/unit-tests/assembly/properties/properties.cpp @@ -38,9 +38,9 @@ std::string get_error_message( std::ostringstream message; error_message_header(message, value, mode); - message << "\t\trho = " << point_property.rho << "\n"; - message << "\t\tmu = " << point_property.mu << "\n"; - message << "\t\tkappa = " << point_property.lambdaplus2mu << "\n"; + message << "\t\trho = " << point_property.rho() << "\n"; + message << "\t\tmu = " << point_property.mu() << "\n"; + message << "\t\tlambdaplus2mu = " << point_property.lambdaplus2mu() << "\n"; return message.str(); } @@ -54,13 +54,13 @@ std::string get_error_message( std::ostringstream message; error_message_header(message, value, mode); - message << "\t\trho = " << point_property.rho << "\n"; - message << "\t\tc11 = " << point_property.c11 << "\n"; - message << "\t\tc13 = " << point_property.c13 << "\n"; - message << "\t\tc15 = " << point_property.c15 << "\n"; - message << "\t\tc33 = " << point_property.c33 << "\n"; - message << "\t\tc35 = " << point_property.c35 << "\n"; - message << "\t\tc55 = " << point_property.c55 << "\n"; + message << "\t\trho = " << point_property.rho() << "\n"; + message << "\t\tc11 = " << point_property.c11() << "\n"; + message << "\t\tc13 = " << point_property.c13() << "\n"; + message << "\t\tc15 = " << point_property.c15() << "\n"; + message << "\t\tc33 = " << point_property.c33() << "\n"; + message << "\t\tc35 = " << point_property.c35() << "\n"; + message << "\t\tc55 = " << point_property.c55() << "\n"; return message.str(); } @@ -74,8 +74,8 @@ std::string get_error_message( std::ostringstream message; error_message_header(message, value, mode); - message << "\t\trho_inverse = " << point_property.rho_inverse << "\n"; - message << "\t\tkappa = " << point_property.kappa << "\n"; + message << "\t\trho_inverse = " << point_property.rho_inverse() << "\n"; + message << "\t\tkappa = " << point_property.kappa() << "\n"; return message.str(); } @@ -112,10 +112,10 @@ get_point_property(const int ispec, const int iz, const int ix, specfem::element::property_tag::isotropic, false> point_property; - point_property.rho = elastic_isotropic.h_rho(ispec_l, iz, ix); - point_property.mu = elastic_isotropic.h_mu(ispec_l, iz, ix); - point_property.lambdaplus2mu = - elastic_isotropic.h_lambdaplus2mu(ispec_l, iz, ix); + point_property.rho(elastic_isotropic.h_rho(ispec_l, iz, ix)); + point_property.mu(elastic_isotropic.h_mu(ispec_l, iz, ix)); + point_property.lambdaplus2mu( + elastic_isotropic.h_lambdaplus2mu(ispec_l, iz, ix)); return point_property; } @@ -134,9 +134,9 @@ get_point_property( specfem::element::property_tag::isotropic, false> point_property_l; - point_property_l.rho = point_property.rho[lane]; - point_property_l.mu = point_property.mu[lane]; - point_property_l.lambdaplus2mu = point_property.lambdaplus2mu[lane]; + point_property_l.rho(point_property.rho()[lane]); + point_property_l.mu(point_property.mu()[lane]); + point_property_l.lambdaplus2mu(point_property.lambdaplus2mu()[lane]); return point_property_l; } @@ -157,13 +157,16 @@ get_point_property(const int ispec, const int iz, const int ix, specfem::element::property_tag::anisotropic, false> point_property; - point_property.rho = elastic_anisotropic.h_rho(ispec_l, iz, ix); - point_property.c11 = elastic_anisotropic.h_c11(ispec_l, iz, ix); - point_property.c13 = elastic_anisotropic.h_c13(ispec_l, iz, ix); - point_property.c15 = elastic_anisotropic.h_c15(ispec_l, iz, ix); - point_property.c33 = elastic_anisotropic.h_c33(ispec_l, iz, ix); - point_property.c35 = elastic_anisotropic.h_c35(ispec_l, iz, ix); - point_property.c55 = elastic_anisotropic.h_c55(ispec_l, iz, ix); + point_property.rho(elastic_anisotropic.h_rho(ispec_l, iz, ix)); + point_property.c11(elastic_anisotropic.h_c11(ispec_l, iz, ix)); + point_property.c13(elastic_anisotropic.h_c13(ispec_l, iz, ix)); + point_property.c15(elastic_anisotropic.h_c15(ispec_l, iz, ix)); + point_property.c33(elastic_anisotropic.h_c33(ispec_l, iz, ix)); + point_property.c35(elastic_anisotropic.h_c35(ispec_l, iz, ix)); + point_property.c55(elastic_anisotropic.h_c55(ispec_l, iz, ix)); + point_property.c12(elastic_anisotropic.h_c12(ispec_l, iz, ix)); + point_property.c23(elastic_anisotropic.h_c23(ispec_l, iz, ix)); + point_property.c25(elastic_anisotropic.h_c25(ispec_l, iz, ix)); return point_property; } @@ -182,13 +185,16 @@ get_point_property( specfem::element::property_tag::anisotropic, false> point_property_l; - point_property_l.rho = point_property.rho[lane]; - point_property_l.c11 = point_property.c11[lane]; - point_property_l.c13 = point_property.c13[lane]; - point_property_l.c15 = point_property.c15[lane]; - point_property_l.c33 = point_property.c33[lane]; - point_property_l.c35 = point_property.c35[lane]; - point_property_l.c55 = point_property.c55[lane]; + point_property_l.rho(point_property.rho()[lane]); + point_property_l.c11(point_property.c11()[lane]); + point_property_l.c13(point_property.c13()[lane]); + point_property_l.c15(point_property.c15()[lane]); + point_property_l.c33(point_property.c33()[lane]); + point_property_l.c35(point_property.c35()[lane]); + point_property_l.c55(point_property.c55()[lane]); + point_property_l.c12(point_property.c12()[lane]); + point_property_l.c23(point_property.c23()[lane]); + point_property_l.c25(point_property.c25()[lane]); return point_property_l; } @@ -209,9 +215,8 @@ get_point_property(const int ispec, const int iz, const int ix, specfem::element::property_tag::isotropic, false> point_property; - point_property.rho_inverse = - acoustic_isotropic.h_rho_inverse(ispec_l, iz, ix); - point_property.kappa = acoustic_isotropic.h_kappa(ispec_l, iz, ix); + point_property.rho_inverse(acoustic_isotropic.h_rho_inverse(ispec_l, iz, ix)); + point_property.kappa(acoustic_isotropic.h_kappa(ispec_l, iz, ix)); return point_property; } @@ -230,8 +235,8 @@ get_point_property( specfem::element::property_tag::isotropic, false> point_property_l; - point_property_l.rho_inverse = point_property.rho_inverse[lane]; - point_property_l.kappa = point_property.kappa[lane]; + point_property_l.rho_inverse(point_property.rho_inverse()[lane]); + point_property_l.kappa(point_property.kappa()[lane]); return point_property_l; } @@ -240,13 +245,13 @@ template void check_eq( const typename specfem::datatype::simd::datatype &p1, const typename specfem::datatype::simd::datatype &p2, - const int &n_simd_elements) { + const int &n_simd_elements, const std::string &msg) { if constexpr (using_simd) { for (int i = 0; i < n_simd_elements; i++) { if (p1[i] != p2[i]) { std::ostringstream message; - message << "\n \t Error in function load_on_host"; + message << "\n \t Error in function load_on_host (SIMD) | " << msg; message << "\n\t Expected: " << p1[i]; message << "\n\t Got: " << p2[i]; @@ -257,7 +262,7 @@ void check_eq( if (p1 != p2) { std::ostringstream message; - message << "\n \t Error in function load_on_host"; + message << "\n \t Error in function load_on_host | " << msg; message << "\n\t Expected: " << p1; message << "\n\t Got: " << p2; @@ -284,19 +289,20 @@ void check_point_properties( specfem::dimension::type::dim2, specfem::element::medium_tag::elastic, specfem::element::property_tag::isotropic, using_simd> &p2, const int &n_simd_elements) { - check_eq(p1.rho, p2.rho, n_simd_elements); - check_eq(p1.mu, p2.mu, n_simd_elements); - check_eq(p1.lambdaplus2mu, p2.lambdaplus2mu, n_simd_elements); - check_eq(p1.lambda, - p2.lambdaplus2mu - + check_eq(p1.rho(), p2.rho(), n_simd_elements, "rho"); + check_eq(p1.mu(), p2.mu(), n_simd_elements, ".mu"); + check_eq(p1.lambdaplus2mu(), p2.lambdaplus2mu(), n_simd_elements, + "lambdaplus2mu"); + check_eq(p1.lambda(), + p2.lambdaplus2mu() - (static_cast::datatype>(2.0)) * - p2.mu, - n_simd_elements); - check_eq(p1.rho_vp, Kokkos::sqrt(p2.rho * p2.lambdaplus2mu), - n_simd_elements); - check_eq(p1.rho_vs, Kokkos::sqrt(p2.rho * p2.mu), - n_simd_elements); + p2.mu(), + n_simd_elements, "lambda"); + check_eq(p1.rho_vp(), Kokkos::sqrt(p2.rho() * p2.lambdaplus2mu()), + n_simd_elements, "rho_vp"); + check_eq(p1.rho_vs(), Kokkos::sqrt(p2.rho() * p2.mu()), + n_simd_elements, "rho_vp"); } template @@ -308,17 +314,17 @@ void check_point_properties( specfem::dimension::type::dim2, specfem::element::medium_tag::elastic, specfem::element::property_tag::anisotropic, using_simd> &p2, const int &n_simd_elements) { - check_eq(p1.rho, p2.rho, n_simd_elements); - check_eq(p1.c11, p2.c11, n_simd_elements); - check_eq(p1.c13, p2.c13, n_simd_elements); - check_eq(p1.c15, p2.c15, n_simd_elements); - check_eq(p1.c33, p2.c33, n_simd_elements); - check_eq(p1.c35, p2.c35, n_simd_elements); - check_eq(p1.c55, p2.c55, n_simd_elements); - check_eq(p1.rho_vp, Kokkos::sqrt(p2.rho * p2.c33), - n_simd_elements); - check_eq(p1.rho_vs, Kokkos::sqrt(p2.rho * p2.c55), - n_simd_elements); + check_eq(p1.rho(), p2.rho(), n_simd_elements, "rho"); + check_eq(p1.c11(), p2.c11(), n_simd_elements, "c11"); + check_eq(p1.c13(), p2.c13(), n_simd_elements, "c13"); + check_eq(p1.c15(), p2.c15(), n_simd_elements, "c15"); + check_eq(p1.c33(), p2.c33(), n_simd_elements, "c33"); + check_eq(p1.c35(), p2.c35(), n_simd_elements, "c35"); + check_eq(p1.c55(), p2.c55(), n_simd_elements, "c55"); + check_eq(p1.rho_vp(), Kokkos::sqrt(p2.rho() * p2.c33()), + n_simd_elements, "rho_vp"); + check_eq(p1.rho_vs(), Kokkos::sqrt(p2.rho() * p2.c55()), + n_simd_elements, "rho_vs"); } template @@ -330,18 +336,23 @@ void check_point_properties( specfem::dimension::type::dim2, specfem::element::medium_tag::acoustic, specfem::element::property_tag::isotropic, using_simd> &p2, const int &n_simd_elements) { - check_eq(p1.rho_inverse, p2.rho_inverse, n_simd_elements); - check_eq(p1.kappa, p2.kappa, n_simd_elements); + check_eq(p1.rho_inverse(), p2.rho_inverse(), n_simd_elements, + "rho_inverse"); + check_eq(p1.kappa(), p2.kappa(), n_simd_elements, "kappa"); check_eq( - p1.kappa_inverse, + p1.kappa_inverse(), (static_cast< typename specfem::datatype::simd::datatype>( 1.0)) / - p2.kappa, - n_simd_elements); - check_eq(p1.rho_vpinverse, - Kokkos::sqrt(p2.rho_inverse * p2.kappa_inverse), - n_simd_elements); + p2.kappa(), + n_simd_elements, "kappa_inverse"); + check_eq( + p1.rho_vpinverse(), + Kokkos::sqrt(p2.rho_inverse() * + (static_cast::datatype>(1.0) / + p2.kappa())), + n_simd_elements, "rho_vpinverse"); } template ; + constexpr int simd_size = specfem::datatype::simd::size(); @@ -379,14 +394,19 @@ void check_to_value(const specfem::compute::properties properties, get_point_property(ielement + j, iz, ix, properties); const type_real value = values_to_store(i); - if (point_property != value) { - std::ostringstream message; + for (int l = 0; l < PointType::nprops; l++) { + if (point_property.data[l] != value) { + std::ostringstream message; - message << "\n \t Error at ispec = " << ielement + j - << ", iz = " << iz << ", ix = " << ix; - message << get_error_message(point_property, value); + message << "\n \t Error in function check_to_value"; - throw std::runtime_error(message.str()); + message << "\n \t Error at ispec = " << ielement + j + << ", iz = " << iz << ", ix = " << ix + << ", iprop = " << l; + message << get_error_message(point_property, value); + + throw std::runtime_error(message.str()); + } } } } @@ -411,6 +431,9 @@ void check_compute_to_mesh( const int ngllz = properties.ngllz; std::vector elements; + using PointType = specfem::point::properties; + for (int ispec = 0; ispec < nspec; ispec++) { if ((element_types.get_medium_tag(ispec) == MediumTag) && (element_types.get_property_tag(ispec) == PropertyTag)) { @@ -445,17 +468,31 @@ void check_compute_to_mesh( std::get >( materials[ispec_mesh]); auto value = material.get_properties(); - if (point_property != value) { - std::ostringstream message; + for (int l = 0; l < PointType::nprops; l++) { + if (point_property.data[l] != value.data[l]) { + std::ostringstream message; - message << "\n \t Error at ispec = " << ielement << ", iz = " << iz - << ", ix = " << ix; + message << "\n \t Error in function check_compute_to_mesh"; - message << get_error_message(value, 0.0, 1); - message << get_error_message(point_property, 0.0, 2); + message << "\n \t Error at ispec = " << ielement << ", iz = " << iz + << ", ix = " << ix << ", iprop = " << l; + message << get_error_message(value, 0.0, 1); + message << get_error_message(point_property, 0.0, 2); - throw std::runtime_error(message.str()); + throw std::runtime_error(message.str()); + } } + // if (point_property != value) { + // std::ostringstream message; + + // message << "\n \t Error at ispec = " << ielement << ", iz = " << iz + // << ", ix = " << ix; + + // message << get_error_message(value, 0.0, 1); + // message << get_error_message(point_property, 0.0, 2); + + // throw std::runtime_error(message.str()); + // } } } } @@ -515,11 +552,14 @@ void check_store_on_host(specfem::compute::properties &properties, const auto index = get_index(ielement, n_simd_elements, iz, ix); const type_real value = values_to_store_h(i); - PointType point(value); + PointType point; + for (int l = 0; l < PointType::nprops; l++) { + point.data[l] = value; + } PointType point_loaded; specfem::compute::store_on_host(index, properties, point); specfem::compute::load_on_host(index, properties, point_loaded); - check_point_properties(point, point_loaded, + check_point_properties(point_loaded, point, n_simd_elements); } } @@ -614,28 +654,33 @@ void check_load_on_device(specfem::compute::properties &properties, for (int lane = 0; lane < n_simd_elements; lane++) { const auto point_property_l = get_point_property(lane, point_property); - if (point_property_l != value_l) { - std::ostringstream message; + for (int l = 0; l < PointType::nprops; l++) { + if (point_property_l.data[l] != value_l) { + std::ostringstream message; - message << "\n \t Error in function load_on_device"; + message << "\n \t Error in function load_on_device"; - message << "\n \t Error at ispec = " << ielement << ", iz = " << 0 - << ", ix = " << 0; - message << get_error_message(point_property_l, value_l); + message << "\n \t Error at ispec = " << ielement + << ", iz = " << 0 << ", ix = " << 0 + << ", iprop = " << l; + message << get_error_message(point_property_l, value_l); - throw std::runtime_error(message.str()); + throw std::runtime_error(message.str()); + } } } } else if constexpr (!using_simd) { - if (point_property != value_l) { - std::ostringstream message; - message << "\n \t Error in function load_on_device"; + for (int l = 0; l < PointType::nprops; l++) { + if (point_property.data[l] != value_l) { + std::ostringstream message; + message << "\n \t Error in function load_on_device"; - message << "\n \t Error at ispec = " << ielement << ", iz = " << 0 - << ", ix = " << 0; - message << get_error_message(point_property, value_l); + message << "\n \t Error at ispec = " << ielement << ", iz = " << 0 + << ", ix = " << 0 << ", iprop = " << l; + message << get_error_message(point_property, value_l); - throw std::runtime_error(message.str()); + throw std::runtime_error(message.str()); + } } } }